branch-3.1: [enhancement](time_series) increase the version limit for the time series table #51371

be/src/agent/task_worker_pool.cpp

@@ -1893,8 +1893,9 @@ void PublishVersionWorkerPool::publish_version_callback(const TAgentTaskRequest&
                     if (!tablet->tablet_meta()->tablet_schema()->disable_auto_compaction()) {
                         tablet->published_count.fetch_add(1);
                         int64_t published_count = tablet->published_count.load();
+                        int32_t max_version_config = tablet->max_version_config();
                         if (tablet->exceed_version_limit(
-                                    config::max_tablet_version_num *
+                                    max_version_config *
                                     config::load_trigger_compaction_version_percent / 100) &&
                             published_count % 20 == 0) {
                             auto st = _engine.submit_compaction_task(

be/src/cloud/cloud_delete_task.cpp

@@ -48,14 +48,17 @@ Status CloudDeleteTask::execute(CloudStorageEngine& engine, const TPushReq& requ
     tablet->last_load_time_ms =
             duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
     // check if version number exceed limit
-    if (tablet->fetch_add_approximate_num_rowsets(0) > config::max_tablet_version_num) {
+
+    int32_t max_version_config = tablet->max_version_config();
+    if (tablet->fetch_add_approximate_num_rowsets(0) > max_version_config) {
         LOG_WARNING("tablet exceeds max version num limit")
-                .tag("limit", config::max_tablet_version_num)
+                .tag("limit", max_version_config)
                 .tag("tablet_id", tablet->tablet_id());
         return Status::Error<TOO_MANY_VERSION>(
                 "too many versions, versions={} tablet={}. Please reduce the frequency of loading "
-                "data or adjust the max_tablet_version_num in be.conf to a larger value.",
-                config::max_tablet_version_num, tablet->tablet_id());
+                "data or adjust the max_tablet_version_num or time_series_max_tablet_version_num "
+                "in be.conf to a larger value.",
+                max_version_config, tablet->tablet_id());
     }
 
     // check delete condition if push for delete

be/src/cloud/cloud_rowset_builder.cpp

@@ -89,12 +89,14 @@ Status CloudRowsetBuilder::init() {
 Status CloudRowsetBuilder::check_tablet_version_count() {
     int version_count = cloud_tablet()->fetch_add_approximate_num_rowsets(0);
     // TODO(plat1ko): load backoff algorithm
-    if (version_count > config::max_tablet_version_num) {
+    int32_t max_version_config = cloud_tablet()->max_version_config();
+    if (version_count > max_version_config) {
         return Status::Error<TOO_MANY_VERSION>(
                 "failed to init rowset builder. version count: {}, exceed limit: {}, "
                 "tablet: {}. Please reduce the frequency of loading data or adjust the "
-                "max_tablet_version_num in be.conf to a larger value.",
-                version_count, config::max_tablet_version_num, _tablet->tablet_id());
+                "max_tablet_version_num or time_series_max_tablet_version_numin be.conf to a "
+                "larger value.",
+                version_count, max_version_config, _tablet->tablet_id());
     }
     return Status::OK();
 }

be/src/cloud/cloud_tablet_mgr.cpp

@@ -343,11 +343,13 @@ Status CloudTabletMgr::get_topn_tablets_to_compact(
         }
         // If tablet has too many rowsets but not be compacted for a long time, compaction should be performed
         // regardless of whether there is a load job recently.
+
+        int32_t max_version_config = t->max_version_config();
         return now - t->last_cumu_compaction_failure_time() < config::min_compaction_failure_interval_ms ||
                now - t->last_cumu_no_suitable_version_ms < config::min_compaction_failure_interval_ms ||
                (now - t->last_load_time_ms > config::cu_compaction_freeze_interval_s * 1000
                && now - t->last_cumu_compaction_success_time_ms < config::cumu_compaction_interval_s * 1000
-               && t->fetch_add_approximate_num_rowsets(0) < config::max_tablet_version_num / 2);
+               && t->fetch_add_approximate_num_rowsets(0) < max_version_config / 2);
     };
     // We don't schedule tablets that are disabled for compaction
     auto disable = [](CloudTablet* t) { return t->tablet_meta()->tablet_schema()->disable_auto_compaction(); };

be/src/common/config.cpp

@@ -824,6 +824,8 @@ DEFINE_Int32(query_cache_max_partition_count, "1024");
 // This is to avoid too many version num.
 DEFINE_mInt32(max_tablet_version_num, "2000");
 
+DEFINE_mInt32(time_series_max_tablet_version_num, "20000");
+
 // Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR. if fe use THREADED_SELECTOR model for thrift server,
 // the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be thrift client to fe constructed with TFramedTransport
 DEFINE_String(thrift_server_type_of_fe, "THREAD_POOL");

be/src/common/config.h

@@ -868,6 +868,8 @@ DECLARE_Int32(query_cache_max_partition_count);
 // This is to avoid too many version num.
 DECLARE_mInt32(max_tablet_version_num);
 
+DECLARE_mInt32(time_series_max_tablet_version_num);
+
 // Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR. if fe use THREADED_SELECTOR model for thrift server,
 // the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be thrift client to fe constructed with TFramedTransport
 DECLARE_String(thrift_server_type_of_fe);

be/src/olap/base_tablet.cpp

@@ -29,6 +29,7 @@
 #include "common/logging.h"
 #include "common/status.h"
 #include "olap/calc_delete_bitmap_executor.h"
+#include "olap/cumulative_compaction_time_series_policy.h"
 #include "olap/delete_bitmap_calculator.h"
 #include "olap/iterators.h"
 #include "olap/memtable.h"
@@ -1736,4 +1737,11 @@ void BaseTablet::get_base_rowset_delete_bitmap_count(
     }
 }
 
+int32_t BaseTablet::max_version_config() {
+    int32_t max_version = tablet_meta()->compaction_policy() == CUMULATIVE_TIME_SERIES_POLICY
+                                  ? config::time_series_max_tablet_version_num
+                                  : config::max_tablet_version_num;
+    return max_version;
+}
+
 } // namespace doris

be/src/olap/base_tablet.h

@@ -79,6 +79,8 @@ class BaseTablet {
     // Property encapsulated in TabletMeta
     const TabletMetaSharedPtr& tablet_meta() { return _tablet_meta; }
 
+    int32 max_version_config();
+
     // FIXME(plat1ko): It is not appropriate to expose this lock
     std::shared_mutex& get_header_lock() { return _meta_lock; }
 

be/src/olap/olap_server.cpp

@@ -1702,8 +1702,9 @@ void StorageEngine::_process_async_publish() {
                 continue;
             }
             if (version != max_version + 1) {
+                int32_t max_version_config = tablet->max_version_config();
                 // Keep only the most recent versions
-                while (tablet_iter->second.size() > config::max_tablet_version_num) {
+                while (tablet_iter->second.size() > max_version_config) {
                     need_removed_tasks.emplace_back(tablet, version);
                     task_iter = tablet_iter->second.erase(task_iter);
                     version = task_iter->first;

be/src/olap/push_handler.cpp

@@ -39,6 +39,7 @@
 #include "common/config.h"
 #include "common/logging.h"
 #include "common/status.h"
+#include "olap/cumulative_compaction_time_series_policy.h"
 #include "olap/delete_handler.h"
 #include "olap/olap_define.h"
 #include "olap/rowset/pending_rowset_helper.h"
@@ -160,13 +161,15 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR
         }
     }
 
+    int32_t max_version_config = tablet->max_version_config();
     // check if version number exceed limit
-    if (tablet->exceed_version_limit(config::max_tablet_version_num)) {
+    if (tablet->exceed_version_limit(max_version_config)) {
         return Status::Status::Error<TOO_MANY_VERSION>(
                 "failed to push data. version count: {}, exceed limit: {}, tablet: {}. Please "
-                "reduce the frequency of loading data or adjust the max_tablet_version_num in "
+                "reduce the frequency of loading data or adjust the max_tablet_version_num or "
+                "time_series_max_tablet_version_num in "
                 "be.conf to a larger value.",
-                tablet->version_count(), config::max_tablet_version_num, tablet->tablet_id());
+                tablet->version_count(), max_version_config, tablet->tablet_id());
     }
 
     int version_count = tablet->version_count() + tablet->stale_version_count();

be/src/olap/rowset_builder.cpp

@@ -151,9 +151,10 @@ Status RowsetBuilder::check_tablet_version_count() {
     bool injection = false;
     DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
                     { injection = true; });
+    int32_t max_version_config = _tablet->max_version_config();
     if (injection) {
         // do not return if injection
-    } else if (!_tablet->exceed_version_limit(config::max_tablet_version_num - 100) ||
+    } else if (!_tablet->exceed_version_limit(max_version_config - 100) ||
                GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
         return Status::OK();
     }
@@ -167,12 +168,13 @@ Status RowsetBuilder::check_tablet_version_count() {
     int version_count = tablet()->version_count();
     DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
                     { version_count = INT_MAX; });
-    if (version_count > config::max_tablet_version_num) {
+    if (version_count > max_version_config) {
         return Status::Error<TOO_MANY_VERSION>(
                 "failed to init rowset builder. version count: {}, exceed limit: {}, "
                 "tablet: {}. Please reduce the frequency of loading data or adjust the "
-                "max_tablet_version_num in be.conf to a larger value.",
-                version_count, config::max_tablet_version_num, _tablet->tablet_id());
+                "max_tablet_version_num or time_series_max_tablet_version_num in be.conf to a "
+                "larger value.",
+                version_count, max_version_config, _tablet->tablet_id());
     }
     return Status::OK();
 }