[enhancement](time_series) increase the version limit for the time series table (#51371)

be/src/agent/task_worker_pool.cpp

@@ -1594,8 +1594,9 @@ void PublishVersionWorkerPool::publish_version_callback(const TAgentTaskRequest&
                     if (!tablet->tablet_meta()->tablet_schema()->disable_auto_compaction()) {
                         tablet->published_count.fetch_add(1);
                         int64_t published_count = tablet->published_count.load();
+                        int32_t max_version_config = tablet->max_version_config();
                         if (tablet->exceed_version_limit(
-                                    config::max_tablet_version_num *
+                                    max_version_config *
                                     config::load_trigger_compaction_version_percent / 100) &&
                             published_count % 20 == 0) {
                             auto st = _engine.submit_compaction_task(

be/src/common/config.cpp

@@ -809,6 +809,8 @@ DEFINE_Int32(query_cache_max_partition_count, "1024");
 // This is to avoid too many version num.
 DEFINE_mInt32(max_tablet_version_num, "2000");
 
+DEFINE_mInt32(time_series_max_tablet_version_num, "20000");
+
 // Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR. if fe use THREADED_SELECTOR model for thrift server,
 // the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be thrift client to fe constructed with TFramedTransport
 DEFINE_String(thrift_server_type_of_fe, "THREAD_POOL");

be/src/common/config.h

@@ -854,6 +854,8 @@ DECLARE_Int32(query_cache_max_partition_count);
 // This is to avoid too many version num.
 DECLARE_mInt32(max_tablet_version_num);
 
+DECLARE_mInt32(time_series_max_tablet_version_num);
+
 // Frontend mainly use two thrift sever type: THREAD_POOL, THREADED_SELECTOR. if fe use THREADED_SELECTOR model for thrift server,
 // the thrift_server_type_of_fe should be set THREADED_SELECTOR to make be thrift client to fe constructed with TFramedTransport
 DECLARE_String(thrift_server_type_of_fe);

be/src/olap/base_tablet.cpp

@@ -19,6 +19,7 @@
 
 #include <fmt/format.h>
 
+#include "olap/cumulative_compaction_time_series_policy.h"
 #include "olap/tablet_fwd.h"
 #include "olap/tablet_schema_cache.h"
 #include "util/doris_metrics.h"
@@ -91,4 +92,11 @@ uint32_t BaseTablet::get_real_compaction_score() const {
                            });
 }
 
-} /* namespace doris */
+int32_t BaseTablet::max_version_config() {
+    int32_t max_version = tablet_meta()->compaction_policy() == CUMULATIVE_TIME_SERIES_POLICY
+                                  ? config::time_series_max_tablet_version_num
+                                  : config::max_tablet_version_num;
+    return max_version;
+}
+
+} // namespace doris

be/src/olap/base_tablet.h

@@ -64,6 +64,8 @@ class BaseTablet {
     // Property encapsulated in TabletMeta
     const TabletMetaSharedPtr& tablet_meta() { return _tablet_meta; }
 
+    int32 max_version_config();
+
     // FIXME(plat1ko): It is not appropriate to expose this lock
     std::shared_mutex& get_header_lock() { return _meta_lock; }
 

be/src/olap/olap_server.cpp

@@ -1611,8 +1611,9 @@ void StorageEngine::_process_async_publish() {
                 continue;
             }
             if (version != max_version + 1) {
+                int32_t max_version_config = tablet->max_version_config();
                 // Keep only the most recent versions
-                while (tablet_iter->second.size() > config::max_tablet_version_num) {
+                while (tablet_iter->second.size() > max_version_config) {
                     need_removed_tasks.emplace_back(tablet, version);
                     task_iter = tablet_iter->second.erase(task_iter);
                     version = task_iter->first;

be/src/olap/push_handler.cpp

@@ -39,6 +39,7 @@
 #include "common/config.h"
 #include "common/logging.h"
 #include "common/status.h"
+#include "olap/cumulative_compaction_time_series_policy.h"
 #include "olap/delete_handler.h"
 #include "olap/olap_define.h"
 #include "olap/rowset/pending_rowset_helper.h"
@@ -161,13 +162,15 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR
         }
     }
 
+    int32_t max_version_config = tablet->max_version_config();
     // check if version number exceed limit
-    if (tablet->exceed_version_limit(config::max_tablet_version_num)) {
+    if (tablet->exceed_version_limit(max_version_config)) {
         return Status::Status::Error<TOO_MANY_VERSION>(
                 "failed to push data. version count: {}, exceed limit: {}, tablet: {}. Please "
-                "reduce the frequency of loading data or adjust the max_tablet_version_num in "
+                "reduce the frequency of loading data or adjust the max_tablet_version_num or "
+                "time_series_max_tablet_version_num in "
                 "be.conf to a larger value.",
-                tablet->version_count(), config::max_tablet_version_num, tablet->tablet_id());
+                tablet->version_count(), max_version_config, tablet->tablet_id());
     }
 
     int version_count = tablet->version_count() + tablet->stale_version_count();

be/src/olap/rowset_builder.cpp

@@ -144,9 +144,10 @@ Status RowsetBuilder::check_tablet_version_count() {
     bool injection = false;
     DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
                     { injection = true; });
+    int32_t max_version_config = _tablet->max_version_config();
     if (injection) {
         // do not return if injection
-    } else if (!_tablet->exceed_version_limit(config::max_tablet_version_num - 100) ||
+    } else if (!_tablet->exceed_version_limit(max_version_config - 100) ||
                GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
         return Status::OK();
     }
@@ -160,12 +161,13 @@ Status RowsetBuilder::check_tablet_version_count() {
     int version_count = tablet()->version_count();
     DBUG_EXECUTE_IF("RowsetBuilder.check_tablet_version_count.too_many_version",
                     { version_count = INT_MAX; });
-    if (version_count > config::max_tablet_version_num) {
+    if (version_count > max_version_config) {
         return Status::Error<TOO_MANY_VERSION>(
                 "failed to init rowset builder. version count: {}, exceed limit: {}, "
                 "tablet: {}. Please reduce the frequency of loading data or adjust the "
-                "max_tablet_version_num in be.conf to a larger value.",
-                version_count, config::max_tablet_version_num, _tablet->tablet_id());
+                "max_tablet_version_num or time_series_max_tablet_version_num in be.conf to a "
+                "larger value.",
+                version_count, max_version_config, _tablet->tablet_id());
     }
     return Status::OK();
 }