diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp index 74c76a2bb90800..1960a16b41ba1f 100644 --- a/be/src/olap/delta_writer.cpp +++ b/be/src/olap/delta_writer.cpp @@ -139,9 +139,10 @@ OLAPStatus DeltaWriter::init() { writer_context.tablet_id = _req.tablet_id; writer_context.partition_id = _req.partition_id; writer_context.tablet_schema_hash = _req.schema_hash; - writer_context.rowset_type = _storage_engine->default_rowset_type(); if (_tablet->tablet_meta()->preferred_rowset_type() == BETA_ROWSET) { writer_context.rowset_type = BETA_ROWSET; + } else { + writer_context.rowset_type = ALPHA_ROWSET; } writer_context.rowset_path_prefix = _tablet->tablet_path(); writer_context.tablet_schema = &(_tablet->tablet_schema()); diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp index fa5c6bd28c0b83..d7e65deb179436 100644 --- a/be/src/olap/push_handler.cpp +++ b/be/src/olap/push_handler.cpp @@ -282,6 +282,9 @@ OLAPStatus PushHandler::_convert_v2(TabletSharedPtr cur_tablet, context.partition_id = _request.partition_id; context.tablet_schema_hash = cur_tablet->schema_hash(); context.rowset_type = StorageEngine::instance()->default_rowset_type(); + if (cur_tablet->tablet_meta()->preferred_rowset_type() == BETA_ROWSET) { + context.rowset_type = BETA_ROWSET; + } context.rowset_path_prefix = cur_tablet->tablet_path(); context.tablet_schema = &(cur_tablet->tablet_schema()); context.rowset_state = PREPARED; @@ -468,6 +471,9 @@ OLAPStatus PushHandler::_convert(TabletSharedPtr cur_tablet, context.partition_id = _request.partition_id; context.tablet_schema_hash = cur_tablet->schema_hash(); context.rowset_type = StorageEngine::instance()->default_rowset_type(); + if (cur_tablet->tablet_meta()->preferred_rowset_type() == BETA_ROWSET) { + context.rowset_type = BETA_ROWSET; + } context.rowset_path_prefix = cur_tablet->tablet_path(); context.tablet_schema = &(cur_tablet->tablet_schema()); context.rowset_state = PREPARED; diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 5e84aa59577066..626c1f88bd675e 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -35,7 +35,6 @@ #include "olap/tablet.h" #include "olap/wrapper_field.h" #include "runtime/exec_env.h" -#include "runtime/heartbeat_flags.h" #include "runtime/mem_pool.h" #include "runtime/mem_tracker.h" diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 34608762ee551b..bcc6e6a5e98718 100755 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -90,7 +90,7 @@ TabletMeta::TabletMeta(int64_t table_id, int64_t partition_id, tablet_meta_pb.set_cumulative_layer_point(-1); tablet_meta_pb.set_tablet_state(PB_RUNNING); *(tablet_meta_pb.mutable_tablet_uid()) = tablet_uid.to_proto(); - tablet_meta_pb.set_tablet_type(tabletType == TTabletType::TABLET_TYPE_MEMORY ? + tablet_meta_pb.set_tablet_type(tabletType == TTabletType::TABLET_TYPE_DISK ? TabletTypePB::TABLET_TYPE_DISK : TabletTypePB::TABLET_TYPE_MEMORY); TabletSchemaPB* schema = tablet_meta_pb.mutable_schema(); schema->set_num_short_key_columns(tablet_schema.short_key_column_count); diff --git a/docs/en/administrator-guide/alter-table/alter-table-bitmap-index.md b/docs/en/administrator-guide/alter-table/alter-table-bitmap-index.md index 4b4d7d326fec62..c12c82318a113e 100644 --- a/docs/en/administrator-guide/alter-table/alter-table-bitmap-index.md +++ b/docs/en/administrator-guide/alter-table/alter-table-bitmap-index.md @@ -77,7 +77,4 @@ Please refer to [Scheam Change](alter-table-schema-change.html) * `LARGEINT` * `DECIMAL` * `BOOL` -* The bitmap index takes effect only in segmentV2. You need to add the following configuration to the configuration file of be - ``` - default_rowset_type=BETA - ``` +* The bitmap index takes effect only in segmentV2. The table's storage format will be converted to V2 automatically when creating index. diff --git a/docs/zh-CN/administrator-guide/alter-table/alter-table-bitmap-index.md b/docs/zh-CN/administrator-guide/alter-table/alter-table-bitmap-index.md index 0a768a99716e80..6ead33fbc76cef 100644 --- a/docs/zh-CN/administrator-guide/alter-table/alter-table-bitmap-index.md +++ b/docs/zh-CN/administrator-guide/alter-table/alter-table-bitmap-index.md @@ -79,8 +79,4 @@ create/drop index 语法 * `DECIMAL` * `BOOL` -* bitmap索引仅在 segmentV2 下生效，需要在be的配置文件中增加如下配置 - - ``` - default_rowset_type=BETA - ``` +* bitmap索引仅在 Segment V2 下生效。当创建 index 时，表的存储格式将默认转换为 V2 格式。 diff --git a/docs/zh-CN/administrator-guide/segment-v2-usage.md b/docs/zh-CN/administrator-guide/segment-v2-usage.md index c3a15ccbd84ac4..1309182587c90d 100644 --- a/docs/zh-CN/administrator-guide/segment-v2-usage.md +++ b/docs/zh-CN/administrator-guide/segment-v2-usage.md @@ -42,6 +42,8 @@ V2 格式的表可以支持以下新的特性： 4. 字典压缩 5. 延迟物化（Lazy Materialization） +**从 0.13 版本开始，新建表的默认存储格式将为 Segment V2** + ## 集群升级 0.12 版本仅支持从 0.11 版本升级，不支持从 0.11 之前的版本升级。请先确保升级的前的 Doris 集群版本为 0.11。 @@ -129,9 +131,10 @@ PROPERTIES 在 `properties` 中指定 `"storage_format" = "v2"` 后，该表将使用 V2 格式创建。如果是分区表，则之后创建的分区也都是 V2 格式。 -### 全量格式转换(试验功能) +### 全量格式转换(试验功能，不推荐) 通过以下方式可以开启整个集群的全量数据格式转换（V1 -> V2）。全量数据转换是通过 BE 后台的数据 compaction 过程异步进行的。 +**该功能目前并没有很好的方式查看或控制转换进度，并且无法保证数据能够转换完成。可能导致同一张表长期处于同时包含两种数据格式的状态。因此建议使用 ALTER TABLE 针对性的转换。** 1. 从 BE 开启全量格式转换 diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java index 358c21964e396b..50e679a70d96ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java @@ -341,6 +341,9 @@ private RollupJobV2 createMaterializedViewJob(String mvName, String baseIndexNam String newStorageFormatIndexName = NEW_STORAGE_FORMAT_INDEX_NAME_PREFIX + olapTable.getName(); if (mvName.equals(newStorageFormatIndexName)) { mvJob.setStorageFormat(TStorageFormat.V2); + } else { + // use base table's storage foramt as the mv's format + mvJob.setStorageFormat(olapTable.getStorageFormat()); } /* diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java index 6345f458020750..39c80391074777 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java @@ -884,6 +884,10 @@ private void createJob(long dbId, OlapTable olapTable, Map properties storageFormat = properties.get(PROPERTIES_STORAGE_FORMAT); properties.remove(PROPERTIES_STORAGE_FORMAT); } else { - return TStorageFormat.DEFAULT; + return TStorageFormat.V2; } if (storageFormat.equalsIgnoreCase("v1")) { @@ -408,7 +408,7 @@ public static TStorageFormat analyzeStorageFormat(Map properties } else if (storageFormat.equalsIgnoreCase("v2")) { return TStorageFormat.V2; } else if (storageFormat.equalsIgnoreCase("default")) { - return TStorageFormat.DEFAULT; + return TStorageFormat.V2; } else { throw new AnalysisException("unknown storage format: " + storageFormat); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java index 543fe0568b1023..0c0e3f494abd09 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java +++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterJobV2Test.java @@ -62,8 +62,8 @@ public static void beforeClass() throws Exception { Catalog.getCurrentCatalog().createDb(createDbStmt); createTable("CREATE TABLE test.schema_change_test(k1 int, k2 int, k3 int) distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); - - createTable("CREATE TABLE test.segmentv2(k1 int, k2 int, v1 int sum) distributed by hash(k1) buckets 3 properties('replication_num' = '1');"); + + createTable("CREATE TABLE test.segmentv2(k1 int, k2 int, v1 int sum) distributed by hash(k1) buckets 3 properties('replication_num' = '1', 'storage_format' = 'v1');"); } @AfterClass @@ -133,7 +133,7 @@ public void testAlterSegmentV2() throws Exception { Assert.assertNotNull(db); OlapTable tbl = (OlapTable) db.getTable("segmentv2"); Assert.assertNotNull(tbl); - Assert.assertEquals(TStorageFormat.DEFAULT, tbl.getTableProperty().getStorageFormat()); + Assert.assertEquals(TStorageFormat.V1, tbl.getTableProperty().getStorageFormat()); // 1. create a rollup r1 String alterStmtStr = "alter table test.segmentv2 add rollup r1(k2, v1)";