diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 690b47426a0fcb..f7d674eee43b87 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -69,20 +69,17 @@ MemTable::MemTable(int64_t tablet_id, std::shared_ptr tablet_schem _resource_ctx->memory_context()->mem_tracker()->write_tracker()); SCOPED_CONSUME_MEM_TRACKER(_mem_tracker); _vec_row_comparator = std::make_shared(_tablet_schema); - _num_columns = _tablet_schema->num_columns(); if (partial_update_info != nullptr) { _partial_update_mode = partial_update_info->update_mode(); if (_partial_update_mode == UniqueKeyUpdateModePB::UPDATE_FIXED_COLUMNS) { - _num_columns = partial_update_info->partial_update_input_columns.size(); if (partial_update_info->is_schema_contains_auto_inc_column && !partial_update_info->is_input_columns_contains_auto_inc_column) { _is_partial_update_and_auto_inc = true; - _num_columns += 1; } } } - // TODO: Support ZOrderComparator in the future _init_columns_offset_by_slot_descs(slot_descs, tuple_desc); + // TODO: Support ZOrderComparator in the future _row_in_blocks = std::make_unique>>(); _load_mem_limit = MemInfo::mem_limit() * config::load_process_max_memory_limit_percent / 100; } @@ -101,6 +98,7 @@ void MemTable::_init_columns_offset_by_slot_descs(const std::vector NULL, + `metadata` json NULL, + `status` int NULL DEFAULT "1", + `created_at` datetime NULL DEFAULT CURRENT_TIMESTAMP, + `optional_data` varchar(64) NULL, + `required_data` int NOT NULL DEFAULT "0", + `last_status` int NULL DEFAULT "0", + `col1` int NULL DEFAULT "0", + `col2` varchar(32) NULL, + `a_very_long_column_name_that_is_just_under_the_limit` int NULL, + `long_default` varchar(255) NULL DEFAULT "a_very_long_default_value_that_is_just_under_the_limit_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz", + `max_int` int NULL DEFAULT "2147483647", + `min_int` int NULL DEFAULT "-2147483648", + `high_precision` decimal(38,10) NULL DEFAULT "1234567890123456789012345678.1234567890", + `test_col` int NULL DEFAULT "0", + `consistency_check` int NULL DEFAULT "42", + INDEX idx_json_data_inverted (`user_metadata`) USING INVERTED + ) ENGINE=OLAP + UNIQUE KEY(`user_id`, `event_date`, `event_time`) + PARTITION BY RANGE(`event_date`) + (PARTITION p2015 VALUES [('2015-01-01'), ('2016-01-01')), + PARTITION p2016 VALUES [('2016-01-01'), ('2017-01-01')), + PARTITION p2017 VALUES [('2017-01-01'), ('2018-01-01')), + PARTITION p2018 VALUES [('2018-01-01'), ('2019-01-01')), + PARTITION p2019 VALUES [('2019-01-01'), ('2020-01-01')), + PARTITION p2020 VALUES [('2020-01-01'), ('2021-01-01')), + PARTITION p2021 VALUES [('2021-01-01'), ('2022-01-01')), + PARTITION p2022 VALUES [('2022-01-01'), ('2023-01-01')), + PARTITION p2023 VALUES [('2023-01-01'), ('2024-01-01')), + PARTITION p2024 VALUES [('2024-01-01'), ('2025-01-01')), + PARTITION p2025 VALUES [('2025-01-01'), ('2026-01-01')), + PARTITION p2026 VALUES [('2026-01-01'), ('2027-01-01')), + PARTITION p2027 VALUES [('2027-01-01'), ('2028-01-01')), + PARTITION p2028 VALUES [('2028-01-01'), ('2029-01-01')), + PARTITION p2029 VALUES [('2029-01-01'), ('2030-01-01')), + PARTITION p2030 VALUES [('2030-01-01'), ('2030-12-31')), + PARTITION p2031 VALUES [('2031-01-01'), ('2032-01-01')), + PARTITION p2032 VALUES [('2032-01-01'), ('2033-01-01'))) + DISTRIBUTED BY HASH(`user_id`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "min_load_replica_num" = "-1", + "bloom_filter_columns" = "country, city", + "is_being_synced" = "false", + "dynamic_partition.enable" = "true", + "dynamic_partition.time_unit" = "YEAR", + "dynamic_partition.time_zone" = "Asia/Shanghai", + "dynamic_partition.start" = "-10", + "dynamic_partition.end" = "7", + "dynamic_partition.prefix" = "p", + "dynamic_partition.replication_allocation" = "tag.location.default: 1", + "dynamic_partition.buckets" = "3", + "dynamic_partition.create_history_partition" = "false", + "dynamic_partition.history_partition_num" = "-1", + "dynamic_partition.hot_partition_num" = "0", + "dynamic_partition.reserved_history_periods" = "NULL", + "dynamic_partition.storage_policy" = "", + "storage_medium" = "hdd", + "storage_format" = "V2", + "inverted_index_storage_format" = "V3", + "enable_unique_key_merge_on_write" = "true", + "light_schema_change" = "true", + "disable_auto_compaction" = "false", + "enable_single_replica_compaction" = "false", + "group_commit_interval_ms" = "10000", + "group_commit_data_bytes" = "134217728", + "enable_mow_light_delete" = "false" + ); + ''' + + sql ''' + INSERT INTO mow_table(user_id,event_date,event_time,country,city) VALUES + (2000,'2025-09-22','2025-09-22 10:00:00','China','Shanghai'), + (2001,'2025-09-22','2025-09-22 11:00:00','USA','NewYork'), + (2002,'2025-09-22','2025-09-22 12:00:00','US','London'); + ''' + + qt_sql_before 'select user_id, event_date, country, city from mow_table order by user_id' + + // Add a rollup that's missing the 'city' column + sql """ + ALTER TABLE mow_table ADD ROLLUP rollup1(event_date, event_time, user_id, country, update_time) + """ + sleep(10000) + explain { + sql(''' + SELECT event_date, country, count(*) + FROM mow_table + WHERE event_date = '2025-09-22' + GROUP BY event_date, country + ''') + contains "rollup1" + } + + // Test 1: UPDATE column not in rollup1 (city) + sql 'UPDATE mow_table SET city = "beijing" WHERE user_id = 2000' + def result1 = sql 'SELECT city FROM mow_table WHERE user_id = 2000' + assertEquals(1, result1.size()) + assertEquals('beijing', result1[0][0]) + + // Test 2: UPDATE column in rollup1 (country) + sql 'UPDATE mow_table SET country = "CN" WHERE user_id = 2000' + def result2 = sql 'SELECT country FROM mow_table WHERE user_id = 2000' + assertEquals(1, result2.size()) + assertEquals('CN', result2[0][0]) + + // Test 3: UPDATE both columns (one in rollup, one not) + sql 'UPDATE mow_table SET city = "Shenzhen", country = "China" WHERE user_id = 2001' + def result3 = sql 'SELECT city, country FROM mow_table WHERE user_id = 2001' + assertEquals(1, result3.size()) + assertEquals('Shenzhen', result3[0][0]) + assertEquals('China', result3[0][1]) + + qt_sql_after ''' + SELECT user_id, event_date, country, city + FROM mow_table + ORDER BY user_id + ''' + + sql 'drop table if exists mow_table' +}