Skip to content

Commit

Permalink
[Fix](memtable) fix shrink_memtable_by_agg should also update `_row…
Browse files Browse the repository at this point in the history
…_in_blocks` (apache#28536)

Otherwise using the stale `_row_in_blocks` will result in heap-buffer-overflow

```
==2695213==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x62900122e210 at pc 0x56524744aecf bp 0x7f62c595ef7
0 sp 0x7f62c595ef68
READ of size 8 at 0x62900122e210 thread T1627 (MemTableFlushTh)
    #0 0x56524744aece in doris::vectorized::ColumnVector<long>::insert_indices_from(doris::vectorized::IColumn const&, unsigned int const*, unsigned int const*) /mnt/disk2/lihangyu/doris/be/src/vec/columns/column_vector.cpp:378:33
    #1 0x5652472a7538 in doris::vectorized::ColumnNullable::insert_indices_from(doris::vectorized::IColumn const&, unsigned int const*, unsigned int const*) /mnt/disk2/lihangyu/doris/be/src/vec/columns/column_nullable.cpp:310:25
    #2 0x56524782a62a in doris::vectorized::MutableBlock::add_rows(doris::vectorized::Block const*, unsigned int const*, unsigned int const*) /mnt/disk2/lihangyu/doris/be/src/vec/core/block.cpp:961:14
    #3 0x565233f187ae in doris::MemTable::_put_into_output(doris::vectorized::Block&) /mnt/disk2/lihangyu/doris/be/src/olap/memtable.cpp:248:27
    #4 0x565233f1db66 in doris::MemTable::to_block() /mnt/disk2/lihangyu/doris/be/src/olap/memtable.cpp:496:13
    #5 0x565233efae60 in doris::FlushToken::_do_flush_memtable(doris::MemTable*, int, long*) /mnt/disk2/lihangyu/doris/be/src/olap/memtable_flush_executor.cpp:121:62
    apache#6 0x565233efc8d6 in doris::FlushToken::_flush_memtable(doris::MemTable*, int, long) /mnt/disk2/lihangyu/doris/be/src/olap/memtable_flush_executor.cpp:150:16
    apache#7 0x565233f0c5eb in doris::MemtableFlushTask::run() /mnt/disk2/lihangyu/doris/be/src/olap/memtable_flush_executor.cpp:58:23
```
  • Loading branch information
eldenmoon authored Dec 18, 2023
1 parent b06f3ed commit d11365d
Show file tree
Hide file tree
Showing 2 changed files with 23 additions and 1 deletion.
2 changes: 2 additions & 0 deletions be/src/olap/memtable.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -451,6 +451,8 @@ void MemTable::_aggregate() {
_output_mutable_block =
vectorized::MutableBlock::build_mutable_block(empty_input_block.get());
_output_mutable_block.clear_column_data();
_row_in_blocks = temp_row_in_blocks;
_last_sorted_pos = _row_in_blocks.size();
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.

suite("test_insert_with_aggregation_memtable") {
suite("test_insert_with_aggregation_memtable", "nonConcurrent") {
def backendId_to_backendIP = [:]
def backendId_to_backendHttpPort = [:]
def backendId_to_params = [string:[:]]
Expand Down Expand Up @@ -114,6 +114,26 @@ suite("test_insert_with_aggregation_memtable") {
sql "sync"
qt_sql "select * from ${testTable} order by id asc"

// test with mv
def table_name = "agg_shrink"
sql "DROP TABLE IF EXISTS ${table_name}"
sql """
CREATE TABLE IF NOT EXISTS ${table_name} (
k bigint,
v text
)
DUPLICATE KEY(`k`)
DISTRIBUTED BY HASH(k) BUCKETS 4
properties("replication_num" = "1");
"""
set_be_param("write_buffer_size_for_agg", "10240") // change it to 10KB
sql """INSERT INTO ${table_name} SELECT *, '{"k1":1, "k2": "hello world", "k3" : [1234], "k4" : 1.10000, "k5" : [[123]]}' FROM numbers("number" = "4096")"""
sql """INSERT INTO ${table_name} SELECT k, v from ${table_name}"""
sql """INSERT INTO ${table_name} SELECT k, v from ${table_name}"""
createMV("""create materialized view var_cnt as select k, count(k) from ${table_name} group by k""")
sql """INSERT INTO ${table_name} SELECT k, v from ${table_name} limit 8101"""

reset_be_param("enable_shrink_memory")
reset_be_param("write_buffer_size_for_agg")

}

0 comments on commit d11365d

Please sign in to comment.