Skip to content

[opt](outfile) support compressed csv with names and types in outfile #58090

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
morningman merged 5 commits into from
Nov 20, 2025
Merged

[opt](outfile) support compressed csv with names and types in outfile #58090

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9254a8c
[opt](outfile) support compressed csv with names and types in outfile
morningman Nov 17, 2025
81dfc3d
add test
morningman Nov 17, 2025
9bd6170
fix case
morningman Nov 18, 2025
ad02864
3
morningman Nov 18, 2025
f3866e4
add bom case
morningman Nov 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions be/src/vec/runtime/vcsv_transformer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,17 +86,26 @@ VCSVTransformer::VCSVTransformer(RuntimeState* state, doris::io::FileWriter* fil
Status VCSVTransformer::open() {
RETURN_IF_ERROR(get_block_compression_codec(_compress_type, &_compress_codec));
if (_with_bom) {
Slice bom_slice(reinterpret_cast<const char*>(bom), sizeof(bom));
if (_compress_codec) {
return Status::InternalError("compressed csv with bom is not supported yet");
faststring compressed_data;
RETURN_IF_ERROR(_compress_codec->compress(bom_slice, &compressed_data));
RETURN_IF_ERROR(
_file_writer->append(Slice(compressed_data.data(), compressed_data.size())));
} else {
RETURN_IF_ERROR(_file_writer->append(bom_slice));
}
RETURN_IF_ERROR(
_file_writer->append(Slice(reinterpret_cast<const char*>(bom), sizeof(bom))));
}
if (!_csv_header.empty()) {
Slice header(_csv_header.data(), _csv_header.size());
if (_compress_codec) {
return Status::InternalError("compressed csv with header is not supported yet");
faststring compressed_data;
RETURN_IF_ERROR(_compress_codec->compress(header, &compressed_data));
RETURN_IF_ERROR(
_file_writer->append(Slice(compressed_data.data(), compressed_data.size())));
} else {
return _file_writer->append(header);
}
return _file_writer->append(Slice(_csv_header.data(), _csv_header.size()));
}
return Status::OK();
}
Expand Down
240 changes: 240 additions & 0 deletions regression-test/data/export_p0/test_outfile_csv_compress.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,234 @@ c2 text Yes false \N NONE
c1 text Yes false \N NONE
c2 text Yes false \N NONE

-- !select --
1 zhangsan
1 zhangsan1
10 zhangsan10
10 zhangsan110
10 zhangsan1210
10 zhangsan12410
10 zhangsan12510
10 zhangsan1310
10 zhangsan13610
10 zhangsan1410

-- !select --
1048576 1048576

-- !select --
id text Yes false \N NONE
name text Yes false \N NONE

-- !select --
1 zhangsan
1 zhangsan1
10 zhangsan10
10 zhangsan110
10 zhangsan1210
10 zhangsan12410
10 zhangsan12510
10 zhangsan1310
10 zhangsan13610
10 zhangsan1410

-- !select --
1048576 1048576

-- !select --
id text Yes false \N NONE
name text Yes false \N NONE

-- !select --
1 zhangsan
1 zhangsan1
10 zhangsan10
10 zhangsan110
10 zhangsan1210
10 zhangsan12410
10 zhangsan12510
10 zhangsan1310
10 zhangsan13610
10 zhangsan1410

-- !select --
1048576 1048576

-- !select --
id text Yes false \N NONE
name text Yes false \N NONE

-- !select --
1 zhangsan
1 zhangsan1
10 zhangsan10
10 zhangsan110
10 zhangsan1210
10 zhangsan12410
10 zhangsan12510
10 zhangsan1310
10 zhangsan13610
10 zhangsan1410

-- !select --
1048576 1048576

-- !select --
id text Yes false \N NONE
name text Yes false \N NONE

-- !select --
1 zhangsan
1 zhangsan1
10 zhangsan10
10 zhangsan110
10 zhangsan1210
10 zhangsan12410
10 zhangsan12510
10 zhangsan1310
10 zhangsan13610
10 zhangsan1410

-- !select --
1048576 1048576

-- !select --
id text Yes false \N NONE
name text Yes false \N NONE

-- !select --
1 zhangsan
1 zhangsan1
10 zhangsan10
10 zhangsan110
10 zhangsan1210
10 zhangsan12410
10 zhangsan12510
10 zhangsan1310
10 zhangsan13610
10 zhangsan1410

-- !select --
1048576 1048576

-- !select --
id text Yes false \N NONE
name text Yes false \N NONE

-- !select --
1 zhangsan
1 zhangsan1
10 zhangsan10
10 zhangsan110
10 zhangsan1210
10 zhangsan12410
10 zhangsan12510
10 zhangsan1310
10 zhangsan13610
10 zhangsan1410

-- !select --
1048576 1048576

-- !select --
id text Yes false \N NONE
name text Yes false \N NONE

-- !select --
1 zhangsan
1 zhangsan1
10 zhangsan10
10 zhangsan110
10 zhangsan1210
10 zhangsan12410
10 zhangsan12510
10 zhangsan1310
10 zhangsan13610
10 zhangsan1410

-- !select --
1048576 1048576

-- !select --
id text Yes false \N NONE
name text Yes false \N NONE

-- !select --
1 zhangsan
1 zhangsan1
10 zhangsan10
10 zhangsan110
10 zhangsan1210
10 zhangsan12410
10 zhangsan12510
10 zhangsan1310
10 zhangsan13610
10 zhangsan1410

-- !select --
1048576 1048576

-- !select --
id text Yes false \N NONE
name text Yes false \N NONE

-- !select --
1 zhangsan
1 zhangsan1
10 zhangsan10
10 zhangsan110
10 zhangsan1210
10 zhangsan12410
10 zhangsan12510
10 zhangsan1310
10 zhangsan13610
10 zhangsan1410

-- !select --
1048576 1048576

-- !select --
id text Yes false \N NONE
name text Yes false \N NONE

-- !select --
1 zhangsan
1 zhangsan1
10 zhangsan10
10 zhangsan110
10 zhangsan1210
10 zhangsan12410
10 zhangsan12510
10 zhangsan1310
10 zhangsan13610
10 zhangsan1410

-- !select --
1048576 1048576

-- !select --
id text Yes false \N NONE
name text Yes false \N NONE

-- !select --
1 zhangsan
1 zhangsan1
10 zhangsan10
10 zhangsan110
10 zhangsan1210
10 zhangsan12410
10 zhangsan12510
10 zhangsan1310
10 zhangsan13610
10 zhangsan1410

-- !select --
1048576 1048576

-- !select --
id text Yes false \N NONE
name text Yes false \N NONE

-- !select --
1 2

Expand Down Expand Up @@ -176,3 +404,15 @@ c2 text Yes false \N NONE
-- !select --
__dummy_col text Yes false \N NONE

-- !select --
1 zhangsan
1 zhangsan1
10 zhangsan10
10 zhangsan110
10 zhangsan1210
10 zhangsan12410
10 zhangsan12510
10 zhangsan1310
10 zhangsan13610
10 zhangsan1410

Loading
Loading