From 265fd8ffa17b92cffb299891f77006ef7c133804 Mon Sep 17 00:00:00 2001 From: Lloyd-Pottiger Date: Mon, 24 Jun 2024 16:11:45 +0800 Subject: [PATCH] refine Signed-off-by: Lloyd-Pottiger --- .../IO/Compression/CompressionSettings.cpp | 18 +++++++++++-- dbms/src/IO/Compression/CompressionSettings.h | 3 +++ .../ColumnFile/ColumnFilePersisted.cpp | 26 +++++-------------- .../Storages/DeltaMerge/File/DMFileWriter.h | 17 ++++-------- 4 files changed, 30 insertions(+), 34 deletions(-) diff --git a/dbms/src/IO/Compression/CompressionSettings.cpp b/dbms/src/IO/Compression/CompressionSettings.cpp index 2e890a4f249..4a15b2da6bf 100644 --- a/dbms/src/IO/Compression/CompressionSettings.cpp +++ b/dbms/src/IO/Compression/CompressionSettings.cpp @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "CompressionSettings.h" - #include +#include #include #include +#include + namespace DB { CompressionSetting::CompressionSetting(const Settings & settings) @@ -53,4 +54,17 @@ int CompressionSetting::getDefaultLevel(CompressionMethod method) } } +CompressionSettings CompressionSettings::create(CompressionMethod method, const IDataType & type) +{ + CompressionSettings settings(method); + if (type.isInteger()) + { + auto & setting = settings.settings[0]; + auto data_type = magic_enum::enum_cast(type.getSizeOfValueInMemory()); + RUNTIME_CHECK(data_type.has_value()); + setting.data_type = data_type.value(); + } + return settings; +} + } // namespace DB diff --git a/dbms/src/IO/Compression/CompressionSettings.h b/dbms/src/IO/Compression/CompressionSettings.h index 5363b0aca5d..37cc7410039 100644 --- a/dbms/src/IO/Compression/CompressionSettings.h +++ b/dbms/src/IO/Compression/CompressionSettings.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include #include @@ -101,6 +102,8 @@ struct CompressionSettings : settings(settings_) {} + static CompressionSettings create(CompressionMethod method, const IDataType & type); + std::vector settings; }; diff --git a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFilePersisted.cpp b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFilePersisted.cpp index 6e2b5733c53..fdb56b65782 100644 --- a/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFilePersisted.cpp +++ b/dbms/src/Storages/DeltaMerge/ColumnFile/ColumnFilePersisted.cpp @@ -71,33 +71,19 @@ void serializeColumn( const DataTypePtr & type, size_t offset, size_t limit, - CompressionMethod compression_method, - Int64 compression_level) + CompressionMethod /*compression_method*/, + Int64 /*compression_level*/) { - std::unique_ptr> compressed; - if (type->isInteger()) - { - CompressionSettings compression_settings(CompressionMethod::Lightweight); - auto & setting = compression_settings.settings[0]; - auto data_type = magic_enum::enum_cast(type->getSizeOfValueInMemory()); - RUNTIME_CHECK(data_type.has_value()); - setting.data_type = data_type.value(); - compressed = std::make_unique>(buf, compression_settings); - } - else - { - compressed = std::make_unique>( - buf, - CompressionSettings(compression_method, compression_level)); - } + auto compression_settings = CompressionSettings::create(CompressionMethod::Lightweight, *type); + CompressedWriteBuffer compressed(buf, compression_settings); type->serializeBinaryBulkWithMultipleStreams( column, - [&](const IDataType::SubstreamPath &) { return compressed.get(); }, + [&](const IDataType::SubstreamPath &) { return &compressed; }, offset, limit, true, {}); - compressed->next(); + compressed.next(); } void deserializeColumn(IColumn & column, const DataTypePtr & type, std::string_view data_buf, size_t rows) diff --git a/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h b/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h index 592b69c0f2d..1ec08a1630a 100644 --- a/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h +++ b/dbms/src/Storages/DeltaMerge/File/DMFileWriter.h @@ -23,12 +23,10 @@ #include #include -#include "magic_enum.hpp" -namespace DB -{ -namespace DM +namespace DB::DM { + namespace detail { static inline DB::ChecksumAlgo getAlgorithmOrNone(DMFile & dmfile) @@ -40,6 +38,7 @@ static inline size_t getFrameSizeOrDefault(DMFile & dmfile) return dmfile.getConfiguration() ? dmfile.getConfiguration()->getChecksumFrameLength() : DBMS_DEFAULT_BUFFER_SIZE; } } // namespace detail + class DMFileWriter { public: @@ -71,12 +70,7 @@ class DMFileWriter { if (type->isInteger()) { - assert(compression_settings.settings.size() == 1); - CompressionSettings settings(CompressionMethod::Lightweight); - auto & setting = settings.settings[0]; - auto data_type = magic_enum::enum_cast(type->getSizeOfValueInMemory()); - RUNTIME_CHECK(data_type.has_value()); - setting.data_type = data_type.value(); + auto settings = CompressionSettings::create(CompressionMethod::Lightweight, *type); compressed_buf = CompressedWriteBuffer<>::build(*plain_file, settings, !dmfile->getConfiguration()); } else @@ -197,5 +191,4 @@ class DMFileWriter bool is_empty_file = true; }; -} // namespace DM -} // namespace DB +} // namespace DB::DM