Skip to content

Commit

Permalink
Compression: fix zstd compression level be negative number (#9323)
Browse files Browse the repository at this point in the history 
close #9322

Compression: fix zstd compression level could be a negative number

Signed-off-by: Lloyd-Pottiger <yan1579196623@gmail.com>
  • Loading branch information
@Lloyd-Pottiger
Lloyd-Pottiger authored Aug 14, 2024
1 parent 9d130eb commit 76b7f36
Show file tree
Hide file tree
Showing 2 changed files with 162 additions and 16 deletions.
34 changes: 18 additions & 16 deletions dbms/src/IO/Compression/CompressionCodecFactory.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ CompressionCodecPtr CompressionCodecFactory::getStaticCodec<CompressionCodecLZ4>
if (lz4_map.size() >= MAX_LZ4_MAP_SIZE)
lz4_map.clear();
auto [it, inserted] = lz4_map.emplace(setting.level, std::make_shared<CompressionCodecLZ4>(setting.level));
assert(inserted);
return it->second;
}

Expand All @@ -113,27 +114,28 @@ CompressionCodecPtr CompressionCodecFactory::getStaticCodec<CompressionCodecLZ4H
if (lz4hc_map.size() >= MAX_LZ4HC_MAP_SIZE)
lz4hc_map.clear();
auto [it, inserted] = lz4hc_map.emplace(setting.level, std::make_shared<CompressionCodecLZ4HC>(setting.level));
assert(inserted);
return it->second;
}

template <>
CompressionCodecPtr CompressionCodecFactory::getStaticCodec<CompressionCodecZSTD>(const CompressionSetting & setting)
{
// ZSTD level is in the range [1, 22], the maximum size of the map is 22.
static std::vector<CompressionCodecPtr> zstd_map = {
std::make_shared<CompressionCodecZSTD>(1), std::make_shared<CompressionCodecZSTD>(2),
std::make_shared<CompressionCodecZSTD>(3), std::make_shared<CompressionCodecZSTD>(4),
std::make_shared<CompressionCodecZSTD>(5), std::make_shared<CompressionCodecZSTD>(6),
std::make_shared<CompressionCodecZSTD>(7), std::make_shared<CompressionCodecZSTD>(8),
std::make_shared<CompressionCodecZSTD>(9), std::make_shared<CompressionCodecZSTD>(10),
std::make_shared<CompressionCodecZSTD>(11), std::make_shared<CompressionCodecZSTD>(12),
std::make_shared<CompressionCodecZSTD>(13), std::make_shared<CompressionCodecZSTD>(14),
std::make_shared<CompressionCodecZSTD>(15), std::make_shared<CompressionCodecZSTD>(16),
std::make_shared<CompressionCodecZSTD>(17), std::make_shared<CompressionCodecZSTD>(18),
std::make_shared<CompressionCodecZSTD>(19), std::make_shared<CompressionCodecZSTD>(20),
std::make_shared<CompressionCodecZSTD>(21), std::make_shared<CompressionCodecZSTD>(22),
};
return zstd_map[setting.level - 1];
static constexpr auto MAX_ZSTD_MAP_SIZE = 10;
static std::shared_mutex zstd_mutex;
static std::unordered_map<int, CompressionCodecPtr> zstd_map;
{
std::shared_lock lock(zstd_mutex);
auto it = zstd_map.find(setting.level);
if (it != zstd_map.end())
return it->second;
}
std::unique_lock lock(zstd_mutex);
if (zstd_map.size() >= MAX_ZSTD_MAP_SIZE)
zstd_map.clear();
auto [it, inserted] = zstd_map.emplace(setting.level, std::make_shared<CompressionCodecZSTD>(setting.level));
assert(inserted);
return it->second;
}

template <>
Expand Down Expand Up @@ -234,7 +236,7 @@ Codecs CompressionCodecFactory::createCodecs(const CompressionSettings & setting
if (auto codec = create(setting); codec)
codecs.push_back(std::move(codec));
}
assert(!codecs.empty());
RUNTIME_CHECK(!codecs.empty());
return codecs;
}

Expand Down
144 changes: 144 additions & 0 deletions dbms/src/IO/Compression/tests/gtest_codec_factory.cpp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
// Copyright 2024 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include <Common/config.h>
#include <IO/Compression/CompressionCodecFactory.h>
#include <TestUtils/TiFlashTestBasic.h>
#include <gtest/gtest.h>

#include <limits>


namespace DB::tests
{

TEST(TestCodecFactory, TestBasic)
try
{
{
auto codec = CompressionCodecFactory::create(
CompressionSettings(CompressionMethod::LZ4, std::numeric_limits<int>::min()));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(CompressionSettings(CompressionMethod::LZ4, -1));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(CompressionSettings(CompressionMethod::LZ4, 0));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(CompressionSettings(CompressionMethod::LZ4, 1));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(
CompressionSettings(CompressionMethod::LZ4, std::numeric_limits<int>::max()));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(
CompressionSettings(CompressionMethod::LZ4HC, std::numeric_limits<int>::min()));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(CompressionSettings(CompressionMethod::LZ4HC, -1));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(CompressionSettings(CompressionMethod::LZ4HC, 0));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(CompressionSettings(CompressionMethod::LZ4HC, 1));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(
CompressionSettings(CompressionMethod::LZ4HC, std::numeric_limits<int>::max()));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(
CompressionSettings(CompressionMethod::ZSTD, std::numeric_limits<int>::min()));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(CompressionSettings(CompressionMethod::ZSTD, -1));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(CompressionSettings(CompressionMethod::ZSTD, 0));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(CompressionSettings(CompressionMethod::ZSTD, 1));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(
CompressionSettings(CompressionMethod::ZSTD, std::numeric_limits<int>::max()));
ASSERT_TRUE(codec != nullptr);
}
{
auto codec = CompressionCodecFactory::create(CompressionSettings(CompressionMethod::NONE, 1));
ASSERT_TRUE(codec != nullptr);
}
#if USE_QPL
{
auto codec = CompressionCodecFactory::create(CompressionSettings(CompressionMethod::DEFLATE_QPL, 1));
ASSERT_TRUE(codec != nullptr);
}
#endif
}
CATCH

TEST(TestCodecFactory, TestMultipleCodec)
try
{
{
std::vector<CompressionSetting> settings{
CompressionSetting(CompressionMethodByte::FOR),
};
ASSERT_ANY_THROW(CompressionCodecFactory::create(CompressionSettings(settings)));
}
{
std::vector<CompressionSetting> settings{
CompressionSetting(CompressionMethodByte::DeltaFOR),
CompressionSetting(CompressionMethodByte::RunLength),
};
ASSERT_ANY_THROW(CompressionCodecFactory::create(CompressionSettings(settings)));
}
{
std::vector<CompressionSetting> settings{
CompressionSetting(CompressionMethodByte::DeltaFOR),
CompressionSetting(CompressionMethodByte::LZ4),
};
auto codec = CompressionCodecFactory::create(CompressionSettings(settings));
ASSERT_TRUE(codec != nullptr);
ASSERT_TRUE(codec->isCompression());
}
{
std::vector<CompressionSetting> settings{
CompressionSetting(CompressionMethodByte::ZSTD),
};
auto codec = CompressionCodecFactory::create(CompressionSettings(settings));
ASSERT_TRUE(codec != nullptr);
ASSERT_TRUE(codec->isCompression());
}
}
CATCH

} // namespace DB::tests

0 comments on commit 76b7f36

Please sign in to comment.