Skip to content

Commit

Permalink
rename
Browse files Browse the repository at this point in the history 
Signed-off-by: Lloyd-Pottiger <yan1579196623@gmail.com>
  • Loading branch information
@Lloyd-Pottiger
Lloyd-Pottiger committed Jun 3, 2024
1 parent c840f56 commit c0eb5d8
Show file tree
Hide file tree
Showing 5 changed files with 74 additions and 73 deletions.
18 changes: 9 additions & 9 deletions dbms/src/IO/Compression/CompressionCodecDeltaFOR.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ template <std::integral T>
UInt32 compressData(const char * source, UInt32 source_size, char * dest)
{
const auto count = source_size / sizeof(T);
DB::Compression::DeltaEncoding<T>(reinterpret_cast<const T *>(source), count, reinterpret_cast<T *>(dest));
DB::Compression::deltaEncoding<T>(reinterpret_cast<const T *>(source), count, reinterpret_cast<T *>(dest));
// Cast deltas to signed type to better compress negative values.
// For example, if we have a sequence of UInt8 values [3, 2, 1, 0], the deltas will be [3, -1, -1, -1]
// If we compress them as UInt8, we will get [3, 255, 255, 255], which is not optimal.
Expand Down Expand Up @@ -114,16 +114,16 @@ void CompressionCodecDeltaFOR::doDecompressData(
switch (bytes_size)
{
case 1:
DB::Compression::DeltaFORDecoding<UInt8>(&source[1], source_size_no_header, dest, uncompressed_size);
DB::Compression::deltaFORDecoding<UInt8>(&source[1], source_size_no_header, dest, uncompressed_size);
break;
case 2:
DB::Compression::DeltaFORDecoding<UInt16>(&source[1], source_size_no_header, dest, uncompressed_size);
DB::Compression::deltaFORDecoding<UInt16>(&source[1], source_size_no_header, dest, uncompressed_size);
break;
case 4:
DB::Compression::DeltaFORDecoding<UInt32>(&source[1], source_size_no_header, dest, uncompressed_size);
DB::Compression::deltaFORDecoding<UInt32>(&source[1], source_size_no_header, dest, uncompressed_size);
break;
case 8:
DB::Compression::DeltaFORDecoding<UInt64>(&source[1], source_size_no_header, dest, uncompressed_size);
DB::Compression::deltaFORDecoding<UInt64>(&source[1], source_size_no_header, dest, uncompressed_size);
break;
default:
throw Exception(
Expand Down Expand Up @@ -158,16 +158,16 @@ void CompressionCodecDeltaFOR::ordinaryDecompress(
switch (bytes_size)
{
case 1:
DB::Compression::OrdinaryDeltaFORDecoding<UInt8>(&source[1], source_size_no_header, dest, dest_size);
DB::Compression::ordinaryDeltaFORDecoding<UInt8>(&source[1], source_size_no_header, dest, dest_size);
break;
case 2:
DB::Compression::OrdinaryDeltaFORDecoding<UInt16>(&source[1], source_size_no_header, dest, dest_size);
DB::Compression::ordinaryDeltaFORDecoding<UInt16>(&source[1], source_size_no_header, dest, dest_size);
break;
case 4:
DB::Compression::OrdinaryDeltaFORDecoding<UInt32>(&source[1], source_size_no_header, dest, dest_size);
DB::Compression::ordinaryDeltaFORDecoding<UInt32>(&source[1], source_size_no_header, dest, dest_size);
break;
case 8:
DB::Compression::OrdinaryDeltaFORDecoding<UInt64>(&source[1], source_size_no_header, dest, dest_size);
DB::Compression::ordinaryDeltaFORDecoding<UInt64>(&source[1], source_size_no_header, dest, dest_size);
break;
default:
throw Exception(
Expand Down
16 changes: 8 additions & 8 deletions dbms/src/IO/Compression/CompressionCodecIntegerLightweight.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,17 +78,17 @@ size_t CompressionCodecIntegerLightweight::compressDataForType(const char * sour
{
case Mode::CONSTANT:
{
compressed_size += Compression::ConstantEncoding(std::get<0>(state), dest);
compressed_size += Compression::constantEncoding(std::get<0>(state), dest);
break;
}
case Mode::CONSTANT_DELTA:
{
compressed_size += Compression::ConstantDeltaEncoding(values[0], std::get<0>(state), dest);
compressed_size += Compression::constantDeltaEncoding(values[0], std::get<0>(state), dest);
break;
}
case Mode::RunLength:
{
compressed_size += Compression::RunLengthEncoding<T>(std::get<1>(state), dest);
compressed_size += Compression::runLengthEncoding<T>(std::get<1>(state), dest);
break;
}
case Mode::FOR:
Expand Down Expand Up @@ -146,19 +146,19 @@ void CompressionCodecIntegerLightweight::decompressDataForType(
switch (mode)
{
case Mode::CONSTANT:
Compression::ConstantDecoding<T>(source, source_size, dest, output_size);
Compression::constantDecoding<T>(source, source_size, dest, output_size);
break;
case Mode::CONSTANT_DELTA:
Compression::ConstantDeltaDecoding<T>(source, source_size, dest, output_size);
Compression::constantDeltaDecoding<T>(source, source_size, dest, output_size);
break;
case Mode::RunLength:
Compression::RunLengthDecoding<T>(source, source_size, dest, output_size);
Compression::runLengthDecoding<T>(source, source_size, dest, output_size);
break;
case Mode::FOR:
Compression::FORDecoding<T>(source, source_size, dest, output_size);
break;
case Mode::DELTA_FOR:
Compression::DeltaFORDecoding<T>(source, source_size, dest, output_size);
Compression::deltaFORDecoding<T>(source, source_size, dest, output_size);
break;
case Mode::LZ4:
if (unlikely(LZ4_decompress_safe(source, dest, source_size, output_size) < 0))
Expand Down Expand Up @@ -305,7 +305,7 @@ void CompressionCodecIntegerLightweight::CompressContext::analyze(std::span<cons
// Assume that the compression ratio of LZ4 is 3.0
// The official document says that the compression ratio of LZ4 is 2.1, https://github.com/lz4/lz4
size_t estimate_lz_size = values.size() * sizeof(T) / 3;
size_t rle_size = rle.empty() ? std::numeric_limits<size_t>::max() : Compression::RunLengthPairsSize(rle);
size_t rle_size = rle.empty() ? std::numeric_limits<size_t>::max() : Compression::runLengthPairsSize(rle);
if (needAnalyzeRunLength() && rle_size < delta_for_size && rle_size < for_size && rle_size < estimate_lz_size)
{
state = std::move(rle);
Expand Down
12 changes: 6 additions & 6 deletions dbms/src/IO/Compression/CompressionCodecRunLength.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest)
++rle_vec.back().second;
}

if (DB::Compression::RunLengthPairsSize<T>(rle_vec) > source_size)
if (DB::Compression::runLengthPairsSize<T>(rle_vec) >= source_size)
{
dest[0] = JUST_COPY_CODE;
memcpy(&dest[1], source, source_size);
Expand All @@ -74,7 +74,7 @@ UInt32 compressDataForType(const char * source, UInt32 source_size, char * dest)

dest[0] = sizeof(T);
dest += 1;
return 1 + DB::Compression::RunLengthEncoding<T>(rle_vec, dest);
return 1 + DB::Compression::runLengthEncoding<T>(rle_vec, dest);
}

} // namespace
Expand Down Expand Up @@ -134,16 +134,16 @@ void CompressionCodecRunLength::doDecompressData(
switch (bytes_size)
{
case 1:
DB::Compression::RunLengthDecoding<UInt8>(&source[1], source_size - 1, dest, uncompressed_size);
DB::Compression::runLengthDecoding<UInt8>(&source[1], source_size - 1, dest, uncompressed_size);
break;
case 2:
DB::Compression::RunLengthDecoding<UInt16>(&source[1], source_size - 1, dest, uncompressed_size);
DB::Compression::runLengthDecoding<UInt16>(&source[1], source_size - 1, dest, uncompressed_size);
break;
case 4:
DB::Compression::RunLengthDecoding<UInt32>(&source[1], source_size - 1, dest, uncompressed_size);
DB::Compression::runLengthDecoding<UInt32>(&source[1], source_size - 1, dest, uncompressed_size);
break;
case 8:
DB::Compression::RunLengthDecoding<UInt64>(&source[1], source_size - 1, dest, uncompressed_size);
DB::Compression::runLengthDecoding<UInt64>(&source[1], source_size - 1, dest, uncompressed_size);
break;
default:
throw Exception(
Expand Down
62 changes: 31 additions & 31 deletions dbms/src/IO/Compression/EncodingUtil.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

#include "EncodingUtil.h"
#include <IO/Compression/EncodingUtil.h>

#if defined(__AVX2__)
#include <immintrin.h>
Expand All @@ -22,7 +22,7 @@ namespace DB::Compression
{

template <std::integral T>
void ApplyFrameOfReference(T * dst, T frame_of_reference, UInt32 count)
void applyFrameOfReference(T * dst, T frame_of_reference, UInt32 count)
{
if (frame_of_reference == 0)
return;
Expand Down Expand Up @@ -61,17 +61,17 @@ void ApplyFrameOfReference(T * dst, T frame_of_reference, UInt32 count)
}
}

template void ApplyFrameOfReference<UInt8>(UInt8 *, UInt8, UInt32);
template void ApplyFrameOfReference<UInt16>(UInt16 *, UInt16, UInt32);
template void ApplyFrameOfReference<UInt32>(UInt32 *, UInt32, UInt32);
template void ApplyFrameOfReference<UInt64>(UInt64 *, UInt64, UInt32);
template void ApplyFrameOfReference<Int8>(Int8 *, Int8, UInt32);
template void ApplyFrameOfReference<Int16>(Int16 *, Int16, UInt32);
template void ApplyFrameOfReference<Int32>(Int32 *, Int32, UInt32);
template void ApplyFrameOfReference<Int64>(Int64 *, Int64, UInt32);
template void applyFrameOfReference<UInt8>(UInt8 *, UInt8, UInt32);
template void applyFrameOfReference<UInt16>(UInt16 *, UInt16, UInt32);
template void applyFrameOfReference<UInt32>(UInt32 *, UInt32, UInt32);
template void applyFrameOfReference<UInt64>(UInt64 *, UInt64, UInt32);
template void applyFrameOfReference<Int8>(Int8 *, Int8, UInt32);
template void applyFrameOfReference<Int16>(Int16 *, Int16, UInt32);
template void applyFrameOfReference<Int32>(Int32 *, Int32, UInt32);
template void applyFrameOfReference<Int64>(Int64 *, Int64, UInt32);

template <std::integral T>
void SubtractFrameOfReference(T * dst, T frame_of_reference, UInt32 count)
void subtractFrameOfReference(T * dst, T frame_of_reference, UInt32 count)
{
if (frame_of_reference == 0)
return;
Expand Down Expand Up @@ -110,14 +110,14 @@ void SubtractFrameOfReference(T * dst, T frame_of_reference, UInt32 count)
}
}

template void SubtractFrameOfReference<Int8>(Int8 *, Int8, UInt32);
template void SubtractFrameOfReference<Int16>(Int16 *, Int16, UInt32);
template void SubtractFrameOfReference<Int32>(Int32 *, Int32, UInt32);
template void SubtractFrameOfReference<Int64>(Int64 *, Int64, UInt32);
template void SubtractFrameOfReference<UInt8>(UInt8 *, UInt8, UInt32);
template void SubtractFrameOfReference<UInt16>(UInt16 *, UInt16, UInt32);
template void SubtractFrameOfReference<UInt32>(UInt32 *, UInt32, UInt32);
template void SubtractFrameOfReference<UInt64>(UInt64 *, UInt64, UInt32);
template void subtractFrameOfReference<Int8>(Int8 *, Int8, UInt32);
template void subtractFrameOfReference<Int16>(Int16 *, Int16, UInt32);
template void subtractFrameOfReference<Int32>(Int32 *, Int32, UInt32);
template void subtractFrameOfReference<Int64>(Int64 *, Int64, UInt32);
template void subtractFrameOfReference<UInt8>(UInt8 *, UInt8, UInt32);
template void subtractFrameOfReference<UInt16>(UInt16 *, UInt16, UInt32);
template void subtractFrameOfReference<UInt32>(UInt32 *, UInt32, UInt32);
template void subtractFrameOfReference<UInt64>(UInt64 *, UInt64, UInt32);

template <std::integral T>
UInt8 FOREncodingWidth(std::vector<T> & values, T frame_of_reference)
Expand All @@ -128,7 +128,7 @@ UInt8 FOREncodingWidth(std::vector<T> & values, T frame_of_reference)
// For example, we have a sequence of Int8 values [-128, 1, 127], after subtracting frame of reference -128, the values are [0, -127, -1].
// The minimum bit width required to store the values is 8 rather than the width of `max_value - min_value = -1`.
// So we need to calculate the minimum bit width of the values after subtracting frame of reference.
SubtractFrameOfReference<T>(values.data(), frame_of_reference, values.size());
subtractFrameOfReference<T>(values.data(), frame_of_reference, values.size());
T max_value = *std::max_element(values.cbegin(), values.cend());
T min_value = *std::min_element(values.cbegin(), values.cend());
return BitpackingPrimitives::minimumBitWidth<T>(min_value, max_value);
Expand All @@ -150,7 +150,7 @@ template UInt8 FOREncodingWidth<UInt32>(std::vector<UInt32> &, UInt32);
template UInt8 FOREncodingWidth<UInt64>(std::vector<UInt64> &, UInt64);

template <std::integral T>
void DeltaDecoding(const char * source, UInt32 source_size, char * dest)
void deltaDecoding(const char * source, UInt32 source_size, char * dest)
{
ordinaryDeltaDecoding<T>(source, source_size, dest);
}
Expand All @@ -159,7 +159,7 @@ void DeltaDecoding(const char * source, UInt32 source_size, char * dest)
// Note: using SIMD to rewrite compress does not improve performance.

template <>
void DeltaDecoding<UInt32>(const char * __restrict__ raw_source, UInt32 raw_source_size, char * __restrict__ raw_dest)
void deltaDecoding<UInt32>(const char * __restrict__ raw_source, UInt32 raw_source_size, char * __restrict__ raw_dest)
{
const auto * source = reinterpret_cast<const UInt32 *>(raw_source);
auto source_size = raw_source_size / sizeof(UInt32);
Expand All @@ -183,7 +183,7 @@ void DeltaDecoding<UInt32>(const char * __restrict__ raw_source, UInt32 raw_sour
}

template <>
void DeltaDecoding<UInt64>(const char * __restrict__ raw_source, UInt32 raw_source_size, char * __restrict__ raw_dest)
void deltaDecoding<UInt64>(const char * __restrict__ raw_source, UInt32 raw_source_size, char * __restrict__ raw_dest)
{
const auto * source = reinterpret_cast<const UInt64 *>(raw_source);
auto source_size = raw_source_size / sizeof(UInt64);
Expand Down Expand Up @@ -219,14 +219,14 @@ void DeltaDecoding<UInt64>(const char * __restrict__ raw_source, UInt32 raw_sour
#endif

template <std::integral T>
void DeltaFORDecoding(const char * src, UInt32 source_size, char * dest, UInt32 dest_size)
void deltaFORDecoding(const char * src, UInt32 source_size, char * dest, UInt32 dest_size)
{
static_assert(std::is_integral<T>::value, "Integral required.");
OrdinaryDeltaFORDecoding<T>(src, source_size, dest, dest_size);
ordinaryDeltaFORDecoding<T>(src, source_size, dest, dest_size);
}

template <>
void DeltaFORDecoding<UInt32>(const char * src, UInt32 source_size, char * dest, UInt32 dest_size)
void deltaFORDecoding<UInt32>(const char * src, UInt32 source_size, char * dest, UInt32 dest_size)
{
const auto count = dest_size / sizeof(UInt32);
auto round_size = BitpackingPrimitives::roundUpToAlgorithmGroupSize(count);
Expand All @@ -235,11 +235,11 @@ void DeltaFORDecoding<UInt32>(const char * src, UInt32 source_size, char * dest,
char tmp_buffer[required_size];
memset(tmp_buffer, 0, required_size);
FORDecoding<Int32>(src, source_size, tmp_buffer, required_size);
DeltaDecoding<UInt32>(reinterpret_cast<const char *>(tmp_buffer), dest_size, dest);
deltaDecoding<UInt32>(reinterpret_cast<const char *>(tmp_buffer), dest_size, dest);
}

template <>
void DeltaFORDecoding<UInt64>(const char * src, UInt32 source_size, char * dest, UInt32 dest_size)
void deltaFORDecoding<UInt64>(const char * src, UInt32 source_size, char * dest, UInt32 dest_size)
{
const auto count = dest_size / sizeof(UInt64);
const auto round_size = BitpackingPrimitives::roundUpToAlgorithmGroupSize(count);
Expand All @@ -248,10 +248,10 @@ void DeltaFORDecoding<UInt64>(const char * src, UInt32 source_size, char * dest,
char tmp_buffer[required_size];
memset(tmp_buffer, 0, required_size);
FORDecoding<Int64>(src, source_size, tmp_buffer, required_size);
DeltaDecoding<UInt64>(reinterpret_cast<const char *>(tmp_buffer), dest_size, dest);
deltaDecoding<UInt64>(reinterpret_cast<const char *>(tmp_buffer), dest_size, dest);
}

template void DeltaFORDecoding<UInt8>(const char *, UInt32, char *, UInt32);
template void DeltaFORDecoding<UInt16>(const char *, UInt32, char *, UInt32);
template void deltaFORDecoding<UInt8>(const char *, UInt32, char *, UInt32);
template void deltaFORDecoding<UInt16>(const char *, UInt32, char *, UInt32);

} // namespace DB::Compression
Loading

0 comments on commit c0eb5d8

Please sign in to comment.