diff --git a/storage/tianmu/common/defs.h b/storage/tianmu/common/defs.h index d4f8b8335..595e65bf4 100644 --- a/storage/tianmu/common/defs.h +++ b/storage/tianmu/common/defs.h @@ -18,6 +18,7 @@ #define TIANMU_COMMON_DEFS_H_ #pragma once +#include #include namespace Tianmu { @@ -42,19 +43,17 @@ constexpr const char *COL_FILTER_HIST_DIR = "hist"; constexpr const char *COL_KN_FILE = "KN"; constexpr const char *COL_META_FILE = "META"; constexpr const char *COL_DN_FILE = "DN"; -/* - The size of the file where the DPN metadata resides, in bytes - At present, the size of a single DPN is 88 bytes, and the storage limit is 8.589 billion lines -*/ -constexpr size_t COL_DN_FILE_SIZE = 11 * 1024 * 1024; + +constexpr uint8_t MAX_PSS = 16; // max pack size shift +constexpr uint8_t DFT_PSS = 16; +constexpr size_t MAX_ROW_NUMS_EACH_PACK = 1 << common::MAX_PSS; // max 64K rows per pack + +constexpr size_t MAX_CMPR_SIZE = 0x007D000000; constexpr const char *COL_DATA_FILE = "DATA"; constexpr const char *COL_VERSION_DIR = "v"; constexpr uint32_t COL_FILE_VERSION = 3; constexpr uint32_t MAX_COLUMNS_PER_TABLE = 4000; - -constexpr uint8_t MAX_PSS = 16; -constexpr uint8_t DFT_PSS = 16; -constexpr size_t MAX_CMPR_SIZE = 0x007D000000; +constexpr uint32_t MAX_ROW_NUM_SHIFT = 33; // max 8G (1 << 33) rows per storage limitation using PACK_INDEX = uint32_t; constexpr PACK_INDEX INVALID_PACK_INDEX = -1; diff --git a/storage/tianmu/core/table_share.h b/storage/tianmu/core/table_share.h index bb00c93a9..bfdd07bc9 100644 --- a/storage/tianmu/core/table_share.h +++ b/storage/tianmu/core/table_share.h @@ -35,7 +35,7 @@ struct TABLE_META { uint64_t magic = common::FILE_MAGIC; uint32_t ver = common::TABLE_DATA_VERSION; uint32_t id; - uint32_t pss = 16; + uint32_t pss = common::MAX_PSS; }; class TableShare final { diff --git a/storage/tianmu/vc/column_share.cpp b/storage/tianmu/vc/column_share.cpp index af779f312..8053e1c50 100644 --- a/storage/tianmu/vc/column_share.cpp +++ b/storage/tianmu/vc/column_share.cpp @@ -54,7 +54,7 @@ static constexpr size_t DPN_INC_CNT = ALLOC_UNIT / sizeof(DPN); ColumnShare::~ColumnShare() { if (start != nullptr) { - if (::munmap(start, common::COL_DN_FILE_SIZE) != 0) { + if (::munmap(start, COL_DN_FILE_SIZE) != 0) { // DO NOT throw in dtor! TIANMU_LOG(LogCtl_Level::WARN, "Failed to unmap DPN file. Error %d(%s)", errno, std::strerror(errno)); } @@ -85,12 +85,12 @@ void ColumnShare::map_dpn() { ASSERT(sb.st_size % sizeof(DPN) == 0); capacity = sb.st_size / sizeof(DPN); - auto addr = ::mmap(0, common::COL_DN_FILE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, dn_fd, 0); + auto addr = ::mmap(0, COL_DN_FILE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, dn_fd, 0); if (addr == MAP_FAILED) { throw std::system_error(errno, std::system_category(), "mmap() " + dpn_file.string()); } - // TODO: should we mlock(addr, common::COL_DN_FILE_SIZE)? + // TODO: should we mlock(addr, COL_DN_FILE_SIZE)? start = static_cast(addr); } @@ -238,8 +238,7 @@ int ColumnShare::alloc_dpn(common::TX_ID xid, const DPN *from) { return i; } - ASSERT((capacity + DPN_INC_CNT) <= (common::COL_DN_FILE_SIZE / sizeof(DPN)), - "Failed to allocate new DN: " + m_path.string()); + ASSERT((capacity + DPN_INC_CNT) <= (COL_DN_FILE_SIZE / sizeof(DPN)), "Failed to allocate new DN: " + m_path.string()); capacity += DPN_INC_CNT; // NOTICE: @@ -270,7 +269,7 @@ void ColumnShare::alloc_seg(DPN *dpn) { } void ColumnShare::sync_dpns() { - int ret = ::msync(start, common::COL_DN_FILE_SIZE, MS_SYNC); + int ret = ::msync(start, COL_DN_FILE_SIZE, MS_SYNC); if (ret != 0) throw std::system_error(errno, std::system_category(), "msync() " + m_path.string()); } diff --git a/storage/tianmu/vc/column_share.h b/storage/tianmu/vc/column_share.h index bb05c7f40..38ddd1115 100644 --- a/storage/tianmu/vc/column_share.h +++ b/storage/tianmu/vc/column_share.h @@ -22,6 +22,7 @@ #include "common/assert.h" #include "common/common_definitions.h" +#include "common/defs.h" #include "common/mysql_gate.h" #include "data/dpn.h" #include "util/fs.h" @@ -30,6 +31,16 @@ namespace Tianmu { namespace core { class TableShare; +/* + Number of DPN to reach the max row limiration. + It is calcualted by (max number of row limiration) / (max number of row per pack ) +*/ +constexpr size_t MAX_DPN_NUMS_EACH_COLUMN = + 1 << (common::MAX_ROW_NUM_SHIFT - common::MAX_PSS); // max 128K packs per each column +/* + The size of the file where the DPN metadata resides, in bytes +*/ +constexpr size_t COL_DN_FILE_SIZE = MAX_DPN_NUMS_EACH_COLUMN * sizeof(core::DPN); struct COL_META { uint32_t magic; @@ -77,11 +88,11 @@ class ColumnShare final { } DPN *get_dpn_ptr(common::PACK_INDEX i) { - ASSERT(i < common::COL_DN_FILE_SIZE / sizeof(DPN), "bad dpn index: " + std::to_string(i)); + ASSERT(i < COL_DN_FILE_SIZE / sizeof(DPN), "bad dpn index: " + std::to_string(i)); return &start[i]; } const DPN *get_dpn_ptr(common::PACK_INDEX i) const { - ASSERT(i < common::COL_DN_FILE_SIZE / sizeof(DPN), "bad dpn index: " + std::to_string(i)); + ASSERT(i < COL_DN_FILE_SIZE / sizeof(DPN), "bad dpn index: " + std::to_string(i)); return &start[i]; } diff --git a/storage/tianmu/vc/tianmu_attr.cpp b/storage/tianmu/vc/tianmu_attr.cpp index 1e1088f88..830d460e7 100644 --- a/storage/tianmu/vc/tianmu_attr.cpp +++ b/storage/tianmu/vc/tianmu_attr.cpp @@ -151,7 +151,7 @@ void TianmuAttr::Create(const fs::path &dir, const AttributeTypeInfo &ati, uint8 } fdn.WriteExact(&dpn, sizeof(dpn)); fdn.Flush(); - fs::resize_file(dir / common::COL_DN_FILE, common::COL_DN_FILE_SIZE); + fs::resize_file(dir / common::COL_DN_FILE, core::COL_DN_FILE_SIZE); } // create filter directories