Skip to content

BTreeIndex Precharge RowId #754

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kungasc merged 11 commits into from
Dec 29, 2023
Merged

BTreeIndex Precharge RowId #754

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
4e59b12
int32 levels count
kungasc Dec 26, 2023
ba5dcfa
wip precharge
kungasc Dec 26, 2023
55199fe
minor
kungasc Dec 26, 2023
ea37fec
wip precharge rows
kungasc Dec 27, 2023
635ea67
test charge
kungasc Dec 27, 2023
0b2aa25
fix typo
kungasc Dec 27, 2023
5d0e50f
one node works
kungasc Dec 27, 2023
eb9b94c
rows count
kungasc Dec 28, 2023
78d7a3b
reverse
kungasc Dec 28, 2023
5be885a
fix style
kungasc Dec 28, 2023
b623877
cr: fix grammar
kungasc Dec 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ydb/core/tablet_flat/benchmark/b_part_index.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ namespace {

Cerr << "DataBytes = " << part->Stat.Bytes << " DataPages = " << IndexTools::CountMainPages(*part) << Endl;
Cerr << "FlatIndexBytes = " << part->GetPageSize(part->IndexPages.Groups[groups ? 1 : 0], {}) << " BTreeIndexBytes = " << part->IndexPages.BTreeGroups[groups ? 1 : 0].IndexSize << Endl;
Cerr << "Levels = " << part->IndexPages.BTreeGroups[groups ? 1 : 0].LevelsCount << Endl;
Cerr << "Levels = " << part->IndexPages.BTreeGroups[groups ? 1 : 0].LevelCount << Endl;

// 100 MB
UNIT_ASSERT_GE(part->Stat.Bytes, 100ull*1024*1024);
Expand Down
26 changes: 13 additions & 13 deletions ydb/core/tablet_flat/flat_page_btree_index.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ namespace NKikimr::NTable::NPage {

struct TShortChild {
TPageId PageId;
TRowId Count;
TRowId RowCount;
ui64 DataSize;

auto operator<=>(const TShortChild&) const = default;
Expand All @@ -95,22 +95,22 @@ namespace NKikimr::NTable::NPage {

struct TChild {
TPageId PageId;
TRowId Count;
TRowId RowCount;
ui64 DataSize;
TRowId ErasedCount;
TRowId ErasedRowCount;

auto operator<=>(const TChild&) const = default;

TString ToString() const noexcept
{
return TStringBuilder() << "PageId: " << PageId << " Count: " << Count << " DataSize: " << DataSize << " Erased: " << ErasedCount;
return TStringBuilder() << "PageId: " << PageId << " RowCount: " << RowCount << " DataSize: " << DataSize << " ErasedRowCount: " << ErasedRowCount;
}
} Y_PACKED;

static_assert(sizeof(TChild) == 28, "Invalid TBtreeIndexNode TChild size");

static_assert(offsetof(TChild, PageId) == offsetof(TShortChild, PageId));
static_assert(offsetof(TChild, Count) == offsetof(TShortChild, Count));
static_assert(offsetof(TChild, RowCount) == offsetof(TShortChild, RowCount));
static_assert(offsetof(TChild, DataSize) == offsetof(TShortChild, DataSize));

#pragma pack(pop)
Expand Down Expand Up @@ -306,7 +306,7 @@ namespace NKikimr::NTable::NPage {
{
if (Header->IsShortChildFormat) {
const TShortChild* const shortChild = TDeref<const TShortChild>::At(Children, pos * sizeof(TShortChild));
return { shortChild->PageId, shortChild->Count, shortChild->DataSize, 0 };
return { shortChild->PageId, shortChild->RowCount, shortChild->DataSize, 0 };
} else {
return *TDeref<const TChild>::At(Children, pos * sizeof(TChild));
}
Expand All @@ -316,7 +316,7 @@ namespace NKikimr::NTable::NPage {
return beginRowId <= rowId && rowId < endRowId;
}

TRecIdx Seek(TRowId rowId, std::optional<TRecIdx> on) const noexcept
TRecIdx Seek(TRowId rowId, std::optional<TRecIdx> on = { }) const noexcept
{
const TRecIdx childrenCount = GetChildrenCount();
if (on && on >= childrenCount) {
Expand All @@ -326,19 +326,19 @@ namespace NKikimr::NTable::NPage {

auto range = xrange(0u, childrenCount);
const auto cmp = [this](TRowId rowId, TPos pos) {
return rowId < GetShortChild(pos).Count;
return rowId < GetShortChild(pos).RowCount;
};

TRecIdx result;
if (!on) {
// Will do a full binary search on full range
} else if (GetShortChild(*on).Count <= rowId) {
} else if (GetShortChild(*on).RowCount <= rowId) {
// Try a short linear search first
result = *on;
for (int linear = 0; linear < 4; ++linear) {
result++;
Y_ABORT_UNLESS(result < childrenCount, "Should always seek some child");
if (GetShortChild(result).Count > rowId) {
if (GetShortChild(result).RowCount > rowId) {
return result;
}
}
Expand All @@ -352,7 +352,7 @@ namespace NKikimr::NTable::NPage {
if (result == 0) {
return 0;
}
if (GetShortChild(result - 1).Count <= rowId) {
if (GetShortChild(result - 1).RowCount <= rowId) {
return result;
}
result--;
Expand Down Expand Up @@ -464,14 +464,14 @@ namespace NKikimr::NTable::NPage {
};

struct TBtreeIndexMeta : public TBtreeIndexNode::TChild {
size_t LevelsCount;
ui32 LevelCount;
ui64 IndexSize;

auto operator<=>(const TBtreeIndexMeta&) const = default;

TString ToString() const noexcept
{
return TStringBuilder() << TBtreeIndexNode::TChild::ToString() << " LevelsCount: " << LevelsCount << " IndexSize: " << IndexSize;
return TStringBuilder() << TBtreeIndexNode::TChild::ToString() << " LevelCount: " << LevelCount << " IndexSize: " << IndexSize;
}
};
}
31 changes: 16 additions & 15 deletions ydb/core/tablet_flat/flat_page_btree_index_writer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ namespace NKikimr::NTable::NPage {
}

void AddChild(TChild child) {
Y_ABORT_UNLESS(child.ErasedCount == 0 || !IsShortChildFormat(), "Short format can't have ErasedCount");
Y_ABORT_UNLESS(child.ErasedRowCount == 0 || !IsShortChildFormat(), "Short format can't have ErasedRowCount");
Children.push_back(child);
}

Expand Down Expand Up @@ -249,7 +249,7 @@ namespace NKikimr::NTable::NPage {
void PlaceChild(const TChild& child) noexcept
{
if (IsShortChildFormat()) {
Place<TShortChild>() = TShortChild{child.PageId, child.Count, child.DataSize};
Place<TShortChild>() = TShortChild{child.PageId, child.RowCount, child.DataSize};
} else {
Place<TChild>() = child;
}
Expand Down Expand Up @@ -375,20 +375,21 @@ namespace NKikimr::NTable::NPage {
}

void AddShortChild(TShortChild child) {
AddChild(TChild{child.PageId, child.Count, child.DataSize, 0});
AddChild(TChild{child.PageId, child.RowCount, child.DataSize, 0});
}

void AddChild(TChild child) {
// aggregate in order to perform search by row id from any leaf node
child.Count = (ChildrenCount += child.Count);
child.DataSize = (ChildrenSize += child.DataSize);
child.ErasedCount = (ChildrenErasedCount += child.ErasedCount);
child.RowCount = (ChildRowCount += child.RowCount);
child.DataSize = (ChildSize += child.DataSize);
child.ErasedRowCount = (ChildErasedRowCount += child.ErasedRowCount);

Levels[0].PushChild(child);
}

std::optional<TBtreeIndexMeta> Flush(IPageWriter &pager, bool last) {
for (size_t levelIndex = 0; levelIndex < Levels.size(); levelIndex++) {
Y_ABORT_UNLESS(Levels.size() < Max<ui32>(), "Levels size is out of bounds");
for (ui32 levelIndex = 0; levelIndex < Levels.size(); levelIndex++) {
if (last && !Levels[levelIndex].GetKeysCount()) {
Y_ABORT_UNLESS(Levels[levelIndex].GetChildrenCount() == 1, "Should be root");
return TBtreeIndexMeta{ Levels[levelIndex].PopChild(), levelIndex, IndexSize };
Expand All @@ -408,13 +409,13 @@ namespace NKikimr::NTable::NPage {
IndexSize = 0;
Writer.Reset();
Levels = { TLevel() };
ChildrenCount = 0;
ChildrenErasedCount = 0;
ChildrenSize = 0;
ChildRowCount = 0;
ChildErasedRowCount = 0;
ChildSize = 0;
}

private:
bool TryFlush(size_t levelIndex, IPageWriter &pager, bool last) {
bool TryFlush(ui32 levelIndex, IPageWriter &pager, bool last) {
if (!last && Levels[levelIndex].GetKeysCount() <= 2 * NodeKeysMax) {
// Note: node should meet both NodeKeysMin and NodeSize restrictions for split

Expand Down Expand Up @@ -462,7 +463,7 @@ namespace NKikimr::NTable::NPage {
if (levelIndex + 1 == Levels.size()) {
Levels.emplace_back();
}
Levels[levelIndex + 1].PushChild(TChild{pageId, lastChild.Count, lastChild.DataSize, lastChild.ErasedCount});
Levels[levelIndex + 1].PushChild(TChild{pageId, lastChild.RowCount, lastChild.DataSize, lastChild.ErasedRowCount});
if (!last) {
Levels[levelIndex + 1].PushKey(Levels[levelIndex].PopKey());
}
Expand Down Expand Up @@ -497,9 +498,9 @@ namespace NKikimr::NTable::NPage {
const ui32 NodeKeysMin;
const ui32 NodeKeysMax;

TRowId ChildrenCount = 0;
TRowId ChildrenErasedCount = 0;
ui64 ChildrenSize = 0;
TRowId ChildRowCount = 0;
TRowId ChildErasedRowCount = 0;
ui64 ChildSize = 0;
};

}
21 changes: 10 additions & 11 deletions ydb/core/tablet_flat/flat_part_btree_index_iter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
#pragma once

#include "flat_part_iface.h"
#include "flat_page_index.h"
#include "flat_table_part.h"
#include "flat_part_index_iter_iface.h"

Expand Down Expand Up @@ -116,7 +115,7 @@ class TPartBtreeIndexIt : public IIndexIter {
, GroupId(groupId)
, GroupInfo(part->Scheme->GetLayout(groupId))
, Meta(groupId.IsHistoric() ? part->IndexPages.BTreeHistoric[groupId.Index] : part->IndexPages.BTreeGroups[groupId.Index])
, State(Reserve(Meta.LevelsCount + 1))
, State(Reserve(Meta.LevelCount + 1))
{
const static TCellsIterable EmptyKey(static_cast<const char*>(nullptr), TColumns());
State.emplace_back(Meta, 0, GetEndRowId(), EmptyKey, EmptyKey);
Expand Down Expand Up @@ -181,7 +180,7 @@ class TPartBtreeIndexIt : public IIndexIter {
EReady Next() override {
Y_ABORT_UNLESS(!IsExhausted());

if (Meta.LevelsCount == 0) {
if (Meta.LevelCount == 0) {
return Exhaust();
}

Expand All @@ -195,7 +194,7 @@ class TPartBtreeIndexIt : public IIndexIter {
PushNextState(*State.back().Pos + 1);
}

for (size_t level : xrange(State.size() - 1, Meta.LevelsCount)) {
for (ui32 level : xrange<ui32>(State.size() - 1, Meta.LevelCount)) {
if (!TryLoad(State[level])) {
// exiting with an intermediate state
Y_DEBUG_ABORT_UNLESS(!IsLeaf() && !IsExhausted());
Expand All @@ -212,7 +211,7 @@ class TPartBtreeIndexIt : public IIndexIter {
EReady Prev() override {
Y_ABORT_UNLESS(!IsExhausted());

if (Meta.LevelsCount == 0) {
if (Meta.LevelCount == 0) {
return Exhaust();
}

Expand All @@ -226,7 +225,7 @@ class TPartBtreeIndexIt : public IIndexIter {
PushNextState(*State.back().Pos - 1);
}

for (size_t level : xrange(State.size() - 1, Meta.LevelsCount)) {
for (ui32 level : xrange<ui32>(State.size() - 1, Meta.LevelCount)) {
if (!TryLoad(State[level])) {
// exiting with an intermediate state
Y_DEBUG_ABORT_UNLESS(!IsLeaf() && !IsExhausted());
Expand All @@ -247,7 +246,7 @@ class TPartBtreeIndexIt : public IIndexIter {
}

TRowId GetEndRowId() const override {
return Meta.Count;
return Meta.RowCount;
}

TPageId GetPageId() const override {
Expand Down Expand Up @@ -287,7 +286,7 @@ class TPartBtreeIndexIt : public IIndexIter {
State[0].Pos = { };
}

for (size_t level : xrange(State.size() - 1, Meta.LevelsCount)) {
for (ui32 level : xrange<ui32>(State.size() - 1, Meta.LevelCount)) {
auto &state = State[level];
Y_DEBUG_ABORT_UNLESS(seek.BelongsTo(state));
if (!TryLoad(state)) {
Expand Down Expand Up @@ -317,7 +316,7 @@ class TPartBtreeIndexIt : public IIndexIter {
bool IsLeaf() const noexcept {
// Note: it is possible to have 0 levels in B-Tree
// so we may have exhausted state with leaf (data) node
return State.size() == Meta.LevelsCount + 1 && !IsExhausted();
return State.size() == Meta.LevelCount + 1 && !IsExhausted();
}

EReady Exhaust() {
Expand All @@ -335,8 +334,8 @@ class TPartBtreeIndexIt : public IIndexIter {

auto child = current.Node->GetChild(pos);

TRowId beginRowId = pos ? current.Node->GetChild(pos - 1).Count : current.BeginRowId;
TRowId endRowId = child.Count;
TRowId beginRowId = pos ? current.Node->GetChild(pos - 1).RowCount : current.BeginRowId;
TRowId endRowId = child.RowCount;

TCellsIterable beginKey = pos ? current.Node->GetKeyCellsIterable(pos - 1, GroupInfo.ColsKeyIdx) : current.BeginKey;
TCellsIterable endKey = pos < current.Node->GetKeysCount() ? current.Node->GetKeyCellsIterable(pos, GroupInfo.ColsKeyIdx) : current.EndKey;
Expand Down
8 changes: 4 additions & 4 deletions ydb/core/tablet_flat/flat_part_charge.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ namespace NTable {
}
}

TResult Do(const TCells key1, const TCells key2, const TRowId row1, const TRowId row2,
TResult Do(const TCells key1, const TCells key2, TRowId row1, TRowId row2,
const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept override
{
auto index = Index.TryLoadRaw();
Expand Down Expand Up @@ -110,7 +110,7 @@ namespace NTable {
return { ready, overshot };
}

TResult DoReverse(const TCells key1, const TCells key2, const TRowId row1, const TRowId row2,
TResult DoReverse(const TCells key1, const TCells key2, TRowId row1, TRowId row2,
const TKeyCellDefaults &keyDefaults, ui64 itemsLimit, ui64 bytesLimit) const noexcept override
{
auto index = Index.TryLoadRaw();
Expand Down Expand Up @@ -252,7 +252,7 @@ namespace NTable {
}
}
if (itemsLimit && prechargeCurrentFirstRowId <= prechargeCurrentLastRowId) {
ui64 left = itemsLimit - items; // we count only foolprof taken rows, so here we may precharge some extra rows
ui64 left = itemsLimit - items; // we count only foolproof taken rows, so here we may precharge some extra rows
if (prechargeCurrentLastRowId - prechargeCurrentFirstRowId > left) {
prechargeCurrentLastRowId = prechargeCurrentFirstRowId + left;
}
Expand Down Expand Up @@ -347,7 +347,7 @@ namespace NTable {
}

if (itemsLimit && prechargeCurrentFirstRowId >= prechargeCurrentLastRowId) {
ui64 left = itemsLimit - items; // we count only foolprof taken rows, so here we may precharge some extra rows
ui64 left = itemsLimit - items; // we count only foolproof taken rows, so here we may precharge some extra rows
if (prechargeCurrentFirstRowId - prechargeCurrentLastRowId > left) {
prechargeCurrentLastRowId = prechargeCurrentFirstRowId - left;
}
Expand Down
Loading