Skip to content

Commit

Permalink
KIKIMR-19522 BTreeIndex Charge Items Limit (#1338)
Browse files Browse the repository at this point in the history
  • Loading branch information
@kunga
kunga authored Jan 31, 2024
1 parent ef309ff commit 694286b
Show file tree
Hide file tree
Showing 7 changed files with 373 additions and 240 deletions.
39 changes: 25 additions & 14 deletions ydb/core/tablet_flat/flat_page_btree_index.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,11 @@ namespace NKikimr::NTable::NPage {

auto operator<=>(const TChild&) const = default;

TString ToString() const noexcept
{
TRowId GetNonErasedRowCount() const noexcept {
return RowCount - ErasedRowCount;
}

TString ToString() const noexcept {
return TStringBuilder() << "PageId: " << PageId << " RowCount: " << RowCount << " DataSize: " << DataSize << " ErasedRowCount: " << ErasedRowCount;
}
} Y_PACKED;
Expand Down Expand Up @@ -268,6 +271,11 @@ namespace NKikimr::NTable::NPage {
return ReadUnaligned<NPage::TLabel>(Raw.data());
}

bool IsShortChildFormat() const noexcept
{
return Header->IsShortChildFormat;
}

bool IsFixedFormat() const noexcept
{
return Header->FixedKeySize != Header->MaxFixedKeySize;
Expand All @@ -293,23 +301,26 @@ namespace NKikimr::NTable::NPage {
return GetCells<TCellsIter>(pos, columns);
}

const TShortChild* GetShortChildRef(TRecIdx pos) const noexcept
{
return TDeref<const TShortChild>::At(Children,
pos * (Header->IsShortChildFormat ? sizeof(TShortChild) : sizeof(TChild)));
}

const TShortChild& GetShortChild(TRecIdx pos) const noexcept
{
if (Header->IsShortChildFormat) {
return *TDeref<const TShortChild>::At(Children, pos * sizeof(TShortChild));
} else {
return *TDeref<const TShortChild>::At(Children, pos * sizeof(TChild));
}
return *GetShortChildRef(pos);
}

TChild GetChild(TRecIdx pos) const noexcept
const TChild* GetChildRef(TRecIdx pos) const noexcept
{
if (Header->IsShortChildFormat) {
const TShortChild* const shortChild = TDeref<const TShortChild>::At(Children, pos * sizeof(TShortChild));
return { shortChild->PageId, shortChild->RowCount, shortChild->DataSize, 0 };
} else {
return *TDeref<const TChild>::At(Children, pos * sizeof(TChild));
}
Y_DEBUG_ABORT_UNLESS(!Header->IsShortChildFormat, "GetShortChildRef should be used instead");
return TDeref<const TChild>::At(Children, pos * sizeof(TChild));
}

const TChild& GetChild(TRecIdx pos) const noexcept
{
return *GetChildRef(pos);
}

static bool Has(TRowId rowId, TRowId beginRowId, TRowId endRowId) noexcept {
Expand Down
18 changes: 9 additions & 9 deletions ydb/core/tablet_flat/flat_part_btree_index_iter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,16 @@ class TPartBtreeIndexIt : public IIndexIter {
using TBtreeIndexMeta = NPage::TBtreeIndexMeta;

struct TNodeState {
TChild Meta;
TPageId PageId;
TRowId BeginRowId;
TRowId EndRowId;
TCellsIterable BeginKey;
TCellsIterable EndKey;
std::optional<TBtreeIndexNode> Node;
std::optional<TRecIdx> Pos;

TNodeState(TChild meta, TRowId beginRowId, TRowId endRowId, TCellsIterable beginKey, TCellsIterable endKey)
: Meta(meta)
TNodeState(TPageId pageId, TRowId beginRowId, TRowId endRowId, TCellsIterable beginKey, TCellsIterable endKey)
: PageId(pageId)
, BeginRowId(beginRowId)
, EndRowId(endRowId)
, BeginKey(beginKey)
Expand Down Expand Up @@ -118,7 +118,7 @@ class TPartBtreeIndexIt : public IIndexIter {
, State(Reserve(Meta.LevelCount + 1))
{
const static TCellsIterable EmptyKey(static_cast<const char*>(nullptr), TColumns());
State.emplace_back(Meta, 0, GetEndRowId(), EmptyKey, EmptyKey);
State.emplace_back(Meta.PageId, 0, GetEndRowId(), EmptyKey, EmptyKey);
}

EReady Seek(TRowId rowId) override {
Expand Down Expand Up @@ -251,7 +251,7 @@ class TPartBtreeIndexIt : public IIndexIter {

TPageId GetPageId() const override {
Y_ABORT_UNLESS(IsLeaf());
return State.back().Meta.PageId;
return State.back().PageId;
}

TRowId GetRowId() const override {
Expand Down Expand Up @@ -332,23 +332,23 @@ class TPartBtreeIndexIt : public IIndexIter {
Y_ABORT_UNLESS(pos < current.Node->GetChildrenCount(), "Should point to some child");
current.Pos.emplace(pos);

auto child = current.Node->GetChild(pos);
auto& child = current.Node->GetShortChild(pos);

TRowId beginRowId = pos ? current.Node->GetChild(pos - 1).RowCount : current.BeginRowId;
TRowId beginRowId = pos ? current.Node->GetShortChild(pos - 1).RowCount : current.BeginRowId;
TRowId endRowId = child.RowCount;

TCellsIterable beginKey = pos ? current.Node->GetKeyCellsIterable(pos - 1, GroupInfo.ColsKeyIdx) : current.BeginKey;
TCellsIterable endKey = pos < current.Node->GetKeysCount() ? current.Node->GetKeyCellsIterable(pos, GroupInfo.ColsKeyIdx) : current.EndKey;

State.emplace_back(child, beginRowId, endRowId, beginKey, endKey);
State.emplace_back(child.PageId, beginRowId, endRowId, beginKey, endKey);
}

bool TryLoad(TNodeState& state) {
if (state.Node) {
return true;
}

auto page = Env->TryGetPage(Part, state.Meta.PageId);
auto page = Env->TryGetPage(Part, state.PageId);
if (page) {
state.Node.emplace(*page);
return true;
Expand Down
61 changes: 32 additions & 29 deletions ydb/core/tablet_flat/flat_part_charge.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,7 @@ namespace NTable {
ui64 items = 0;
ui64 bytes = 0;

TRowId prechargedFirstRowId, prechargedLastRowId;
std::optional<std::pair<TRowId, TRowId>> prechargedRowsRange;
bool needExactBounds = Groups || HistoryIndex;

for (auto current = first;
Expand All @@ -235,6 +235,15 @@ namespace NTable {
prechargeCurrentFirstRowId = Max<TRowId>(); // no precharge
}
}
if (itemsLimit && prechargeCurrentFirstRowId <= prechargeCurrentLastRowId) {
ui64 left = itemsLimit - items; // we count only foolproof taken rows, so here we may precharge some extra rows
if (prechargeCurrentLastRowId - prechargeCurrentFirstRowId > left) {
prechargeCurrentLastRowId = prechargeCurrentFirstRowId + left;
}
}
if (prechargeCurrentFirstRowId <= prechargeCurrentLastRowId) {
items += prechargeCurrentLastRowId - prechargeCurrentFirstRowId + 1;
}
if (key2Page && key2Page <= current) {
if (key2Page == current) {
if (needExactBounds && page) {
Expand All @@ -251,29 +260,22 @@ namespace NTable {
prechargeCurrentFirstRowId = Max<TRowId>(); // no precharge
}
}
if (itemsLimit && prechargeCurrentFirstRowId <= prechargeCurrentLastRowId) {
ui64 left = itemsLimit - items; // we count only foolproof taken rows, so here we may precharge some extra rows
if (prechargeCurrentLastRowId - prechargeCurrentFirstRowId > left) {
prechargeCurrentLastRowId = prechargeCurrentFirstRowId + left;
}
}

if (prechargeCurrentFirstRowId <= prechargeCurrentLastRowId) {
if (!items) {
prechargedFirstRowId = prechargeCurrentFirstRowId;
if (prechargedRowsRange) {
prechargedRowsRange->second = prechargeCurrentLastRowId;
} else {
prechargedRowsRange.emplace(prechargeCurrentFirstRowId, prechargeCurrentLastRowId);
}
prechargedLastRowId = prechargeCurrentLastRowId;
if (Groups) {
for (auto& g : Groups) {
ready &= DoPrechargeGroup(g, prechargeCurrentFirstRowId, prechargeCurrentLastRowId, bytes);
}
}
items += prechargeCurrentLastRowId - prechargeCurrentFirstRowId + 1;
}
}

if (items && HistoryIndex) {
ready &= DoPrechargeHistory(prechargedFirstRowId, prechargedLastRowId);
if (prechargedRowsRange && HistoryIndex) {
ready &= DoPrechargeHistory(prechargedRowsRange->first, prechargedRowsRange->second);
}
}

Expand Down Expand Up @@ -304,7 +306,7 @@ namespace NTable {
ui64 items = 0;
ui64 bytes = 0;

TRowId prechargedFirstRowId, prechargedLastRowId;
std::optional<std::pair<TRowId, TRowId>> prechargedRowsRange;
bool needExactBounds = Groups || HistoryIndex;

for (auto current = first;
Expand Down Expand Up @@ -335,6 +337,15 @@ namespace NTable {
prechargeCurrentLastRowId = Max<TRowId>(); // no precharge
}
}
if (itemsLimit && prechargeCurrentFirstRowId >= prechargeCurrentLastRowId) {
ui64 left = itemsLimit - items; // we count only foolproof taken rows, so here we may precharge some extra rows
if (prechargeCurrentFirstRowId - prechargeCurrentLastRowId > left) {
prechargeCurrentLastRowId = prechargeCurrentFirstRowId - left;
}
}
if (prechargeCurrentFirstRowId >= prechargeCurrentLastRowId) {
items += prechargeCurrentFirstRowId - prechargeCurrentLastRowId + 1;
}
if (key2Page && key2Page >= current) {
if (key2Page == current) {
if (needExactBounds && page) {
Expand All @@ -349,34 +360,26 @@ namespace NTable {
prechargeCurrentLastRowId = Max<TRowId>(); // no precharge
}
}

if (itemsLimit && prechargeCurrentFirstRowId >= prechargeCurrentLastRowId) {
ui64 left = itemsLimit - items; // we count only foolproof taken rows, so here we may precharge some extra rows
if (prechargeCurrentFirstRowId - prechargeCurrentLastRowId > left) {
prechargeCurrentLastRowId = prechargeCurrentFirstRowId - left;
}
}

if (prechargeCurrentFirstRowId >= prechargeCurrentLastRowId) {
if (!items) {
prechargedFirstRowId = prechargeCurrentFirstRowId;
if (prechargedRowsRange) {
prechargedRowsRange->second = prechargeCurrentLastRowId;
} else {
prechargedRowsRange.emplace(prechargeCurrentFirstRowId, prechargeCurrentLastRowId);
}
prechargedLastRowId = prechargeCurrentLastRowId;
if (Groups) {
for (auto& g : Groups) {
ready &= DoPrechargeGroupReverse(g, prechargeCurrentFirstRowId, prechargeCurrentLastRowId, bytes);
}
}
items += prechargeCurrentFirstRowId - prechargeCurrentLastRowId + 1;
}

if (current.Off() == 0) {
break;
}
}

if (items && HistoryIndex) {
ready &= DoPrechargeHistory(prechargedFirstRowId, prechargedLastRowId);
if (prechargedRowsRange && HistoryIndex) {
ready &= DoPrechargeHistory(prechargedRowsRange->first, prechargedRowsRange->second);
}
}

Expand Down
Loading

0 comments on commit 694286b

Please sign in to comment.