Skip to content

Commit

Permalink
clean TUnifiedBlobId. prepare for use TLogoBlobId only (ydb-platform#…
Browse files Browse the repository at this point in the history 
…2171)
  • Loading branch information
@ivanmorozov333
ivanmorozov333 committed Apr 16, 2024
1 parent b676a62 commit 563f027
Show file tree
Hide file tree
Showing 6 changed files with 22 additions and 319 deletions.
13 changes: 1 addition & 12 deletions ydb/core/tx/columnshard/blob_cache.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,18 +62,8 @@ class TBlobCache: public TActorBootstrapped<TBlobCache> {
// (e.g. DS blobs from the same tablet residing on the same DS group, or 2 small blobs from the same tablet)
std::tuple<ui64, ui32, EReadVariant> BlobSource() const {
const TUnifiedBlobId& blobId = BlobRange.BlobId;

Y_ABORT_UNLESS(blobId.IsValid());

if (blobId.IsDsBlob()) {
// Tablet & group restriction
return {blobId.GetTabletId(), blobId.GetDsGroup(), ReadVariant()};
} else if (blobId.IsSmallBlob()) {
// Tablet restriction, no group restrictions
return {blobId.GetTabletId(), 0, ReadVariant()};
}

return {0, 0, EReadVariant::FAST};
return {blobId.GetTabletId(), blobId.GetDsGroup(), ReadVariant()};
}
};

Expand Down Expand Up @@ -409,7 +399,6 @@ class TBlobCache: public TActorBootstrapped<TBlobCache> {
ui64 requestSize = 0;
ui32 dsGroup = std::get<1>(target);
TReadItem::EReadVariant readVariant = std::get<2>(target);
Y_ABORT_UNLESS(rangesGroup.begin()->BlobId.IsDsBlob());

std::vector<ui64> dsReads;

Expand Down
5 changes: 0 additions & 5 deletions ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,16 +150,13 @@ bool TBlobManager::LoadState(IBlobManagerDb& db, const TTabletId selfTabletId) {
}

for (auto it = BlobsToDelete.GetIterator(); it.IsValid(); ++it) {
AFL_VERIFY(it.GetBlobId().IsDsBlob());
BlobsManagerCounters.OnDeleteBlobMarker(it.GetBlobId().BlobSize());
}
BlobsManagerCounters.OnBlobsDelete(BlobsToDelete);

// Build the list of steps that cannot be garbage collected before Keep flag is set on the blobs
THashSet<TGenStep> genStepsWithBlobsToKeep;
for (const auto& unifiedBlobId : blobsToKeep) {
Y_ABORT_UNLESS(unifiedBlobId.IsDsBlob(), "Not a DS blob id in Keep table: %s", unifiedBlobId.ToStringNew().c_str());

TLogoBlobID blobId = unifiedBlobId.GetLogoBlobId();
TGenStep genStep{blobId.Generation(), blobId.Step()};
Y_ABORT_UNLESS(genStep > LastCollectedGenStep);
Expand Down Expand Up @@ -332,8 +329,6 @@ void TBlobManager::DoSaveBlobBatch(TBlobBatch&& blobBatch, IBlobManagerDb& db) {
// Add this batch to KeepQueue
TGenStep edgeGenStep = EdgeGenStep();
for (auto&& blobId: blobBatch.BatchInfo->GetBlobIds()) {
Y_DEBUG_ABORT_UNLESS(blobId.IsDsBlob(), "Not a DS blob id: %s", blobId.ToStringNew().c_str());

auto logoBlobId = blobId.GetLogoBlobId();
TGenStep genStep{logoBlobId.Generation(), logoBlobId.Step()};

Expand Down
114 changes: 6 additions & 108 deletions ydb/core/tx/columnshard/common/blob.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,59 +5,6 @@

namespace NKikimr::NOlap {

// Format: "S3-f(logoBlobId)-group"
// Example: "S3-42-72075186224038245_51_31595_2_0_11952_0-2181038103"
TString DsIdToS3Key(const TUnifiedBlobId& dsid, const ui64 pathId) {
TString blobId = dsid.GetLogoBlobId().ToString();
for (auto&& c : blobId) {
switch (c) {
case ':':
c = '_';
break;
case '[':
case ']':
c = '-';
}
}
TString result =
"S3-" +
::ToString(pathId) +
blobId +
::ToString(dsid.GetDsGroup())
;
return result;
}

TUnifiedBlobId S3KeyToDsId(const TString& s, TString& error, ui64& pathId) {
TVector<TString> keyBucket;
Split(s, "-", keyBucket);

ui32 dsGroup;
if (keyBucket.size() != 4 || keyBucket[0] != "S3"
|| !TryFromString<ui32>(keyBucket[3], dsGroup)
|| !TryFromString<ui64>(keyBucket[1], pathId))
{
error = TStringBuilder() << "Wrong S3 key '" << s << "'";
return TUnifiedBlobId();
}

TString blobId = "[" + keyBucket[2] + "]";
for (size_t i = 0; i < blobId.size(); ++i) {
switch (blobId[i]) {
case '_':
blobId[i] = ':';
break;
}
}

TLogoBlobID logoBlobId;
if (!TLogoBlobID::Parse(logoBlobId, blobId, error)) {
return TUnifiedBlobId();
}

return TUnifiedBlobId(dsGroup, logoBlobId);
}

namespace {

#define PARSE_INT_COMPONENT(fieldType, fieldName, endChar) \
Expand Down Expand Up @@ -103,49 +50,10 @@ TUnifiedBlobId ParseExtendedDsBlobId(const TString& s, TString& error) {
return TUnifiedBlobId(dsGroup, logoBlobId);
}

// Format: "SM[tabletId:generation:step:cookie:size]"
// Example: "SM[72075186224038245:51:31184:0:2528]"
TUnifiedBlobId ParseSmallBlobId(const TString& s, TString& error) {
Y_ABORT_UNLESS(s.size() > 2);
const char* str = s.c_str();
Y_ABORT_UNLESS(str[0] == 'S' && str[1] == 'M');
i64 pos = 2;
i64 endPos = s.size();
if (str[pos++] != '[') {
error = "opening [ not found";
return TUnifiedBlobId();
}

PARSE_INT_COMPONENT(ui64, tabletId, ':');
PARSE_INT_COMPONENT(ui32, gen, ':');
PARSE_INT_COMPONENT(ui32, step, ':');
PARSE_INT_COMPONENT(ui32, cookie, ':');
PARSE_INT_COMPONENT(ui32, size, ']');

if (pos != endPos) {
error = "Extra characters after closing ]";
return TUnifiedBlobId();
}

return TUnifiedBlobId(tabletId, gen, step, cookie, size);
}

// Format: "s = S3_key"
TUnifiedBlobId ParseS3BlobId(const TString& s, TString& error) {
ui64 pathId;
TUnifiedBlobId dsBlobId = S3KeyToDsId(s, error, pathId);
if (!dsBlobId.IsValid()) {
return TUnifiedBlobId();
}

return TUnifiedBlobId(dsBlobId, TUnifiedBlobId::S3_BLOB, pathId);
}

}

TUnifiedBlobId TUnifiedBlobId::ParseFromString(const TString& str,
const IBlobGroupSelector* dsGroupSelector, TString& error)
{
const IBlobGroupSelector* dsGroupSelector, TString& error) {
if (str.size() <= 2) {
error = TStringBuilder() << "Wrong blob id: '" << str << "'";
return TUnifiedBlobId();
Expand All @@ -160,24 +68,14 @@ TUnifiedBlobId TUnifiedBlobId::ParseFromString(const TString& str,
error = "Cannot parse TLogoBlobID: " + error;
return TUnifiedBlobId();
}
if (logoBlobId.Channel() == TSmallBlobId::FAKE_CHANNEL) {
// Small blob
return TUnifiedBlobId(logoBlobId.TabletID(), logoBlobId.Generation(), logoBlobId.Step(),
logoBlobId.Cookie(), logoBlobId.BlobSize());
} else {
// DS blob
if (!dsGroupSelector) {
error = "Need TBlobGroupSelector to resolve DS group for the blob";
return TUnifiedBlobId();
}
return TUnifiedBlobId(dsGroupSelector->GetGroup(logoBlobId), logoBlobId);
// DS blob
if (!dsGroupSelector) {
error = "Need TBlobGroupSelector to resolve DS group for the blob";
return TUnifiedBlobId();
}
return TUnifiedBlobId(dsGroupSelector->GetGroup(logoBlobId), logoBlobId);
} else if (str[0] == 'D' && str[1] == 'S') {
return ParseExtendedDsBlobId(str, error);
} else if (str[0] == 'S' && str[1] == 'M') {
return ParseSmallBlobId(str, error);
} else if (str[0] == 'S' && str[1] == '3') {
return ParseS3BlobId(str, error);
}

error = TStringBuilder() << "Wrong blob id: '" << str << "'";
Expand Down
Loading

0 comments on commit 563f027

Please sign in to comment.