forked from ydb-platform/ydb
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
fix splitter build. correct libraries usage for blobs too. (ydb-platf…
- Loading branch information
1 parent
a45af4b
commit 8b6fa2f
Showing
9 changed files
with
697 additions
and
691 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,218 +1 @@ | ||
| #include "blob.h" | ||
| #include "defs.h" | ||
| #include <ydb/core/tx/columnshard/common/protos/blob_range.pb.h> | ||
|
|
||
| #include <charconv> | ||
|
|
||
| namespace NKikimr::NOlap { | ||
|
|
||
| // Format: "S3-f(logoBlobId)-group" | ||
| // Example: "S3-42-72075186224038245_51_31595_2_0_11952_0-2181038103" | ||
| TString DsIdToS3Key(const TUnifiedBlobId& dsid, const ui64 pathId) { | ||
| TString blobId = dsid.GetLogoBlobId().ToString(); | ||
| for (auto&& c : blobId) { | ||
| switch (c) { | ||
| case ':': | ||
| c = '_'; | ||
| break; | ||
| case '[': | ||
| case ']': | ||
| c = '-'; | ||
| } | ||
| } | ||
| TString result = | ||
| "S3-" + | ||
| ::ToString(pathId) + | ||
| blobId + | ||
| ::ToString(dsid.GetDsGroup()) | ||
| ; | ||
| return result; | ||
| } | ||
|
|
||
| TUnifiedBlobId S3KeyToDsId(const TString& s, TString& error, ui64& pathId) { | ||
| TVector<TString> keyBucket; | ||
| Split(s, "-", keyBucket); | ||
|
|
||
| ui32 dsGroup; | ||
| if (keyBucket.size() != 4 || keyBucket[0] != "S3" | ||
| || !TryFromString<ui32>(keyBucket[3], dsGroup) | ||
| || !TryFromString<ui64>(keyBucket[1], pathId)) | ||
| { | ||
| error = TStringBuilder() << "Wrong S3 key '" << s << "'"; | ||
| return TUnifiedBlobId(); | ||
| } | ||
|
|
||
| TString blobId = "[" + keyBucket[2] + "]"; | ||
| for (size_t i = 0; i < blobId.size(); ++i) { | ||
| switch (blobId[i]) { | ||
| case '_': | ||
| blobId[i] = ':'; | ||
| break; | ||
| } | ||
| } | ||
|
|
||
| TLogoBlobID logoBlobId; | ||
| if (!TLogoBlobID::Parse(logoBlobId, blobId, error)) { | ||
| return TUnifiedBlobId(); | ||
| } | ||
|
|
||
| return TUnifiedBlobId(dsGroup, logoBlobId); | ||
| } | ||
|
|
||
| namespace { | ||
|
|
||
| #define PARSE_INT_COMPONENT(fieldType, fieldName, endChar) \ | ||
| if (pos >= endPos) { \ | ||
| error = "Failed to parse " #fieldName " component"; \ | ||
| return TUnifiedBlobId(); \ | ||
| } \ | ||
| fieldType fieldName = -1; \ | ||
| { \ | ||
| auto [ptr, ec] { std::from_chars(str + pos, str + endPos, fieldName) }; \ | ||
| if (ec != std::errc()) { \ | ||
| error = "Failed to parse " #fieldName " component"; \ | ||
| return TUnifiedBlobId(); \ | ||
| } else { \ | ||
| pos = ptr - str; \ | ||
| } \ | ||
| if (str[pos++] != endChar) { \ | ||
| error = #endChar " not found after " #fieldName; \ | ||
| return TUnifiedBlobId(); \ | ||
| } \ | ||
| } | ||
|
|
||
| // Format: "DS:group:logoBlobId" | ||
| // Example: "DS:2181038103:[72075186224038245:51:31595:2:0:11952:0]" | ||
| TUnifiedBlobId ParseExtendedDsBlobId(const TString& s, TString& error) { | ||
| Y_ABORT_UNLESS(s.size() > 2); | ||
| const char* str = s.c_str(); | ||
| Y_ABORT_UNLESS(str[0] == 'D' && str[1] == 'S'); | ||
| i64 pos = 2; | ||
| i64 endPos = s.size(); | ||
| if (str[pos++] != ':') { | ||
| error = "Starting ':' not found"; | ||
| return TUnifiedBlobId(); | ||
| } | ||
|
|
||
| PARSE_INT_COMPONENT(ui32, dsGroup, ':'); | ||
|
|
||
| TLogoBlobID logoBlobId; | ||
| if (!TLogoBlobID::Parse(logoBlobId, s.substr(pos), error)) { | ||
| return TUnifiedBlobId(); | ||
| } | ||
|
|
||
| return TUnifiedBlobId(dsGroup, logoBlobId); | ||
| } | ||
|
|
||
| // Format: "SM[tabletId:generation:step:cookie:size]" | ||
| // Example: "SM[72075186224038245:51:31184:0:2528]" | ||
| TUnifiedBlobId ParseSmallBlobId(const TString& s, TString& error) { | ||
| Y_ABORT_UNLESS(s.size() > 2); | ||
| const char* str = s.c_str(); | ||
| Y_ABORT_UNLESS(str[0] == 'S' && str[1] == 'M'); | ||
| i64 pos = 2; | ||
| i64 endPos = s.size(); | ||
| if (str[pos++] != '[') { | ||
| error = "opening [ not found"; | ||
| return TUnifiedBlobId(); | ||
| } | ||
|
|
||
| PARSE_INT_COMPONENT(ui64, tabletId, ':'); | ||
| PARSE_INT_COMPONENT(ui32, gen, ':'); | ||
| PARSE_INT_COMPONENT(ui32, step, ':'); | ||
| PARSE_INT_COMPONENT(ui32, cookie, ':'); | ||
| PARSE_INT_COMPONENT(ui32, size, ']'); | ||
|
|
||
| if (pos != endPos) { | ||
| error = "Extra characters after closing ]"; | ||
| return TUnifiedBlobId(); | ||
| } | ||
|
|
||
| return TUnifiedBlobId(tabletId, gen, step, cookie, size); | ||
| } | ||
|
|
||
| // Format: "s = S3_key" | ||
| TUnifiedBlobId ParseS3BlobId(const TString& s, TString& error) { | ||
| ui64 pathId; | ||
| TUnifiedBlobId dsBlobId = S3KeyToDsId(s, error, pathId); | ||
| if (!dsBlobId.IsValid()) { | ||
| return TUnifiedBlobId(); | ||
| } | ||
|
|
||
| return TUnifiedBlobId(dsBlobId, TUnifiedBlobId::S3_BLOB, pathId); | ||
| } | ||
|
|
||
| } | ||
|
|
||
| TUnifiedBlobId TUnifiedBlobId::ParseFromString(const TString& str, | ||
| const IBlobGroupSelector* dsGroupSelector, TString& error) | ||
| { | ||
| if (str.size() <= 2) { | ||
| error = TStringBuilder() << "Wrong blob id: '" << str << "'"; | ||
| return TUnifiedBlobId(); | ||
| } | ||
|
|
||
| if (str[0] == '[') { | ||
| // If blobId starts with '[' this must be a logoblobId and if channel is set to FAKE_CHANNEL | ||
| // this is a fake logoblobid used for small blob | ||
| TLogoBlobID logoBlobId; | ||
| bool parsed = TLogoBlobID::Parse(logoBlobId, str, error); | ||
| if (!parsed) { | ||
| error = "Cannot parse TLogoBlobID: " + error; | ||
| return TUnifiedBlobId(); | ||
| } | ||
| if (logoBlobId.Channel() == TSmallBlobId::FAKE_CHANNEL) { | ||
| // Small blob | ||
| return TUnifiedBlobId(logoBlobId.TabletID(), logoBlobId.Generation(), logoBlobId.Step(), | ||
| logoBlobId.Cookie(), logoBlobId.BlobSize()); | ||
| } else { | ||
| // DS blob | ||
| if (!dsGroupSelector) { | ||
| error = "Need TBlobGroupSelector to resolve DS group for the blob"; | ||
| return TUnifiedBlobId(); | ||
| } | ||
| return TUnifiedBlobId(dsGroupSelector->GetGroup(logoBlobId), logoBlobId); | ||
| } | ||
| } else if (str[0] == 'D' && str[1] == 'S') { | ||
| return ParseExtendedDsBlobId(str, error); | ||
| } else if (str[0] == 'S' && str[1] == 'M') { | ||
| return ParseSmallBlobId(str, error); | ||
| } else if (str[0] == 'S' && str[1] == '3') { | ||
| return ParseS3BlobId(str, error); | ||
| } | ||
|
|
||
| error = TStringBuilder() << "Wrong blob id: '" << str << "'"; | ||
| return TUnifiedBlobId(); | ||
| } | ||
|
|
||
| NKikimr::TConclusionStatus TBlobRange::DeserializeFromProto(const NKikimrColumnShardProto::TBlobRange& proto) { | ||
| auto parsed = TUnifiedBlobId::BuildFromString(proto.GetBlobId(), nullptr); | ||
| if (!parsed) { | ||
| return parsed; | ||
| } | ||
| BlobId = parsed.DetachResult(); | ||
|
|
||
| Offset = proto.GetOffset(); | ||
| Size = proto.GetSize(); | ||
| return TConclusionStatus::Success(); | ||
| } | ||
|
|
||
| NKikimr::TConclusion<NKikimr::NOlap::TBlobRange> TBlobRange::BuildFromProto(const NKikimrColumnShardProto::TBlobRange& proto) { | ||
| TBlobRange result; | ||
| auto parsed = result.DeserializeFromProto(proto); | ||
| if (!parsed) { | ||
| return parsed; | ||
| } else { | ||
| return result; | ||
| } | ||
| } | ||
|
|
||
| NKikimrColumnShardProto::TBlobRange TBlobRange::SerializeToProto() const { | ||
| NKikimrColumnShardProto::TBlobRange result; | ||
| result.SetBlobId(BlobId.ToStringNew()); | ||
| result.SetOffset(Offset); | ||
| result.SetSize(Size); | ||
| return result; | ||
| } | ||
|
|
||
| } |
Oops, something went wrong.