diff --git a/ydb/core/protos/node_whiteboard.proto b/ydb/core/protos/node_whiteboard.proto index 876fc64fd79d..9ebf822018b8 100644 --- a/ydb/core/protos/node_whiteboard.proto +++ b/ydb/core/protos/node_whiteboard.proto @@ -11,6 +11,7 @@ option java_package = "ru.yandex.kikimr.proto"; extend google.protobuf.FieldOptions { optional uint64 InsignificantChangeAmount = 70553; optional uint32 InsignificantChangePercent = 70554; + optional bool DefaultField = 70555; } enum EFlag { @@ -47,28 +48,29 @@ message TTabletStateInfo { Reserved16 = 16; } - optional uint64 TabletId = 1; + optional uint64 TabletId = 1 [(DefaultField) = true]; optional uint64 CreateTime = 2; - optional uint64 ChangeTime = 3; - optional ETabletState State = 4; + optional uint64 ChangeTime = 3 [(DefaultField) = true]; + optional ETabletState State = 4 [(DefaultField) = true]; optional uint32 UserState = 5; // implementation-dependent - optional uint32 Generation = 6; - optional NKikimrTabletBase.TTabletTypes.EType Type = 7; + optional uint32 Generation = 6 [(DefaultField) = true]; + optional NKikimrTabletBase.TTabletTypes.EType Type = 7 [(DefaultField) = true]; optional string Host = 8; repeated uint32 ChannelGroupIDs = 9; // BS Group per channel repeated TCustomTabletAttribute Attributes = 10; optional uint32 NodeId = 11; // filled during merge - optional bool Leader = 12; // leader or follower + optional bool Leader = 12 [(DefaultField) = true]; // leader or follower optional uint32 Count = 13; // filled during group count - optional uint32 FollowerId = 14; + optional uint32 FollowerId = 14 [(DefaultField) = true]; optional EFlag Overall = 15; // filled during merge - optional NKikimrSubDomains.TDomainKey TenantId = 16; - optional fixed64 HiveId = 17; + optional NKikimrSubDomains.TDomainKey TenantId = 16 [(DefaultField) = true]; + optional fixed64 HiveId = 17 [(DefaultField) = true]; optional string EndOfRangeKeyPrefix = 18; // filled during merge } message TEvTabletStateRequest { optional uint64 ChangedSince = 1; + repeated int32 FieldsRequired = 2 [packed = true]; optional string Format = 5; // it could be "packed5" optional string GroupBy = 20; // it's either empty or "Type,State" for now repeated fixed64 FilterTabletId = 22; @@ -99,6 +101,7 @@ message TNodeStateInfo { message TEvNodeStateRequest { optional uint64 ChangedSince = 1; + repeated int32 FieldsRequired = 2 [packed = true]; } message TEvNodeStateResponse { @@ -108,34 +111,35 @@ message TEvNodeStateResponse { } message TPDiskStateInfo { - optional uint32 PDiskId = 1; + optional uint32 PDiskId = 1 [(DefaultField) = true]; optional uint64 CreateTime = 2; - optional uint64 ChangeTime = 3; - optional string Path = 4; - optional uint64 Guid = 5; - optional uint64 Category = 6; - optional uint64 AvailableSize = 7 [(InsignificantChangeAmount) = 104857600]; // 100Mb - optional uint64 TotalSize = 8; - optional NKikimrBlobStorage.TPDiskState.E State = 9; + optional uint64 ChangeTime = 3 [(DefaultField) = true]; + optional string Path = 4 [(DefaultField) = true]; + optional uint64 Guid = 5 [(DefaultField) = true]; + optional uint64 Category = 6 [(DefaultField) = true]; + optional uint64 AvailableSize = 7 [(DefaultField) = true, (InsignificantChangeAmount) = 104857600]; // 100Mb + optional uint64 TotalSize = 8 [(DefaultField) = true]; + optional NKikimrBlobStorage.TPDiskState.E State = 9 [(DefaultField) = true]; optional uint32 NodeId = 10; // filled during merge optional uint32 Count = 13; // filled during group count - optional EFlag Device = 14; - optional EFlag Realtime = 15; + optional EFlag Device = 14 [(DefaultField) = true]; + optional EFlag Realtime = 15 [(DefaultField) = true]; // State as flag - to be filled optional EFlag StateFlag = 16; // overall state - to be filled optional EFlag Overall = 17; - optional string SerialNumber = 18; - optional uint64 SystemSize = 19; - optional uint64 LogUsedSize = 20; - optional uint64 LogTotalSize = 21; - optional uint32 ExpectedSlotCount = 22; - optional uint64 EnforcedDynamicSlotSize = 23; - optional uint32 NumActiveSlots = 24; + optional string SerialNumber = 18 [(DefaultField) = true]; + optional uint64 SystemSize = 19 [(DefaultField) = true]; + optional uint64 LogUsedSize = 20 [(DefaultField) = true]; + optional uint64 LogTotalSize = 21 [(DefaultField) = true]; + optional uint32 ExpectedSlotCount = 22 [(DefaultField) = true]; + optional uint64 EnforcedDynamicSlotSize = 23 [(DefaultField) = true]; + optional uint32 NumActiveSlots = 24 [(DefaultField) = true]; } message TEvPDiskStateRequest { optional uint64 ChangedSince = 1; + repeated int32 FieldsRequired = 2 [packed = true]; } message TEvPDiskStateResponse { @@ -172,13 +176,13 @@ message TVDiskSatisfactionRank { } message TVDiskStateInfo { - optional NKikimrBlobStorage.TVDiskID VDiskId = 1; + optional NKikimrBlobStorage.TVDiskID VDiskId = 1 [(DefaultField) = true]; optional uint64 CreateTime = 2; - optional uint64 ChangeTime = 3; - optional uint32 PDiskId = 4; - optional uint32 VDiskSlotId = 5; - optional uint64 Guid = 6; - optional uint64 Kind = 7; + optional uint64 ChangeTime = 3 [(DefaultField) = true]; + optional uint32 PDiskId = 4 [(DefaultField) = true]; + optional uint32 VDiskSlotId = 5 [(DefaultField) = true]; + optional uint64 Guid = 6 [(DefaultField) = true]; + optional uint64 Kind = 7 [(DefaultField) = true]; optional uint32 NodeId = 9; // filled during merge optional uint32 Count = 17; // filled during group count @@ -186,48 +190,49 @@ message TVDiskStateInfo { optional EFlag Overall = 10; // Current state of VDisk - optional EVDiskState VDiskState = 11; + optional EVDiskState VDiskState = 11 [(DefaultField) = true]; // Disk space flags - optional EFlag DiskSpace = 12; + optional EFlag DiskSpace = 12 [(DefaultField) = true]; // Compaction satisfaction rank - optional TVDiskSatisfactionRank SatisfactionRank = 13; + optional TVDiskSatisfactionRank SatisfactionRank = 13 [(DefaultField) = true]; // Is VDisk replicated? (i.e. contains all blobs it must have) - optional bool Replicated = 14; + optional bool Replicated = 14 [(DefaultField) = true]; // Does this VDisk has any yet unreplicated phantom-like blobs? - optional bool UnreplicatedPhantoms = 20 [default = false]; + optional bool UnreplicatedPhantoms = 20 [default = false, (DefaultField) = true]; // The same for the non-phantom-like blobs. - optional bool UnreplicatedNonPhantoms = 21 [default = false]; + optional bool UnreplicatedNonPhantoms = 21 [default = false, (DefaultField) = true]; // Replication progress (0 to 1). Only for replication, not blob scrubbing. - optional float ReplicationProgress = 30; + optional float ReplicationProgress = 30 [(DefaultField) = true]; // Replication ETA. - optional uint32 ReplicationSecondsRemaining = 31; + optional uint32 ReplicationSecondsRemaining = 31 [(DefaultField) = true]; // How many unsynced VDisks from current BlobStorage group we see - optional uint64 UnsyncedVDisks = 15 [default = 0]; + optional uint64 UnsyncedVDisks = 15 [default = 0, (DefaultField) = true]; // How much this VDisk have allocated on corresponding PDisk - optional uint64 AllocatedSize = 16 [(InsignificantChangeAmount) = 536870912]; // 512MiB + optional uint64 AllocatedSize = 16 [(InsignificantChangeAmount) = 536870912, (DefaultField) = true]; // 512MiB // How much space is available for VDisk corresponding to PDisk's hard space limits - optional uint64 AvailableSize = 28 [(InsignificantChangeAmount) = 536870912]; // 512MiB + optional uint64 AvailableSize = 28 [(InsignificantChangeAmount) = 536870912, (DefaultField) = true]; // 512MiB // Does this disk has some unreadable but not yet restored blobs? - optional bool HasUnreadableBlobs = 24; - optional fixed64 IncarnationGuid = 25; - optional bool DonorMode = 26; - optional fixed64 InstanceGuid = 27; // VDisk actor instance guid - repeated NKikimrBlobStorage.TVSlotId Donors = 29; + optional bool HasUnreadableBlobs = 24 [(DefaultField) = true]; + optional fixed64 IncarnationGuid = 25 [(DefaultField) = true]; + optional bool DonorMode = 26 [(DefaultField) = true]; + optional fixed64 InstanceGuid = 27 [(DefaultField) = true]; // VDisk actor instance guid + repeated NKikimrBlobStorage.TVSlotId Donors = 29 [(DefaultField) = true]; // VDisk (Skeleton) Front Queue Status - optional EFlag FrontQueues = 18; + optional EFlag FrontQueues = 18 [(DefaultField) = true]; // VDisk storage pool label - optional string StoragePoolName = 19; + optional string StoragePoolName = 19 [(DefaultField) = true]; // Read bytes per second from PDisk for TEvVGet blobs only - optional uint64 ReadThroughput = 22; + optional uint64 ReadThroughput = 22 [(DefaultField) = true]; // Write bytes per second to PDisk for TEvVPut blobs and replication bytes only - optional uint64 WriteThroughput = 23; + optional uint64 WriteThroughput = 23 [(DefaultField) = true]; } message TEvVDiskStateRequest { optional uint64 ChangedSince = 1; + repeated int32 FieldsRequired = 2 [packed = true]; } message TEvVDiskStateResponse { @@ -237,29 +242,30 @@ message TEvVDiskStateResponse { } message TBSGroupStateInfo { - optional uint32 GroupID = 1; - optional string ErasureSpecies = 2; - repeated NKikimrBlobStorage.TVDiskID VDiskIds = 3; + optional uint32 GroupID = 1 [(DefaultField) = true]; + optional string ErasureSpecies = 2 [(DefaultField) = true]; + repeated NKikimrBlobStorage.TVDiskID VDiskIds = 3 [(DefaultField) = true]; optional uint64 ChangeTime = 4; optional uint32 NodeId = 5; // filled during merge - optional uint32 GroupGeneration = 6; + optional uint32 GroupGeneration = 6 [(DefaultField) = true]; optional EFlag Overall = 7; - optional EFlag Latency = 8; + optional EFlag Latency = 8 [(DefaultField) = true]; optional uint32 Count = 13; // filled during group count - optional string StoragePoolName = 14; // from BS_CONTROLLER - optional uint64 AllocatedSize = 15 [(InsignificantChangeAmount) = 100000000]; - optional uint64 AvailableSize = 16 [(InsignificantChangeAmount) = 100000000]; - optional uint64 ReadThroughput = 17; - optional uint64 WriteThroughput = 18; - optional bool Encryption = 19; - repeated uint32 VDiskNodeIds = 20; - optional uint64 BlobDepotId = 21; // if set, then this is virtual group - optional bool NoVDisksInGroup = 22; - optional uint64 BlobDepotOnlineTime = 23; + optional string StoragePoolName = 14 [(DefaultField) = true]; // from BS_CONTROLLER + optional uint64 AllocatedSize = 15 [(InsignificantChangeAmount) = 100000000, (DefaultField) = true]; + optional uint64 AvailableSize = 16 [(InsignificantChangeAmount) = 100000000, (DefaultField) = true]; + optional uint64 ReadThroughput = 17 [(DefaultField) = true]; + optional uint64 WriteThroughput = 18 [(DefaultField) = true]; + optional bool Encryption = 19 [(DefaultField) = true]; + repeated uint32 VDiskNodeIds = 20 [(DefaultField) = true]; + optional uint64 BlobDepotId = 21 [(DefaultField) = true]; // if set, then this is virtual group + optional bool NoVDisksInGroup = 22 [(DefaultField) = true]; + optional uint64 BlobDepotOnlineTime = 23 [(DefaultField) = true]; } message TEvBSGroupStateRequest { optional uint64 ChangedSince = 1; + repeated int32 FieldsRequired = 2 [packed = true]; } message TEvBSGroupStateResponse { @@ -299,12 +305,12 @@ message TSystemStateInfo { optional uint64 LimitBytes = 2; } - optional uint64 StartTime = 1; - optional uint64 ChangeTime = 2; + optional uint64 StartTime = 1 [(DefaultField) = true]; + optional uint64 ChangeTime = 2 [(DefaultField) = true]; optional TLegacyNodeLocation SystemLocation = 3; - repeated double LoadAverage = 4; - optional uint32 NumberOfCpus = 5; - optional EFlag SystemState = 6; + repeated double LoadAverage = 4 [(DefaultField) = true]; + optional uint32 NumberOfCpus = 5 [(DefaultField) = true]; + optional EFlag SystemState = 6 [(DefaultField) = true]; optional EFlag MessageBusState = 7; optional EFlag GRpcState = 8; optional uint32 NodeId = 9; // filled during merge @@ -315,28 +321,29 @@ message TSystemStateInfo { optional uint32 RackId = 17; optional string Rack = 18; optional string Host = 19; - optional string Version = 20; - repeated TPoolStats PoolStats = 21; - repeated TEndpoint Endpoints = 22; - repeated string Roles = 23; - repeated string Tenants = 24; - optional string ClusterName = 25; - optional uint64 MemoryUsed = 26; - optional uint64 MemoryLimit = 27; + optional string Version = 20 [(DefaultField) = true]; + repeated TPoolStats PoolStats = 21 [(DefaultField) = true]; + repeated TEndpoint Endpoints = 22 [(DefaultField) = true]; + repeated string Roles = 23 [(DefaultField) = true]; + repeated string Tenants = 24 [(DefaultField) = true]; + optional string ClusterName = 25 [(DefaultField) = true]; + optional uint64 MemoryUsed = 26 [(DefaultField) = true]; + optional uint64 MemoryLimit = 27 [(DefaultField) = true]; optional EConfigState ConfigState = 28 [default = Consistent]; optional uint64 MemoryUsedInAlloc = 29; - optional double MaxDiskUsage = 30; - optional NActorsInterconnect.TNodeLocation Location = 31; + optional double MaxDiskUsage = 30 [(DefaultField) = true]; + optional NActorsInterconnect.TNodeLocation Location = 31 [(DefaultField) = true]; optional int64 MaxClockSkewWithPeerUs = 32; // a positive value means the peer is ahead in time; a negative value means it's behind optional uint32 MaxClockSkewPeerId = 33; optional uint64 DisconnectTime = 34; optional TNodeSharedCache SharedCacheStats = 35; // TODO: use memory stats - optional uint32 TotalSessions = 36; - optional string NodeName = 37; + optional uint32 TotalSessions = 36 [(DefaultField) = true]; + optional string NodeName = 37 [(DefaultField) = true]; } message TEvSystemStateRequest { optional uint64 ChangedSince = 1; + repeated int32 FieldsRequired = 2 [packed = true]; } message TEvSystemStateResponse { diff --git a/ydb/core/tablet/node_whiteboard.cpp b/ydb/core/tablet/node_whiteboard.cpp index ec31fd234b2d..c85ea2f6ed72 100644 --- a/ydb/core/tablet/node_whiteboard.cpp +++ b/ydb/core/tablet/node_whiteboard.cpp @@ -386,6 +386,136 @@ class TNodeWhiteboardService : public TActorBootstrapped return modified; } + static void CopyField(::google::protobuf::Message& protoTo, + const ::google::protobuf::Message& protoFrom, + const ::google::protobuf::Reflection& reflectionTo, + const ::google::protobuf::Reflection& reflectionFrom, + const ::google::protobuf::FieldDescriptor* field) { + using namespace ::google::protobuf; + if (field->is_repeated()) { + FieldDescriptor::CppType type = field->cpp_type(); + int size = reflectionFrom.FieldSize(protoFrom, field); + if (size != 0) { + reflectionTo.ClearField(&protoTo, field); + for (int i = 0; i < size; ++i) { + switch (type) { + case FieldDescriptor::CPPTYPE_INT32: + reflectionTo.AddInt32(&protoTo, field, reflectionFrom.GetRepeatedInt32(protoFrom, field, i)); + break; + case FieldDescriptor::CPPTYPE_INT64: + reflectionTo.AddInt64(&protoTo, field, reflectionFrom.GetRepeatedInt64(protoFrom, field, i)); + break; + case FieldDescriptor::CPPTYPE_UINT32: + reflectionTo.AddUInt32(&protoTo, field, reflectionFrom.GetRepeatedUInt32(protoFrom, field, i)); + break; + case FieldDescriptor::CPPTYPE_UINT64: + reflectionTo.AddUInt64(&protoTo, field, reflectionFrom.GetRepeatedUInt64(protoFrom, field, i)); + break; + case FieldDescriptor::CPPTYPE_DOUBLE: + reflectionTo.AddDouble(&protoTo, field, reflectionFrom.GetRepeatedDouble(protoFrom, field, i)); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + reflectionTo.AddFloat(&protoTo, field, reflectionFrom.GetRepeatedFloat(protoFrom, field, i)); + break; + case FieldDescriptor::CPPTYPE_BOOL: + reflectionTo.AddBool(&protoTo, field, reflectionFrom.GetRepeatedBool(protoFrom, field, i)); + break; + case FieldDescriptor::CPPTYPE_ENUM: + reflectionTo.AddEnum(&protoTo, field, reflectionFrom.GetRepeatedEnum(protoFrom, field, i)); + break; + case FieldDescriptor::CPPTYPE_STRING: + reflectionTo.AddString(&protoTo, field, reflectionFrom.GetRepeatedString(protoFrom, field, i)); + break; + case FieldDescriptor::CPPTYPE_MESSAGE: + reflectionTo.AddMessage(&protoTo, field)->CopyFrom(reflectionFrom.GetRepeatedMessage(protoFrom, field, i)); + break; + } + } + } + } else { + if (reflectionFrom.HasField(protoFrom, field)) { + FieldDescriptor::CppType type = field->cpp_type(); + switch (type) { + case FieldDescriptor::CPPTYPE_INT32: + reflectionTo.SetInt32(&protoTo, field, reflectionFrom.GetInt32(protoFrom, field)); + break; + case FieldDescriptor::CPPTYPE_INT64: + reflectionTo.SetInt64(&protoTo, field, reflectionFrom.GetInt64(protoFrom, field)); + break; + case FieldDescriptor::CPPTYPE_UINT32: + reflectionTo.SetUInt32(&protoTo, field, reflectionFrom.GetUInt32(protoFrom, field)); + break; + case FieldDescriptor::CPPTYPE_UINT64: + reflectionTo.SetUInt64(&protoTo, field, reflectionFrom.GetUInt64(protoFrom, field)); + break; + case FieldDescriptor::CPPTYPE_DOUBLE: + reflectionTo.SetDouble(&protoTo, field, reflectionFrom.GetDouble(protoFrom, field)); + break; + case FieldDescriptor::CPPTYPE_FLOAT: + reflectionTo.SetFloat(&protoTo, field, reflectionFrom.GetFloat(protoFrom, field)); + break; + case FieldDescriptor::CPPTYPE_BOOL: + reflectionTo.SetBool(&protoTo, field, reflectionFrom.GetBool(protoFrom, field)); + break; + case FieldDescriptor::CPPTYPE_ENUM: + reflectionTo.SetEnum(&protoTo, field, reflectionFrom.GetEnum(protoFrom, field)); + break; + case FieldDescriptor::CPPTYPE_STRING: + reflectionTo.SetString(&protoTo, field, reflectionFrom.GetString(protoFrom, field)); + break; + case FieldDescriptor::CPPTYPE_MESSAGE: + reflectionTo.MutableMessage(&protoTo, field)->CopyFrom(reflectionFrom.GetMessage(protoFrom, field)); + break; + } + } + } + } + + static void SelectiveCopy(::google::protobuf::Message& protoTo, const ::google::protobuf::Message& protoFrom, const ::google::protobuf::RepeatedField& fields) { + using namespace ::google::protobuf; + const Descriptor& descriptor = *protoTo.GetDescriptor(); + const Reflection& reflectionTo = *protoTo.GetReflection(); + const Reflection& reflectionFrom = *protoFrom.GetReflection(); + for (auto fieldNumber : fields) { + const FieldDescriptor* field = descriptor.FindFieldByNumber(fieldNumber); + if (field) { + CopyField(protoTo, protoFrom, reflectionTo, reflectionFrom, field); + } + } + } + + template + static ::google::protobuf::RepeatedField GetDefaultFields(const TMessage& message) { + using namespace ::google::protobuf; + const Descriptor& descriptor = *message.GetDescriptor(); + ::google::protobuf::RepeatedField defaultFields; + int fieldCount = descriptor.field_count(); + for (int index = 0; index < fieldCount; ++index) { + const FieldDescriptor* field = descriptor.field(index); + const auto& options(field->options()); + if (options.HasExtension(NKikimrWhiteboard::DefaultField)) { + if (options.GetExtension(NKikimrWhiteboard::DefaultField)) { + defaultFields.Add(field->number()); + } + } + } + return defaultFields; + } + + template + static void Copy(TMessage& to, const TMessage& from, const TRequest& request) { + if (request.FieldsRequiredSize() > 0) { + if (request.FieldsRequiredSize() == 1 && request.GetFieldsRequired(0) == -1) { // all fields + to.CopyFrom(from); + } else { + SelectiveCopy(to, from, request.GetFieldsRequired()); + } + } else { + static auto defaultFields = GetDefaultFields(to); + SelectiveCopy(to, from, defaultFields); + } + } + void SetRole(TStringBuf roleName) { for (const auto& role : SystemStateInfo.GetRoles()) { if (role == roleName) { @@ -709,14 +839,6 @@ class TNodeWhiteboardService : public TActorBootstrapped } } - static void CopyTabletStateInfo( - NKikimrWhiteboard::TTabletStateInfo& dst, - const NKikimrWhiteboard::TTabletStateInfo& src, - const NKikimrWhiteboard::TEvTabletStateRequest&) - { - dst = src; - } - void Handle(TEvWhiteboard::TEvTabletStateRequest::TPtr &ev, const TActorContext &ctx) { auto now = TMonotonic::Now(); const auto& request = ev->Get()->Record; @@ -739,7 +861,7 @@ class TNodeWhiteboardService : public TActorBootstrapped for (const auto& pr : TabletStateInfo) { if (pr.second.changetime() >= changedSince) { NKikimrWhiteboard::TTabletStateInfo& tabletStateInfo = *record.add_tabletstateinfo(); - CopyTabletStateInfo(tabletStateInfo, pr.second, request); + Copy(tabletStateInfo, pr.second, request); } } } else { @@ -748,12 +870,12 @@ class TNodeWhiteboardService : public TActorBootstrapped if (it != TabletStateInfo.end()) { if (it->second.changetime() >= changedSince) { NKikimrWhiteboard::TTabletStateInfo& tabletStateInfo = *record.add_tabletstateinfo(); - CopyTabletStateInfo(tabletStateInfo, it->second, request); + Copy(tabletStateInfo, it->second, request); } } } } - } else if (request.groupby() == "Type,State") { // the only supported group-by for now + } else if (request.groupby() == "Type,State" || request.groupby() == "NodeId,Type,State") { // the only supported group-by for now std::unordered_map, NKikimrWhiteboard::TTabletStateInfo> stateGroupBy; for (const auto& [id, stateInfo] : TabletStateInfo) { @@ -784,7 +906,7 @@ class TNodeWhiteboardService : public TActorBootstrapped for (const auto& pr : NodeStateInfo) { if (pr.second.GetChangeTime() >= changedSince) { NKikimrWhiteboard::TNodeStateInfo &nodeStateInfo = *record.AddNodeStateInfo(); - nodeStateInfo.CopyFrom(pr.second); + Copy(nodeStateInfo, pr.second, request); } } response->Record.SetResponseTime(ctx.Now().MilliSeconds()); @@ -815,7 +937,7 @@ class TNodeWhiteboardService : public TActorBootstrapped for (const auto& pr : PDiskStateInfo) { if (pr.second.GetChangeTime() >= changedSince) { NKikimrWhiteboard::TPDiskStateInfo &pDiskStateInfo = *record.AddPDiskStateInfo(); - pDiskStateInfo.CopyFrom(pr.second); + Copy(pDiskStateInfo, pr.second, request); } } response->Record.SetResponseTime(ctx.Now().MilliSeconds()); @@ -839,7 +961,7 @@ class TNodeWhiteboardService : public TActorBootstrapped for (const auto& pr : VDiskStateInfo) { if (pr.second.GetChangeTime() >= changedSince) { NKikimrWhiteboard::TVDiskStateInfo &vDiskStateInfo = *record.AddVDiskStateInfo(); - vDiskStateInfo.CopyFrom(pr.second); + Copy(vDiskStateInfo, pr.second, request); } } response->Record.SetResponseTime(ctx.Now().MilliSeconds()); @@ -854,7 +976,7 @@ class TNodeWhiteboardService : public TActorBootstrapped for (const auto& pr : BSGroupStateInfo) { if (pr.second.GetChangeTime() >= changedSince) { NKikimrWhiteboard::TBSGroupStateInfo &bSGroupStateInfo = *record.AddBSGroupStateInfo(); - bSGroupStateInfo.CopyFrom(pr.second); + Copy(bSGroupStateInfo, pr.second, request); } } response->Record.SetResponseTime(ctx.Now().MilliSeconds()); @@ -868,7 +990,7 @@ class TNodeWhiteboardService : public TActorBootstrapped auto& record = response->Record; if (SystemStateInfo.GetChangeTime() >= changedSince) { NKikimrWhiteboard::TSystemStateInfo &systemStateInfo = *record.AddSystemStateInfo(); - systemStateInfo.CopyFrom(SystemStateInfo); + Copy(systemStateInfo, SystemStateInfo, request); } response->Record.SetResponseTime(ctx.Now().MilliSeconds()); ctx.Send(ev->Sender, response.Release(), 0, ev->Cookie); diff --git a/ydb/core/viewer/json_wb_req.h b/ydb/core/viewer/json_wb_req.h index 08a46ce37f25..31a984ac6b25 100644 --- a/ydb/core/viewer/json_wb_req.h +++ b/ydb/core/viewer/json_wb_req.h @@ -63,6 +63,13 @@ class TJsonWhiteboardRequest : public TWhiteboardRequest(params.Get("static"), false); } + if (params.Has("fields_required")) { + if (params.Get("fields_required") == "all") { + TBase::RequestSettings.FieldsRequired = {-1}; + } else { + SplitIds(params.Get("fields_required"), ',', TBase::RequestSettings.FieldsRequired); + } + } TBase::RequestSettings.Format = params.Get("format"); TBase::Bootstrap(); } diff --git a/ydb/core/viewer/protos/viewer.proto b/ydb/core/viewer/protos/viewer.proto index e77956162a03..0b217623ecba 100644 --- a/ydb/core/viewer/protos/viewer.proto +++ b/ydb/core/viewer/protos/viewer.proto @@ -637,12 +637,16 @@ message TEvViewerRequest { oneof Request { NKikimrWhiteboard.TEvTabletStateRequest TabletRequest = 11; NKikimrWhiteboard.TEvSystemStateRequest SystemRequest = 12; + NKikimrWhiteboard.TEvVDiskStateRequest VDiskRequest = 16; + NKikimrWhiteboard.TEvPDiskStateRequest PDiskRequest = 17; + NKikimrWhiteboard.TEvBSGroupStateRequest BSGroupRequest = 18; THttpProxyRequest QueryRequest = 13; THttpProxyRequest RenderRequest = 14; TSchemeCacheRequest AutocompleteRequest = 15; - bytes Reserved16 = 16; - bytes Reserved17 = 17; - bytes Reserved18 = 18; + bytes Reserved19 = 19; + bytes Reserved20 = 20; + bytes Reserved21 = 21; + bytes Reserved22 = 22; } } @@ -651,12 +655,16 @@ message TEvViewerResponse { oneof Response { NKikimrWhiteboard.TEvTabletStateResponse TabletResponse = 11; NKikimrWhiteboard.TEvSystemStateResponse SystemResponse = 12; + NKikimrWhiteboard.TEvVDiskStateResponse VDiskResponse = 16; + NKikimrWhiteboard.TEvPDiskStateResponse PDiskResponse = 17; + NKikimrWhiteboard.TEvBSGroupStateResponse BSGroupResponse = 18; NKikimrKqp.TEvQueryResponse QueryResponse = 13; NKikimrGraph.TEvMetricsResult RenderResponse = 14; TQueryAutocomplete AutocompleteResponse = 15; - bytes Reserved16 = 16; - bytes Reserved17 = 17; - bytes Reserved18 = 18; + bytes Reserved19 = 19; + bytes Reserved20 = 20; + bytes Reserved21 = 21; + bytes Reserved22 = 22; } } diff --git a/ydb/core/viewer/viewer.h b/ydb/core/viewer/viewer.h index bb5ae1e333fb..d1f83dbb4932 100644 --- a/ydb/core/viewer/viewer.h +++ b/ydb/core/viewer/viewer.h @@ -48,7 +48,7 @@ struct TRequestSettings { TDuration RetryPeriod = TDuration::MilliSeconds(500); TString Format; std::optional StaticNodesOnly; - bool DistributedMerge = false; + std::vector FieldsRequired; bool Followers = true; // hive tablet info bool Metrics = true; // hive tablet info diff --git a/ydb/core/viewer/viewer_nodes.h b/ydb/core/viewer/viewer_nodes.h index 8bd48849dbed..6fa2c409ef71 100644 --- a/ydb/core/viewer/viewer_nodes.h +++ b/ydb/core/viewer/viewer_nodes.h @@ -121,6 +121,7 @@ class TJsonNodes : public TViewerPipeClient { ui32 UptimeSeconds = 0; bool ProblemNodesOnly = false; TString Filter; + bool AllWhiteboardFields = false; enum class EWith { Everything, @@ -667,6 +668,9 @@ class TJsonNodes : public TViewerPipeClient { } } } + if (FromStringWithDefault(params.Get("all_whiteboard_fields"), false)) { + AllWhiteboardFields = true; + } } void Bootstrap() override { @@ -1507,6 +1511,13 @@ class TJsonNodes : public TViewerPipeClient { } } + template + void InitWhiteboardRequest(TWhiteboardEvent* request) { + if (AllWhiteboardFields) { + request->AddFieldsRequired(-1); + } + } + void SendWhiteboardSystemAndTabletsBatch(TNodeBatch& batch) { TNodeId nodeId = OffloadMerge ? batch.ChooseNodeId() : 0; if (batch.HasStaticNodes && (FieldsNeeded(FieldsVDisks) || FieldsNeeded(FieldsPDisks))) { @@ -1515,7 +1526,7 @@ class TJsonNodes : public TViewerPipeClient { if (nodeId) { if (FieldsNeeded(FieldsSystemState) && SystemViewerResponse.count(nodeId) == 0) { auto viewerRequest = std::make_unique(); - viewerRequest->Record.MutableSystemRequest(); + InitWhiteboardRequest(viewerRequest->Record.MutableSystemRequest()); viewerRequest->Record.SetTimeout(Timeout / 2); for (const TNode* node : batch.NodesToAskAbout) { viewerRequest->Record.MutableLocation()->AddNodeId(node->GetNodeId()); @@ -1543,7 +1554,9 @@ class TJsonNodes : public TViewerPipeClient { TNodeId nodeId = node->GetNodeId(); if (FieldsNeeded(FieldsSystemState)) { if (SystemStateResponse.count(nodeId) == 0) { - SystemStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, new TEvWhiteboard::TEvSystemStateRequest())); + auto request = new TEvWhiteboard::TEvSystemStateRequest(); + InitWhiteboardRequest(&request->Record); + SystemStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, request)); ++WhiteboardStateRequestsInFlight; } } @@ -1567,13 +1580,17 @@ class TJsonNodes : public TViewerPipeClient { if (node->IsStatic()) { if (FieldsNeeded(FieldsVDisks)) { if (VDiskStateResponse.count(nodeId) == 0) { - VDiskStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, new TEvWhiteboard::TEvVDiskStateRequest())); + auto request = new TEvWhiteboard::TEvVDiskStateRequest(); + InitWhiteboardRequest(&request->Record); + VDiskStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, request)); ++WhiteboardStateRequestsInFlight; } } if (FieldsNeeded(FieldsPDisks)) { if (PDiskStateResponse.count(nodeId) == 0) { - PDiskStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, new TEvWhiteboard::TEvPDiskStateRequest())); + auto request = new TEvWhiteboard::TEvPDiskStateRequest(); + InitWhiteboardRequest(&request->Record); + PDiskStateResponse.emplace(nodeId, MakeWhiteboardRequest(nodeId, request)); ++WhiteboardStateRequestsInFlight; } } @@ -1787,15 +1804,40 @@ class TJsonNodes : public TViewerPipeClient { SystemViewerResponse[nodeId].Set(std::move(ev)); NodeBatches.erase(nodeId); WhiteboardRequestDone(); - break; + return; case NKikimrViewer::TEvViewerResponse::ResponseCase::kTabletResponse: TabletViewerResponse[nodeId].Set(std::move(ev)); NodeBatches.erase(nodeId); WhiteboardRequestDone(); - break; + return; default: break; } + TString error("WrongResponse"); + { + auto itSystemViewerResponse = SystemViewerResponse.find(nodeId); + if (itSystemViewerResponse != SystemViewerResponse.end()) { + if (itSystemViewerResponse->second.Error(error)) { + if (NodeBatches.count(nodeId)) { + SendWhiteboardSystemAndTabletsBatch(NodeBatches[nodeId]); + NodeBatches.erase(nodeId); + } + WhiteboardRequestDone(); + } + } + } + { + auto itTabletViewerResponse = TabletViewerResponse.find(nodeId); + if (itTabletViewerResponse != TabletViewerResponse.end()) { + if (itTabletViewerResponse->second.Error(error)) { + if (NodeBatches.count(nodeId)) { + SendWhiteboardSystemAndTabletsBatch(NodeBatches[nodeId]); + NodeBatches.erase(nodeId); + } + WhiteboardRequestDone(); + } + } + } } void Handle(NSysView::TEvSysView::TEvGetStoragePoolsResponse::TPtr& ev) { diff --git a/ydb/core/viewer/viewer_request.cpp b/ydb/core/viewer/viewer_request.cpp index 030f95eea682..a6abd5c1b49d 100644 --- a/ydb/core/viewer/viewer_request.cpp +++ b/ydb/core/viewer/viewer_request.cpp @@ -4,6 +4,9 @@ #include "viewer_render.h" #include "viewer_sysinfo.h" #include "viewer_tabletinfo.h" +#include "viewer_vdiskinfo.h" +#include "viewer_pdiskinfo.h" +#include "viewer_bsgroupinfo.h" #include "wb_req.h" namespace NKikimr::NViewer { @@ -41,16 +44,36 @@ class TViewerWhiteboardRequest : public TWhiteboardRequest void MergeWhiteboardResponses(TEvViewer::TEvViewerResponse* response, TMap& perNodeStateInfo, const TString& fields); + THolder BuildRequest() override; - template<> void MergeWhiteboardResponses(TEvViewer::TEvViewerResponse* response, TMap& perNodeStateInfo, const TString& fields) { + template + void MergeWhiteboardResponses(TEvViewer::TEvViewerResponse* response, TMap& perNodeStateInfo, const TString& fields); + + template<> + void MergeWhiteboardResponses(TEvViewer::TEvViewerResponse* response, TMap& perNodeStateInfo, const TString& fields) { NKikimr::NViewer::MergeWhiteboardResponses(*(response->Record.MutableTabletResponse()), perNodeStateInfo, fields); } - template<> void MergeWhiteboardResponses(TEvViewer::TEvViewerResponse* response, TMap& perNodeStateInfo, const TString& fields) { + template<> + void MergeWhiteboardResponses(TEvViewer::TEvViewerResponse* response, TMap& perNodeStateInfo, const TString& fields) { NKikimr::NViewer::MergeWhiteboardResponses(*(response->Record.MutableSystemResponse()), perNodeStateInfo, fields); } + template<> + void MergeWhiteboardResponses(TEvViewer::TEvViewerResponse* response, TMap& perNodeStateInfo, const TString& fields) { + NKikimr::NViewer::MergeWhiteboardResponses(*(response->Record.MutableVDiskResponse()), perNodeStateInfo, fields); + } + + template<> + void MergeWhiteboardResponses(TEvViewer::TEvViewerResponse* response, TMap& perNodeStateInfo, const TString& fields) { + NKikimr::NViewer::MergeWhiteboardResponses(*(response->Record.MutablePDiskResponse()), perNodeStateInfo, fields); + } + + template<> + void MergeWhiteboardResponses(TEvViewer::TEvViewerResponse* response, TMap& perNodeStateInfo, const TString& fields) { + NKikimr::NViewer::MergeWhiteboardResponses(*(response->Record.MutableBSGroupResponse()), perNodeStateInfo, fields); + } + void ReplyAndPassAway() override { auto response = MakeHolder(); auto& locationResponded = (*response->Record.MutableLocationResponded()); @@ -72,21 +95,59 @@ IActor* CreateViewerRequestHandler(TEvViewer::TEvViewerRequest::TPtr& request) { return new TViewerWhiteboardRequest(request); case NKikimrViewer::TEvViewerRequest::kSystemRequest: return new TViewerWhiteboardRequest(request); + case NKikimrViewer::TEvViewerRequest::kVDiskRequest: + return new TViewerWhiteboardRequest(request); + case NKikimrViewer::TEvViewerRequest::kPDiskRequest: + return new TViewerWhiteboardRequest(request); + case NKikimrViewer::TEvViewerRequest::kBSGroupRequest: + return new TViewerWhiteboardRequest(request); case NKikimrViewer::TEvViewerRequest::kQueryRequest: return new TJsonQueryOld(request); case NKikimrViewer::TEvViewerRequest::kRenderRequest: return new TJsonRender(request); case NKikimrViewer::TEvViewerRequest::kAutocompleteRequest: return new TJsonAutocomplete(request); - case NKikimrViewer::TEvViewerRequest::kReserved16: - case NKikimrViewer::TEvViewerRequest::kReserved17: - case NKikimrViewer::TEvViewerRequest::kReserved18: - case NKikimrViewer::TEvViewerRequest::REQUEST_NOT_SET: + default: return nullptr; } return nullptr; } +template<> +THolder TViewerWhiteboardRequest::BuildRequest() { + auto request = TBase::BuildRequest(); + request->Record.MergeFrom(Event->Get()->Record.GetTabletRequest()); + return request; +} + +template<> +THolder TViewerWhiteboardRequest::BuildRequest() { + auto request = TBase::BuildRequest(); + request->Record.MergeFrom(Event->Get()->Record.GetSystemRequest()); + return request; +} + +template<> +THolder TViewerWhiteboardRequest::BuildRequest() { + auto request = TBase::BuildRequest(); + request->Record.MergeFrom(Event->Get()->Record.GetVDiskRequest()); + return request; +} + +template<> +THolder TViewerWhiteboardRequest::BuildRequest() { + auto request = TBase::BuildRequest(); + request->Record.MergeFrom(Event->Get()->Record.GetPDiskRequest()); + return request; +} + +template<> +THolder TViewerWhiteboardRequest::BuildRequest() { + auto request = TBase::BuildRequest(); + request->Record.MergeFrom(Event->Get()->Record.GetBSGroupRequest()); + return request; +} + bool IsPostContent(const NMon::TEvHttpInfo::TPtr& event) { if (event->Get()->Request.GetMethod() == HTTP_METHOD_POST) { const THttpHeaders& headers = event->Get()->Request.GetHeaders(); diff --git a/ydb/core/viewer/wb_req.h b/ydb/core/viewer/wb_req.h index aa636be58f19..4a9800bb276d 100644 --- a/ydb/core/viewer/wb_req.h +++ b/ydb/core/viewer/wb_req.h @@ -63,6 +63,14 @@ class TWhiteboardRequest : public TViewerPipeClient { request->Record.SetFormat(RequestSettings.Format); } } + + constexpr bool hasFieldsRequired = requires(const TRequestEventType* r) {r->Record.GetFieldsRequired();}; + if constexpr (hasFieldsRequired) { + if (!RequestSettings.FieldsRequired.empty()) { + request->Record.MutableFieldsRequired()->Add(RequestSettings.FieldsRequired.begin(), RequestSettings.FieldsRequired.end()); + } + } + if (RequestSettings.ChangedSince != 0) { request->Record.SetChangedSince(RequestSettings.ChangedSince); }