Merge pull request ydb-platform#6368 from uzhastik/merge_from_ydb_24_2

q-stable-24-2: merge from ydb-24-2

ydb/apps/version/version_definition.cpp

@@ -9,30 +9,14 @@ NKikimrConfig::TCurrentCompatibilityInfo NKikimr::TCompatibilityInfo::MakeCurren
         .Application = "ydb",
         .Version = TVersionConstructor{
             .Year = 24,
-            .Major = 1,
-        },
-        .CanLoadFrom = {
-            TCompatibilityRuleConstructor{
-                .LowerLimit = TVersionConstructor{ .Year = 23, .Major = 4 },
-                .UpperLimit = TVersionConstructor{ .Year = 24, .Major = 1 },
-            },
-        },
-        .StoresReadableBy = {
-            TCompatibilityRuleConstructor{
-                .LowerLimit = TVersionConstructor{ .Year = 23, .Major = 4 },
-                .UpperLimit = TVersionConstructor{ .Year = 24, .Major = 1 },
-            },
+            .Major = 2,
         },
         .CanConnectTo = {
-            TCompatibilityRuleConstructor{
-                .LowerLimit = TVersionConstructor{ .Year = 23, .Major = 4 },
-                .UpperLimit = TVersionConstructor{ .Year = 24, .Major = 1 },
-            },
             TCompatibilityRuleConstructor{
                 .Application = "nbs",
                 .LowerLimit = TVersionConstructor{ .Year = 23, .Major = 3 },
-                .UpperLimit = TVersionConstructor{ .Year = 24, .Major = 1 },
+                .UpperLimit = TVersionConstructor{ .Year = 24, .Major = 2 },
             },
-        }
+        },
     }.ToPB();
 }

ydb/core/base/appdata_fwd.h

@@ -239,6 +239,7 @@ struct TAppData {
     TVector<TString> AdministrationAllowedSIDs; // users/groups which allowed to perform administrative tasks
     TVector<TString> DefaultUserSIDs;
     TString AllAuthenticatedUsers = "all-users@well-known";
+    TVector<TString> RegisterDynamicNodeAllowedSIDs;
     TString TenantName;
     TString NodeName;
 

ydb/core/blobstorage/vdisk/huge/blobstorage_hullhugeheap.cpp

@@ -538,13 +538,26 @@ namespace NKikimr {
                 }
                 Y_ABORT_UNLESS(loadedIt == loadedEnd);
             } else {
-                // entry point size rollback case
                 Y_ABORT_UNLESS(size > ChainDelegators.size());
-                ui32 curChainDelegatorsSize = ChainDelegators.size();
-                Y_FAIL_S("Impossible case; MinHugeBlobInBytes# " << MinHugeBlobInBytes
-                        << " MilestoneBlobInBytes# " << MilestoneBlobInBytes
-                        << " loadedSize# " << size
-                        << " curChainDelegatorsSize# " << curChainDelegatorsSize);
+
+                // skip first delegators, which must not be used
+                for (size_t i = ChainDelegators.size(); i < size; ++i) {
+                    ui32 slotsInChunk;
+                    ::Load(s, slotsInChunk);
+                    ui32 allocatedSlots;
+                    ::Load(s, allocatedSlots);
+                    TMap<ui32, TMask> freeSpace;
+                    ::Load(s, freeSpace);
+                    Y_ABORT_UNLESS(slotsInChunk > ChainDelegators.front().SlotsInChunk, "incompatible format");
+                    Y_ABORT_UNLESS(!allocatedSlots, "incompatible format");
+                    Y_ABORT_UNLESS(freeSpace.empty(), "incompatible format");
+                }
+
+                // load the rest as usual
+                StartMode = EStartMode::Loaded;
+                for (TChainDelegator& delegator : ChainDelegators) {
+                    ::Load(s, delegator);
+                }
             }
         }
 

ydb/core/client/server/grpc_server.cpp

@@ -478,7 +478,7 @@ void TGRpcService::SetupIncomingRequests() {
     // dynamic node registration
     ADD_REQUEST(RegisterNode, TNodeRegistrationRequest, TNodeRegistrationResponse, {
         NMsgBusProxy::TBusMessageContext msg(ctx->BindBusContext(NMsgBusProxy::MTYPE_CLIENT_NODE_REGISTRATION_REQUEST));
-        RegisterRequestActor(CreateMessageBusRegisterNode(msg, DynamicNodeAuthorizationParams));
+        RegisterRequestActor(CreateMessageBusRegisterNode(msg));
     })
 
     // CMS request

ydb/core/client/server/grpc_server.h

@@ -1,6 +1,4 @@
 #pragma once
-#include <ydb/core/grpc_services/auth_processor/dynamic_node_auth_processor.h>
-
 #include <ydb/core/protos/grpc.grpc.pb.h>
 
 #include <ydb/library/actors/core/actorsystem.h>
@@ -60,10 +58,6 @@ class TGRpcService
 public:
     TGRpcService();
 
-    void SetDynamicNodeAuthParams(const TDynamicNodeAuthorizationParams& dynamicNodeAuthorizationParams) {
-        DynamicNodeAuthorizationParams = dynamicNodeAuthorizationParams;
-    }
-
     void InitService(grpc::ServerCompletionQueue* cq, NYdbGrpc::TLoggerPtr logger) override;
     void SetGlobalLimiterHandle(NYdbGrpc::TGlobalLimiter* limiter) override;
 
@@ -98,8 +92,6 @@ class TGRpcService
     std::function<void()> InitCb_;
     // In flight request management.
     NYdbGrpc::TGlobalLimiter* Limiter_ = nullptr;
-
-    TDynamicNodeAuthorizationParams DynamicNodeAuthorizationParams = {};
 };
 
 }

ydb/core/client/server/msgbus_server.h

@@ -1,5 +1,4 @@
 #pragma once
-#include <ydb/core/grpc_services/auth_processor/dynamic_node_auth_processor.h>
 #include <ydb/library/actors/core/actorsystem.h>
 #include <ydb/library/actors/core/actor_bootstrapped.h>
 #include <ydb/public/lib/base/defs.h>
@@ -298,7 +297,7 @@ IActor* CreateMessageBusBlobStorageConfig(TBusMessageContext &msg);
 IActor* CreateMessageBusDrainNode(TBusMessageContext &msg);
 IActor* CreateMessageBusFillNode(TBusMessageContext &msg);
 IActor* CreateMessageBusResolveNode(TBusMessageContext &msg);
-IActor* CreateMessageBusRegisterNode(TBusMessageContext &msg, const TDynamicNodeAuthorizationParams& dynamicNodeAuthorizationParams);
+IActor* CreateMessageBusRegisterNode(TBusMessageContext &msg);
 IActor* CreateMessageBusCmsRequest(TBusMessageContext &msg);
 IActor* CreateMessageBusSqsRequest(TBusMessageContext &msg);
 IActor* CreateMessageBusWhoAmI(TBusMessageContext &msg);

ydb/core/client/server/msgbus_server_console.cpp

@@ -30,8 +30,20 @@ class TConsoleRequestActor : public TMessageBusSecureRequest<TMessageBusServerRe
         : TBase(msg)
         , Request(request)
     {
-        TBase::SetSecurityToken(request.GetSecurityToken());
-        TBase::SetRequireAdminAccess(true);
+        const auto& token = request.GetSecurityToken();
+        if (!token.empty()) {
+            TBase::SetSecurityToken(token);
+        } else {
+            const auto& clientCertificates = msg.FindClientCert();
+            if (!clientCertificates.empty()) {
+                TBase::SetSecurityToken(TString(clientCertificates.front()));
+            }
+        }
+        // Don`t require admin access for GetNodeConfigRequest
+        if (Request.GetRequestCase() != NKikimrClient::TConsoleRequest::kGetNodeConfigRequest) {
+            TBase::SetRequireAdminAccess(true);
+        }
+
     }
 
     void Bootstrap(const TActorContext &ctx)
@@ -120,6 +132,10 @@ class TConsoleRequestActor : public TMessageBusSecureRequest<TMessageBusServerRe
             request->Record.CopyFrom(Request.GetGetNodeConfigItemsRequest());
             NTabletPipe::SendData(ctx, ConsolePipe, request.Release());
         } else if (Request.HasGetNodeConfigRequest()) {
+            if (!CheckAccessGetNodeConfig()) {
+                ReplyWithErrorAndDie(Ydb::StatusIds::UNAUTHORIZED, "Cannot get node config. Access denied. Node is not authorized", ctx);
+                return;
+            }
             auto request = MakeHolder<TEvConsole::TEvGetNodeConfigRequest>();
             request->Record.CopyFrom(Request.GetGetNodeConfigRequest());
             NTabletPipe::SendData(ctx, ConsolePipe, request.Release());
@@ -348,6 +364,21 @@ class TConsoleRequestActor : public TMessageBusSecureRequest<TMessageBusServerRe
         }
     }
 
+    bool CheckAccessGetNodeConfig() const {
+        const auto serializedToken = TBase::GetSerializedToken();
+        // Empty serializedToken means token is not required. Checked in secure_request.h
+        if (!serializedToken.empty() && !AppData()->RegisterDynamicNodeAllowedSIDs.empty()) {
+            NACLib::TUserToken token(serializedToken);
+            for (const auto& sid : AppData()->RegisterDynamicNodeAllowedSIDs) {
+                if (token.IsExist(sid)) {
+                    return true;
+                }
+            }
+            return false;
+        }
+        return true;
+    }
+
 private:
     NKikimrClient::TConsoleRequest Request;
     NKikimrClient::TConsoleResponse Response;

ydb/core/client/server/msgbus_server_node_registration.cpp

@@ -1,4 +1,5 @@
-#include "msgbus_servicereq.h"
+#include "msgbus_server_request.h"
+#include "msgbus_securereq.h"
 #include "grpc_server.h"
 
 #include <ydb/library/actors/core/actor_bootstrapped.h>
@@ -19,36 +20,35 @@ using namespace NNodeBroker;
 
 namespace {
 
-class TNodeRegistrationActor : public TActorBootstrapped<TNodeRegistrationActor>, public TMessageBusSessionIdentHolder
+class TNodeRegistrationActor : public TMessageBusSecureRequest<TMessageBusServerRequestBase<TNodeRegistrationActor>>
 {
     using TActorBase = TActorBootstrapped<TNodeRegistrationActor>;
-
-    struct TNodeAuthorizationResult {
-        bool IsAuthorized = false;
-        bool IsCertificateUsed = false;
-
-        operator bool() const {
-            return IsAuthorized;
-        }
-    };
+    using TBase = TMessageBusSecureRequest<TMessageBusServerRequestBase<TNodeRegistrationActor>>;
 
 public:
     static constexpr NKikimrServices::TActivity::EType ActorActivityType() {
         return NKikimrServices::TActivity::MSGBUS_COMMON;
     }
 
-    TNodeRegistrationActor(NKikimrClient::TNodeRegistrationRequest &request, NMsgBusProxy::TBusMessageContext &msg, const NKikimr::TDynamicNodeAuthorizationParams& dynamicNodeAuthorizationParams)
-        : TMessageBusSessionIdentHolder(msg)
+    TNodeRegistrationActor(NKikimrClient::TNodeRegistrationRequest &request, NMsgBusProxy::TBusMessageContext &msg)
+        : TBase(msg)
         , Request(request)
-        , DynamicNodeAuthorizationParams(dynamicNodeAuthorizationParams)
     {
+        const auto& clientCertificates = msg.FindClientCert();
+        if (!clientCertificates.empty()) {
+            TBase::SetSecurityToken(TString(clientCertificates.front()));
+        } else {
+            TBase::SetSecurityToken(BUILTIN_ACL_ROOT); // NBS compatibility
+        }
     }
 
     void Bootstrap(const TActorContext &ctx)
     {
-        const TNodeAuthorizationResult nodeAuthorizationResult = IsNodeAuthorized();
-        if (!nodeAuthorizationResult.IsAuthorized) {
+        if (!CheckAccess()) {
+            Response.MutableStatus()->SetCode(TStatus::UNAUTHORIZED);
+            Response.MutableStatus()->SetReason("Cannot authorize node. Access denied");
             SendReplyAndDie(ctx);
+            return;
         }
 
         auto dinfo = AppData(ctx)->DomainsInfo;
@@ -90,7 +90,7 @@ class TNodeRegistrationActor : public TActorBootstrapped<TNodeRegistrationActor>
         if (Request.HasPath()) {
             request->Record.SetPath(Request.GetPath());
         }
-        request->Record.SetAuthorizedByCertificate(nodeAuthorizationResult.IsCertificateUsed);
+        request->Record.SetAuthorizedByCertificate(IsNodeAuthorizedByCertificate);
 
         NTabletPipe::SendData(ctx, NodeBrokerPipe, request.Release());
 
@@ -157,7 +157,7 @@ class TNodeRegistrationActor : public TActorBootstrapped<TNodeRegistrationActor>
     void Die(const TActorContext &ctx)
     {
         NTabletPipe::CloseClient(ctx, NodeBrokerPipe);
-        TActorBase::Die(ctx);
+        TBase::Die(ctx);
     }
 
     void SendReplyAndDie(const TActorContext &ctx)
@@ -186,52 +186,34 @@ class TNodeRegistrationActor : public TActorBootstrapped<TNodeRegistrationActor>
     }
 
 private:
-    TNodeAuthorizationResult IsNodeAuthorized() {
-        TNodeAuthorizationResult result {.IsAuthorized = false, .IsCertificateUsed = false};
-        auto* appdata = AppData();
-        if (appdata && appdata->FeatureFlags.GetEnableDynamicNodeAuthorization() && DynamicNodeAuthorizationParams) {
-            const auto& nodeAuthValues = FindClientCert();
-            if (nodeAuthValues.empty()) {
-                Response.MutableStatus()->SetCode(TStatus::UNAUTHORIZED);
-                Response.MutableStatus()->SetReason("Cannot authorize node. Node has not provided certificate");
-                return result;
-            }
-            const auto& pemCert = nodeAuthValues.front();
-            TMap<TString, TString> subjectDescription;
-            X509CertificateReader::X509Ptr x509cert = X509CertificateReader::ReadCertAsPEM(pemCert);
-            for(const auto& term: X509CertificateReader::ReadSubjectTerms(x509cert)) {
-                subjectDescription.insert(term);
-            }
-
-            if (!DynamicNodeAuthorizationParams.IsSubjectDescriptionMatched(subjectDescription)) {
-                Response.MutableStatus()->SetCode(TStatus::UNAUTHORIZED);
-                Response.MutableStatus()->SetReason("Cannot authorize node by certificate");
-                return result;
-            }
-            const auto& host = Request.GetHost();
-            if (!DynamicNodeAuthorizationParams.IsHostMatchAttributeCN(host)) {
-                Response.MutableStatus()->SetCode(TStatus::UNAUTHORIZED);
-                Response.MutableStatus()->SetReason("Cannot authorize node with host: " + host);
-                return result;
+    bool CheckAccess() {
+        const auto serializedToken = TBase::GetSerializedToken();
+        // Empty serializedToken means token is not required. Checked in secure_request.h
+        if (!serializedToken.empty() && !AppData()->RegisterDynamicNodeAllowedSIDs.empty()) {
+            NACLib::TUserToken token(serializedToken);
+            for (const auto& sid : AppData()->RegisterDynamicNodeAllowedSIDs) {
+                if (token.IsExist(sid)) {
+                    IsNodeAuthorizedByCertificate = true;
+                    return true;
+                }
             }
-            result.IsCertificateUsed = true;
+            return false;
         }
-        result.IsAuthorized = true;
-        return result;;
+        return true;
     }
 
     NKikimrClient::TNodeRegistrationRequest Request;
     NKikimrClient::TNodeRegistrationResponse Response;
     TActorId NodeBrokerPipe;
-    const TDynamicNodeAuthorizationParams DynamicNodeAuthorizationParams;
+    bool IsNodeAuthorizedByCertificate = false;
 };
 
 } // namespace
 
-IActor *CreateMessageBusRegisterNode(NMsgBusProxy::TBusMessageContext &msg, const NKikimr::TDynamicNodeAuthorizationParams& dynamicNodeAuthorizationParams) {
+IActor *CreateMessageBusRegisterNode(NMsgBusProxy::TBusMessageContext &msg) {
     NKikimrClient::TNodeRegistrationRequest &record
         = static_cast<TBusNodeRegistrationRequest*>(msg.GetMessage())->Record;
-    return new TNodeRegistrationActor(record, msg, dynamicNodeAuthorizationParams);
+    return new TNodeRegistrationActor(record, msg);
 }
 
 } // namespace NMsgBusProxy

ydb/core/client/server/ya.make

@@ -74,7 +74,7 @@ PEERDIR(
     ydb/core/engine
     ydb/core/engine/minikql
     ydb/core/grpc_services
-    ydb/core/grpc_services/auth_processor
+    ydb/core/security/certificate_check
     ydb/core/grpc_services/base
     ydb/core/keyvalue
     ydb/core/kqp/common

ydb/core/cms/json_proxy_proto.h

@@ -76,6 +76,8 @@ class TJsonProxyProto : public TActorBootstrapped<TJsonProxyProto> {
                 return ReplyWithTypeDescription(*NKikimrConfig::TImmediateControlsConfig::TSchemeShardControls::descriptor(), ctx);
             else if (name == ".NKikimrConfig.TImmediateControlsConfig.TTCMallocControls")
                 return ReplyWithTypeDescription(*NKikimrConfig::TImmediateControlsConfig::TTCMallocControls::descriptor(), ctx);
+            else if (name == ".NKikimrConfig.TImmediateControlsConfig.TVDiskControls")
+                return ReplyWithTypeDescription(*NKikimrConfig::TImmediateControlsConfig::TVDiskControls::descriptor(), ctx);
             else if (name == ".NKikimrConfig.TImmediateControlsConfig.TTabletControls")
                 return ReplyWithTypeDescription(*NKikimrConfig::TImmediateControlsConfig::TTabletControls::descriptor(), ctx);
         }

ydb/core/driver_lib/run/config.h

@@ -84,6 +84,7 @@ union TBasicKikimrServicesMask {
 
         bool EnableDatabaseMetadataCache:1;
         bool EnableGraphService:1;
+        bool EnableCompDiskLimiter:1;
     };
 
     struct {