From 6576464d6b8dca046c27847c1ff60fc6748f329d Mon Sep 17 00:00:00 2001 From: Vasily Gerasimov Date: Mon, 1 Jul 2024 22:25:22 +0300 Subject: [PATCH] Http authorization support (#5982) --- ydb/core/grpc_services/base/base.h | 248 +++++++++++++++++- .../grpc_services/grpc_request_check_actor.h | 34 ++- ydb/core/grpc_services/grpc_request_proxy.cpp | 8 +- ydb/core/grpc_services/grpc_request_proxy.h | 10 - ydb/core/mon/async_http_mon.cpp | 52 +++- ydb/core/mon/mon.cpp | 72 +++-- ydb/core/mon/mon.h | 1 - ydb/core/mon/ya.make | 5 +- ydb/core/testlib/actors/test_runtime.cpp | 21 +- ydb/core/testlib/test_client.cpp | 2 +- ydb/core/testlib/test_client.h | 3 +- ydb/core/viewer/json_query.h | 2 +- ydb/core/viewer/ut/ya.make | 4 +- ydb/core/viewer/viewer_ut.cpp | 142 ++++++++++ ydb/library/actors/testlib/test_runtime.cpp | 3 +- ydb/library/actors/testlib/test_runtime.h | 5 +- 16 files changed, 539 insertions(+), 73 deletions(-) diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h index 2ac8a67a905c..728e7531a3c5 100644 --- a/ydb/core/grpc_services/base/base.h +++ b/ydb/core/grpc_services/base/base.h @@ -51,6 +51,16 @@ using TYdbIssueMessageType = Ydb::Issue::IssueMessage; std::pair SplitPath(const TMaybe& database, const TString& path); std::pair SplitPath(const TString& path); +inline TActorId CreateGRpcRequestProxyId(int n = 0) { + if (n == 0) { + const auto actorId = TActorId(0, "GRpcReqProxy"); + return actorId; + } + + const auto actorId = TActorId(0, TStringBuilder() << "GRpcReqPro" << n); + return actorId; +} + struct TRpcServices { enum EServiceId { EvMakeDirectory = EventSpaceBegin(TKikimrEvents::ES_GRPC_CALLS), @@ -224,7 +234,9 @@ struct TRpcServices { EvAcquireYndxRateLimiterResource, EvGrpcRuntimeRequest, EvNodeCheckRequest, - EvStreamWriteRefreshToken // internal call, pair to EvRefreshToken + EvStreamWriteRefreshToken, // internal call, pair to EvRefreshToken + EvRequestAuthAndCheck, // performs authorization and runs GrpcRequestCheckActor + EvRequestAuthAndCheckResult, // !!! DO NOT ADD NEW REQUEST !!! }; @@ -365,7 +377,7 @@ class IRequestProxyCtx virtual ~IRequestProxyCtx() = default; // auth - virtual const TMaybe GetYdbToken() const = 0; + virtual const TMaybe GetYdbToken() const = 0; virtual void UpdateAuthState(NYdbGrpc::TAuthState::EAuthState state) = 0; virtual void SetInternalToken(const TIntrusiveConstPtr& token) = 0; virtual const NYdbGrpc::TAuthState& GetAuthState() const = 0; @@ -1553,5 +1565,237 @@ class TGRpcRequestValidationWrapper bool RlAllowed; }; +class TEvRequestAuthAndCheckResult : public TEventLocal { +public: + TEvRequestAuthAndCheckResult(Ydb::StatusIds::StatusCode status, const NYql::TIssues& issues) + : Status(status) + , Issues(issues) + {} + + TEvRequestAuthAndCheckResult(Ydb::StatusIds::StatusCode status, const NYql::TIssue& issue) + : Status(status) + { + Issues.AddIssue(issue); + } + + TEvRequestAuthAndCheckResult(Ydb::StatusIds::StatusCode status, const TString& error) + : Status(status) + { + Issues.AddIssue(error); + } + + TEvRequestAuthAndCheckResult(const TString& database, const TMaybe& ydbToken, const TIntrusiveConstPtr& userToken) + : Database(database) + , YdbToken(ydbToken) + , UserToken(userToken) + {} + + Ydb::StatusIds::StatusCode Status = Ydb::StatusIds::SUCCESS; + NYql::TIssues Issues; + TString Database; + TMaybe YdbToken; + TIntrusiveConstPtr UserToken; +}; + +class TEvRequestAuthAndCheck + : public IRequestProxyCtx + , public TEventLocal { +public: + TEvRequestAuthAndCheck(const TString& database, const TMaybe& ydbToken, NActors::TActorId sender) + : Database(database) + , YdbToken(ydbToken) + , Sender(sender) + , AuthState(true) + {} + + // IRequestProxyCtx + const TMaybe GetYdbToken() const override { + return YdbToken; + } + + void UpdateAuthState(NYdbGrpc::TAuthState::EAuthState state) override { + AuthState.State = state; + } + + void SetInternalToken(const TIntrusiveConstPtr& token) override { + UserToken = token; + } + + const NYdbGrpc::TAuthState& GetAuthState() const override { + return AuthState; + } + + void ReplyUnauthenticated(const TString& msg = "") override { + if (msg) { + IssueManager.RaiseIssue(NYql::TIssue{msg}); + } + ReplyWithYdbStatus(Ydb::StatusIds::UNAUTHORIZED); + } + + void ReplyWithYdbStatus(Ydb::StatusIds::StatusCode status) override { + const NActors::TActorContext& ctx = NActors::TActivationContext::AsActorContext(); + if (status == Ydb::StatusIds::SUCCESS) { + ctx.Send(Sender, + new TEvRequestAuthAndCheckResult( + Database, + YdbToken, + UserToken + ) + ); + } else { + ctx.Send(Sender, + new TEvRequestAuthAndCheckResult( + status, + IssueManager.GetIssues() + ) + ); + } + } + + void RaiseIssue(const NYql::TIssue& issue) override { + IssueManager.RaiseIssue(issue); + } + + void RaiseIssues(const NYql::TIssues& issues) override { + IssueManager.RaiseIssues(issues); + } + + TVector FindClientCertPropertyValues() const override { + return {}; + } + + void StartTracing(NWilson::TSpan&& span) override { + Span = std::move(span); + } + void FinishSpan() override { + Span.End(); + } + + bool* IsTracingDecided() override { + return nullptr; + } + + bool Validate(TString& /*error*/) override { + return true; + } + + void SetCounters(IGRpcProxyCounters::TPtr counters) override { + Counters = std::move(counters); + } + + IGRpcProxyCounters::TPtr GetCounters() const override { + return Counters; + } + + void UseDatabase(const TString& database) override { + Database = database; + } + + void SetRespHook(TRespHook&& /*hook*/) override { + } + + void SetRlPath(TMaybe&& path) override { + RlPath = std::move(path); + } + + TRateLimiterMode GetRlMode() const override { + return TRateLimiterMode::Rps; + } + + bool TryCustomAttributeProcess(const TSchemeBoardEvents::TDescribeSchemeResult& /*schemeData*/, ICheckerIface* /*iface*/) override { + return false; + } + + void Pass(const IFacilityProvider& /*facility*/) override { + ReplyWithYdbStatus(Ydb::StatusIds::SUCCESS); + } + + void SetAuditLogHook(TAuditLogHook&& /*hook*/) override { + } + + void SetDiskQuotaExceeded(bool /*disk*/) override { + } + + void AddAuditLogPart(const TStringBuf& name, const TString& value) override { + AuditLogParts.emplace_back(name, value); + } + + const TAuditLogParts& GetAuditLogParts() const override { + return AuditLogParts; + } + + TMaybe GetTraceId() const override { + return {}; + } + + NWilson::TTraceId GetWilsonTraceId() const override { + return Span.GetTraceId(); + } + + const TMaybe GetDatabaseName() const override { + return Database ? TMaybe(Database) : Nothing(); + } + + const TIntrusiveConstPtr& GetInternalToken() const override { + return UserToken; + } + + const TString& GetSerializedToken() const override { + if (UserToken) { + return UserToken->GetSerializedToken(); + } + + return EmptySerializedTokenMessage; + } + + bool IsClientLost() const override { + return false; + } + + const TMaybe GetPeerMetaValues(const TString&) const override { + return {}; + } + + TString GetPeerName() const override { + return {}; + } + + const TString& GetRequestName() const override { + static TString str = "request auth and check internal request"; + return str; + } + + TMaybe GetRlPath() const override { + return RlPath; + } + + TInstant GetDeadline() const override { + return deadline; + } + + + TMaybe GetSdkBuildInfo() const { + return {}; + } + + TMaybe GetGrpcUserAgent() const { + return {}; + } + + TString Database; + TMaybe YdbToken; + NActors::TActorId Sender; + NYdbGrpc::TAuthState AuthState; + NWilson::TSpan Span; + IGRpcProxyCounters::TPtr Counters; + TMaybe RlPath; + TAuditLogParts AuditLogParts; + NYql::TIssueManager IssueManager; + TIntrusiveConstPtr UserToken; + TInstant deadline = TInstant::Now() + TDuration::Seconds(10); + + inline static const TString EmptySerializedTokenMessage; +}; + } // namespace NGRpcService } // namespace NKikimr diff --git a/ydb/core/grpc_services/grpc_request_check_actor.h b/ydb/core/grpc_services/grpc_request_check_actor.h index f829add517dd..54fe15aca653 100644 --- a/ydb/core/grpc_services/grpc_request_check_actor.h +++ b/ydb/core/grpc_services/grpc_request_check_actor.h @@ -38,6 +38,20 @@ bool TGRpcRequestProxyHandleMethods::ValidateAndReplyOnError(TCtx* ctx) { } } +inline const TVector& GetEntriesForAuthAndCheckRequest(TEvRequestAuthAndCheck::TPtr& ev) { + if (ev->Get()->YdbToken && ev->Get()->YdbToken->StartsWith("Bearer")) { + if (AppData()->AuthConfig.GetUseAccessService() + && (AppData()->DomainsConfig.GetSecurityConfig().ViewerAllowedSIDsSize() > 0 || AppData()->DomainsConfig.GetSecurityConfig().MonitoringAllowedSIDsSize() > 0)) { + static TVector entries = { + {NKikimr::TEvTicketParser::TEvAuthorizeTicket::ToPermissions({"ydb.developerApi.get", "ydb.developerApi.update"}), {{"gizmo_id", "gizmo"}}} + }; + return entries; + } + } + static TVector emptyEntries = {}; + return emptyEntries; +} + template class TGrpcRequestCheckActor : public TGRpcRequestProxyHandleMethods @@ -73,7 +87,8 @@ class TGrpcRequestCheckActor } void ProcessCommonAttributes(const TSchemeBoardEvents::TDescribeSchemeResult& schemeData) { - static std::vector allowedAttributes = {"folder_id", "service_account_id", "database_id", "container_id"}; + TVector entries; + static std::vector allowedAttributes = {"folder_id", "service_account_id", "database_id"}; TVector> attributes; attributes.reserve(schemeData.GetPathDescription().UserAttributesSize()); for (const auto& attr : schemeData.GetPathDescription().GetUserAttributes()) { @@ -82,7 +97,16 @@ class TGrpcRequestCheckActor } } if (!attributes.empty()) { - SetEntries({{GetPermissions(), attributes}}); + entries.emplace_back(GetPermissions(), attributes); + } + + if constexpr (std::is_same_v) { + const auto& e = GetEntriesForAuthAndCheckRequest(Request_); + entries.insert(entries.end(), e.begin(), e.end()); + } + + if (!entries.empty()) { + SetEntries(entries); } } @@ -464,6 +488,12 @@ class TGrpcRequestCheckActor ReplyBackAndDie(); } + void HandleAndDie(TEvRequestAuthAndCheck::TPtr& ev) { + GrpcRequestBaseCtx_->FinishSpan(); + ev->Get()->ReplyWithYdbStatus(Ydb::StatusIds::SUCCESS); + PassAway(); + } + template void HandleAndDie(T& event) { GrpcRequestBaseCtx_->FinishSpan(); diff --git a/ydb/core/grpc_services/grpc_request_proxy.cpp b/ydb/core/grpc_services/grpc_request_proxy.cpp index 2838d2b66eed..5a9c0771480f 100644 --- a/ydb/core/grpc_services/grpc_request_proxy.cpp +++ b/ydb/core/grpc_services/grpc_request_proxy.cpp @@ -117,6 +117,11 @@ class TGRpcRequestProxyImpl NYql::TIssues()}); } + void Handle(TEvRequestAuthAndCheck::TPtr& ev, const TActorContext&) { + ev->Get()->FinishSpan(); + ev->Get()->ReplyWithYdbStatus(Ydb::StatusIds::SUCCESS); + } + // returns true and defer event if no updates for given database // otherwice returns false and leave event untouched template @@ -186,7 +191,7 @@ class TGRpcRequestProxyImpl if (maybeDatabaseName && !maybeDatabaseName.GetRef().empty()) { databaseName = CanonizePath(maybeDatabaseName.GetRef()); } else { - if (!AllowYdbRequestsWithoutDatabase && DynamicNode) { + if (!AllowYdbRequestsWithoutDatabase && DynamicNode && !std::is_same_v) { // TEvRequestAuthAndCheck is allowed to be processed without database requestBaseCtx->ReplyUnauthenticated("Requests without specified database are not allowed"); requestBaseCtx->FinishSpan(); return; @@ -590,6 +595,7 @@ void TGRpcRequestProxyImpl::StateFunc(TAutoPtr& ev) { HFunc(TEvCoordinationSessionRequest, PreHandle); HFunc(TEvNodeCheckRequest, PreHandle); HFunc(TEvProxyRuntimeEvent, PreHandle); + HFunc(TEvRequestAuthAndCheck, PreHandle); default: Y_ABORT("Unknown request: %u\n", ev->GetTypeRewrite()); diff --git a/ydb/core/grpc_services/grpc_request_proxy.h b/ydb/core/grpc_services/grpc_request_proxy.h index c665ce1d4ddb..462e55fc9363 100644 --- a/ydb/core/grpc_services/grpc_request_proxy.h +++ b/ydb/core/grpc_services/grpc_request_proxy.h @@ -57,15 +57,5 @@ class TGRpcRequestProxy : public TGRpcRequestProxyHandleMethods, public IFacilit TActorId DiscoveryCacheActorID; }; -inline TActorId CreateGRpcRequestProxyId(int n = 0) { - if (n == 0) { - const auto actorId = TActorId(0, "GRpcReqProxy"); - return actorId; - } - - const auto actorId = TActorId(0, TStringBuilder() << "GRpcReqPro" << n); - return actorId; -} - } // namespace NGRpcService } // namespace NKikimr diff --git a/ydb/core/mon/async_http_mon.cpp b/ydb/core/mon/async_http_mon.cpp index d2c571ae9f4a..7eabb9f8409b 100644 --- a/ydb/core/mon/async_http_mon.cpp +++ b/ydb/core/mon/async_http_mon.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -246,21 +247,41 @@ class THttpMonLegacyActorRequest : public TActorBootstrappedGet()->Request; NHttp::THeaders headers(request->Headers); TStringBuilder response; TStringBuilder body; - body << "

401 Unauthorized

"; - if (!error.empty()) { - body << "

" << error << "

"; + const TString httpError = YdbToHttpError(result.Status); + body << "

" << httpError << "

"; + if (result.Issues) { + body << "

" << result.Issues.ToString() << "

"; } body << ""; TString origin = TString(headers["Origin"]); if (origin.empty()) { origin = "*"; } - response << "HTTP/1.1 401 Unauthorized\r\n"; + response << "HTTP/1.1 " << httpError << "\r\n"; response << "Access-Control-Allow-Origin: " << origin << "\r\n"; response << "Access-Control-Allow-Credentials: true\r\n"; response << "Access-Control-Allow-Headers: Content-Type,Authorization,Origin,Accept\r\n"; @@ -291,17 +312,20 @@ class THttpMonLegacyActorRequest : public TActorBootstrappedGet()->Request; if (ActorMonPage->Authorizer) { - TString user = authorizeResult ? authorizeResult->Token->GetUserSID() : "anonymous"; + TString user = (result && result->UserToken) ? result->UserToken->GetUserSID() : "anonymous"; LOG_NOTICE_S(*TlsActivationContext, NActorsServices::HTTP, (request->Address ? request->Address->ToString() : "") << " " << user << " " << request->Method << " " << request->URL); } - TString serializedToken = authorizeResult ? authorizeResult->SerializedToken : ""; + TString serializedToken; + if (result && result->UserToken) { + serializedToken = result->UserToken->GetSerializedToken(); + } Send(ActorMonPage->TargetActorId, new NMon::TEvHttpInfo( Container, serializedToken), IEventHandle::FlagTrackDelivery); } @@ -325,14 +349,14 @@ class THttpMonLegacyActorRequest : public TActorBootstrappedGet()); - if (result.Error) { - return ReplyUnathorizedAndPassAway(result.Error.Message); + void Handle(NKikimr::NGRpcService::TEvRequestAuthAndCheckResult::TPtr& ev) { + const NKikimr::NGRpcService::TEvRequestAuthAndCheckResult& result(*ev->Get()); + if (result.Status != Ydb::StatusIds::SUCCESS) { + return ReplyErrorAndPassAway(result); } bool found = false; for (const TString& sid : ActorMonPage->AllowedSIDs) { - if (result.Token->IsExist(sid)) { + if (result.UserToken->IsExist(sid)) { found = true; break; } @@ -348,7 +372,7 @@ class THttpMonLegacyActorRequest : public TActorBootstrappedGetTypeRewrite()) { hFunc(TEvents::TEvUndelivered, HandleUndelivered); hFunc(NMon::IEvHttpInfoRes, HandleResponse); - hFunc(NKikimr::TEvTicketParser::TEvAuthorizeTicketResult, Handle); + hFunc(NKikimr::NGRpcService::TEvRequestAuthAndCheckResult, Handle); } } }; diff --git a/ydb/core/mon/mon.cpp b/ydb/core/mon/mon.cpp index 43215ebfc24c..4d7b8c797e2c 100644 --- a/ydb/core/mon/mon.cpp +++ b/ydb/core/mon/mon.cpp @@ -2,9 +2,15 @@ #include #include +#include #include +#include +#include + +#include + namespace NActors { using namespace NMonitoring; @@ -12,18 +18,35 @@ using namespace NKikimr; namespace { -const std::vector& GetEntries(const TString& ticket) { - if (ticket.StartsWith("Bearer")) { - if (AppData()->AuthConfig.GetUseAccessService() - && (AppData()->DomainsConfig.GetSecurityConfig().ViewerAllowedSIDsSize() > 0 || AppData()->DomainsConfig.GetSecurityConfig().MonitoringAllowedSIDsSize() > 0)) { - static std::vector entries = { - {NKikimr::TEvTicketParser::TEvAuthorizeTicket::ToPermissions({"ydb.developerApi.get", "ydb.developerApi.update"}), {{"gizmo_id", "gizmo"}}} - }; - return entries; +bool HasJsonContent(NMonitoring::IMonHttpRequest& request) { + const TStringBuf header = request.GetHeader("Content-Type"); + return header.empty() || AsciiEqualsIgnoreCase(header, "application/json"); // by default we will try to parse json, no error will be generated if parsing fails +} + +TString GetDatabase(NMonitoring::IMonHttpRequest& request) { + if (const auto dbIt = request.GetParams().Find("database"); dbIt != request.GetParams().end()) { + return dbIt->second; + } + if (request.GetMethod() == HTTP_METHOD_POST && HasJsonContent(request)) { + static NJson::TJsonReaderConfig JsonConfig; + NJson::TJsonValue requestData; + if (NJson::ReadJsonTree(request.GetPostContent(), &JsonConfig, &requestData)) { + return requestData["database"].GetString(); // empty if not string or no such key } } - static std::vector emptyEntries = {}; - return emptyEntries; + return {}; +} + +IEventHandle* GetRequestAuthAndCheckHandle(const NActors::TActorId& owner, const TString& database, const TString& ticket) { + return new NActors::IEventHandle( + NGRpcService::CreateGRpcRequestProxyId(), + owner, + new NKikimr::NGRpcService::TEvRequestAuthAndCheck( + database, + ticket ? TMaybe(ticket) : Nothing(), + owner), + IEventHandle::FlagTrackDelivery + ); } } // namespace @@ -32,9 +55,9 @@ NActors::IEventHandle* SelectAuthorizationScheme(const NActors::TActorId& owner, TStringBuf ydbSessionId = request.GetCookie("ydb_session_id"); TStringBuf authorization = request.GetHeader("Authorization"); if (!authorization.empty()) { - return GetAuthorizeTicketHandle(owner, TString(authorization)); + return GetRequestAuthAndCheckHandle(owner, GetDatabase(request), TString(authorization)); } else if (!ydbSessionId.empty()) { - return GetAuthorizeTicketHandle(owner, TString("Login ") + TString(ydbSessionId)); + return GetRequestAuthAndCheckHandle(owner, GetDatabase(request), TString("Login ") + TString(ydbSessionId)); } else { return nullptr; } @@ -45,35 +68,26 @@ NActors::IEventHandle* GetAuthorizeTicketResult(const NActors::TActorId& owner) return new NActors::IEventHandle( owner, owner, - new NKikimr::TEvTicketParser::TEvAuthorizeTicketResult(TString(), { - .Message = "No security credentials were provided", - .Retryable = false - }) + new NKikimr::NGRpcService::TEvRequestAuthAndCheckResult( + Ydb::StatusIds::UNAUTHORIZED, + "No security credentials were provided") ); } else if (!NKikimr::AppData()->DefaultUserSIDs.empty()) { TIntrusivePtr token = new NACLib::TUserToken(NKikimr::AppData()->DefaultUserSIDs); return new NActors::IEventHandle( owner, owner, - new NKikimr::TEvTicketParser::TEvAuthorizeTicketResult(TString(), token) + new NKikimr::NGRpcService::TEvRequestAuthAndCheckResult( + {}, + {}, + token + ) ); } else { return nullptr; } } -IEventHandle* GetAuthorizeTicketHandle(const NActors::TActorId& owner, const TString& ticket) { - return new NActors::IEventHandle( - NKikimr::MakeTicketParserID(), - owner, - new NKikimr::TEvTicketParser::TEvAuthorizeTicket({ - .Ticket = ticket, - .Entries = GetEntries(ticket), - }), - IEventHandle::FlagTrackDelivery - ); -} - IMonPage* TMon::RegisterActorPage(TIndexMonPage* index, const TString& relPath, const TString& title, bool preTag, TActorSystem* actorSystem, const TActorId& actorId, bool useAuth, bool sortPages) { return RegisterActorPage({ diff --git a/ydb/core/mon/mon.h b/ydb/core/mon/mon.h index f1ace0d40de6..69373811a836 100644 --- a/ydb/core/mon/mon.h +++ b/ydb/core/mon/mon.h @@ -13,7 +13,6 @@ namespace NActors { -IEventHandle* GetAuthorizeTicketHandle(const NActors::TActorId& owner, const TString& ticket); IEventHandle* SelectAuthorizationScheme(const NActors::TActorId& owner, NMonitoring::IMonHttpRequest& request); IEventHandle* GetAuthorizeTicketResult(const NActors::TActorId& owner); diff --git a/ydb/core/mon/ya.make b/ydb/core/mon/ya.make index acb89c15cf45..86f82b217bf8 100644 --- a/ydb/core/mon/ya.make +++ b/ydb/core/mon/ya.make @@ -12,12 +12,15 @@ SRCS( ) PEERDIR( - ydb/library/actors/core + library/cpp/json library/cpp/lwtrace/mon library/cpp/string_utils/url ydb/core/base + ydb/core/grpc_services/base ydb/core/protos ydb/library/aclib + ydb/library/actors/core + ydb/library/actors/http ) END() diff --git a/ydb/core/testlib/actors/test_runtime.cpp b/ydb/core/testlib/actors/test_runtime.cpp index 126f18930f4b..22201081f16f 100644 --- a/ydb/core/testlib/actors/test_runtime.cpp +++ b/ydb/core/testlib/actors/test_runtime.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -175,11 +176,19 @@ namespace NActors { if (NeedMonitoring && !SingleSysEnv) { ui16 port = MonitoringPortOffset ? MonitoringPortOffset + nodeIndex : GetPortManager().GetPort(); - node->Mon.Reset(new NActors::TSyncHttpMon({ - .Port = port, - .Threads = 10, - .Title = "KIKIMR monitoring" - })); + if (MonitoringTypeAsync) { + node->Mon.Reset(new NActors::TAsyncHttpMon({ + .Port = port, + .Threads = 10, + .Title = "KIKIMR monitoring" + })); + } else { + node->Mon.Reset(new NActors::TSyncHttpMon({ + .Port = port, + .Threads = 10, + .Title = "KIKIMR monitoring" + })); + } nodeAppData->Mon = node->Mon.Get(); node->Mon->RegisterCountersPage("counters", "Counters", node->DynamicCounters); auto actorsMonPage = node->Mon->RegisterIndexPage("actors", "Actors"); @@ -191,7 +200,7 @@ namespace NActors { node->ActorSystem->Start(); if (nodeAppData->Mon) { - nodeAppData->Mon->Start(); + nodeAppData->Mon->Start(node->ActorSystem.Get()); } } diff --git a/ydb/core/testlib/test_client.cpp b/ydb/core/testlib/test_client.cpp index c81644fe18f7..38fdee3f6dc2 100644 --- a/ydb/core/testlib/test_client.cpp +++ b/ydb/core/testlib/test_client.cpp @@ -225,7 +225,7 @@ namespace Tests { NKikimr::SetupChannelProfiles(app); - Runtime->SetupMonitoring(Settings->MonitoringPortOffset); + Runtime->SetupMonitoring(Settings->MonitoringPortOffset, Settings->MonitoringTypeAsync); Runtime->SetLogBackend(Settings->LogBackend); Runtime->AddAppDataInit([this](ui32 nodeIdx, NKikimr::TAppData& appData) { diff --git a/ydb/core/testlib/test_client.h b/ydb/core/testlib/test_client.h index b4d309bcd410..795491279c1d 100644 --- a/ydb/core/testlib/test_client.h +++ b/ydb/core/testlib/test_client.h @@ -108,6 +108,7 @@ namespace Tests { ui16 GrpcPort = 0; int GrpcMaxMessageSize = 0; // 0 - default (4_MB), -1 - no limit ui16 MonitoringPortOffset = 0; + bool MonitoringTypeAsync = false; NKikimrProto::TAuthConfig AuthConfig; NKikimrPQ::TPQConfig PQConfig; NKikimrPQ::TPQClusterDiscoveryConfig PQClusterDiscoveryConfig; @@ -162,7 +163,7 @@ namespace Tests { TServerSettings& SetGrpcPort(ui16 value) { GrpcPort = value; return *this; } TServerSettings& SetGrpcMaxMessageSize(int value) { GrpcMaxMessageSize = value; return *this; } - TServerSettings& SetMonitoringPortOffset(ui16 value) { MonitoringPortOffset = value; return *this; } + TServerSettings& SetMonitoringPortOffset(ui16 value, bool monitoringTypeAsync = false) { MonitoringPortOffset = value; MonitoringTypeAsync = monitoringTypeAsync; return *this; } TServerSettings& SetSupportsRedirect(bool value) { SupportsRedirect = value; return *this; } TServerSettings& SetTracePath(const TString& value) { TracePath = value; return *this; } TServerSettings& SetDomain(ui32 value) { Domain = value; return *this; } diff --git a/ydb/core/viewer/json_query.h b/ydb/core/viewer/json_query.h index b0f47f135ffb..fa039aac3d8a 100644 --- a/ydb/core/viewer/json_query.h +++ b/ydb/core/viewer/json_query.h @@ -122,7 +122,7 @@ class TJsonQuery : public TViewerPipeClient { if (IsPostContent()) { TStringBuf content = Event->Get()->Request.GetPostContent(); if (!ParsePostContent(content)) { - return ReplyAndPassAway(Viewer->GetHTTPBADREQUEST(Event->Get(), "text/plain", "Bad content recieved")); + return ReplyAndPassAway(Viewer->GetHTTPBADREQUEST(Event->Get(), "text/plain", "Bad content received")); } } if (Query.empty() && Action != "cancel-query") { diff --git a/ydb/core/viewer/ut/ya.make b/ydb/core/viewer/ut/ya.make index 305ff23d21c6..62eaba23bed4 100644 --- a/ydb/core/viewer/ut/ya.make +++ b/ydb/core/viewer/ut/ya.make @@ -2,7 +2,7 @@ UNITTEST_FOR(ydb/core/viewer) FORK_SUBTESTS() -TIMEOUT(600) +TIMEOUT(30) SIZE(MEDIUM) @@ -13,6 +13,8 @@ SRCS( ) PEERDIR( + library/cpp/http/misc + library/cpp/http/simple ydb/core/testlib/default ) diff --git a/ydb/core/viewer/viewer_ut.cpp b/ydb/core/viewer/viewer_ut.cpp index ce737a3511b7..bbc2e5cf1938 100644 --- a/ydb/core/viewer/viewer_ut.cpp +++ b/ydb/core/viewer/viewer_ut.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include #include #include @@ -23,6 +25,8 @@ #include #include +#include + using namespace NKikimr; using namespace NViewer; using namespace NKikimrWhiteboard; @@ -1503,4 +1507,142 @@ Y_UNIT_TEST_SUITE(Viewer) { Y_UNIT_TEST(JsonStorageListingV2PDiskIdFilter) { JsonStorage9Nodes9GroupsListingTest("v2", false, true, true, 4, 8); } + + struct TFakeTicketParserActor : public TActor { + TFakeTicketParserActor() + : TActor(&TFakeTicketParserActor::StFunc) + {} + + STFUNC(StFunc) { + switch (ev->GetTypeRewrite()) { + hFunc(TEvTicketParser::TEvAuthorizeTicket, Handle); + default: + break; + } + } + + void Handle(TEvTicketParser::TEvAuthorizeTicket::TPtr& ev) { + LOG_INFO_S(*TlsActivationContext, NKikimrServices::TICKET_PARSER, "Ticket parser: got TEvAuthorizeTicket event: " << ev->Get()->Ticket << " " << ev->Get()->Database << " " << ev->Get()->Entries.size()); + ++AuthorizeTicketRequests; + + if (ev->Get()->Database != "/Root") { + Fail(ev, TStringBuilder() << "Incorrect database " << ev->Get()->Database); + return; + } + + if (ev->Get()->Ticket != "test_ydb_token") { + Fail(ev, TStringBuilder() << "Incorrect token " << ev->Get()->Ticket); + return; + } + + bool databaseIdFound = false; + bool folderIdFound = false; + for (const TEvTicketParser::TEvAuthorizeTicket::TEntry& entry : ev->Get()->Entries) { + for (const std::pair& attr : entry.Attributes) { + if (attr.first == "database_id") { + databaseIdFound = true; + if (attr.second != "test_database_id") { + Fail(ev, TStringBuilder() << "Incorrect database_id " << attr.second); + return; + } + } else if (attr.first == "folder_id") { + folderIdFound = true; + if (attr.second != "test_folder_id") { + Fail(ev, TStringBuilder() << "Incorrect folder_id " << attr.second); + return; + } + } + } + } + if (!databaseIdFound) { + Fail(ev, "database_id not found"); + return; + } + if (!folderIdFound) { + Fail(ev, "folder_id not found"); + return; + } + + Success(ev); + } + + void Fail(TEvTicketParser::TEvAuthorizeTicket::TPtr& ev, const TString& message) { + ++AuthorizeTicketFails; + TEvTicketParser::TError err; + err.Retryable = false; + err.Message = message ? message : "Test error"; + LOG_INFO_S(*TlsActivationContext, NKikimrServices::TICKET_PARSER, "Send TEvAuthorizeTicketResult: " << err.Message); + Send(ev->Sender, new TEvTicketParser::TEvAuthorizeTicketResult(ev->Get()->Ticket, err)); + } + + void Success(TEvTicketParser::TEvAuthorizeTicket::TPtr& ev) { + ++AuthorizeTicketSuccesses; + NACLib::TUserToken::TUserTokenInitFields args; + args.UserSID = "user_name"; + args.GroupSIDs.push_back("group_name"); + TIntrusivePtr userToken = MakeIntrusive(args); + LOG_INFO_S(*TlsActivationContext, NKikimrServices::TICKET_PARSER, "Send TEvAuthorizeTicketResult success"); + Send(ev->Sender, new TEvTicketParser::TEvAuthorizeTicketResult(ev->Get()->Ticket, userToken)); + } + + size_t AuthorizeTicketRequests = 0; + size_t AuthorizeTicketSuccesses = 0; + size_t AuthorizeTicketFails = 0; + }; + + Y_UNIT_TEST(AuthorizeYdbTokenWithDatabaseAttributes) { + TPortManager tp; + ui16 port = tp.GetPort(2134); + ui16 grpcPort = tp.GetPort(2135); + ui16 monPort = tp.GetPort(8765); + auto settings = TServerSettings(port); + settings.InitKikimrRunConfig() + .SetNodeCount(1) + .SetUseRealThreads(true) + .SetDomainName("Root") + .SetMonitoringPortOffset(monPort, true); // authorization is implemented only in async mon + + auto& securityConfig = *settings.AppConfig->MutableDomainsConfig()->MutableSecurityConfig(); + securityConfig.SetEnforceUserTokenCheckRequirement(true); + + TFakeTicketParserActor* ticketParser = nullptr; + settings.CreateTicketParser = [&](const TTicketParserSettings&) -> IActor* { + ticketParser = new TFakeTicketParserActor(); + return ticketParser; + }; + + TServer server(settings); + server.EnableGRpc(grpcPort); + TClient client(settings); + + const auto alterAttrsStatus = client.AlterUserAttributes("/", "Root", { + { "folder_id", "test_folder_id" }, + { "database_id", "test_database_id" }, + }); + UNIT_ASSERT_EQUAL(alterAttrsStatus, NMsgBusProxy::MSTATUS_OK); + + TTestActorRuntime& runtime = *server.GetRuntime(); + runtime.SetLogPriority(NKikimrServices::GRPC_SERVER, NLog::PRI_TRACE); + runtime.SetLogPriority(NKikimrServices::TICKET_PARSER, NLog::PRI_TRACE); + + TKeepAliveHttpClient httpClient("localhost", monPort); + TStringStream responseStream; + TKeepAliveHttpClient::THeaders headers; + headers["Content-Type"] = "application/json"; + headers["Authorization"] = "test_ydb_token"; + TString requestBody = R"json({ + "query": "SELECT 42;", + "database": "/Root", + "action": "execute-script", + "syntax": "yql_v1", + "stats": "profile" + })json"; + const TKeepAliveHttpClient::THttpCode statusCode = httpClient.DoPost("/viewer/query?timeout=600000&base64=false&schema=modern", requestBody, &responseStream, headers); + const TString response = responseStream.ReadAll(); + UNIT_ASSERT_EQUAL_C(statusCode, HTTP_OK, statusCode << ": " << response); + + UNIT_ASSERT(ticketParser); + UNIT_ASSERT_VALUES_EQUAL_C(ticketParser->AuthorizeTicketRequests, 1, response); + UNIT_ASSERT_VALUES_EQUAL_C(ticketParser->AuthorizeTicketSuccesses, 1, response); + } } diff --git a/ydb/library/actors/testlib/test_runtime.cpp b/ydb/library/actors/testlib/test_runtime.cpp index c93d8ea27ff6..a5ab8d9fcd1e 100644 --- a/ydb/library/actors/testlib/test_runtime.cpp +++ b/ydb/library/actors/testlib/test_runtime.cpp @@ -1594,9 +1594,10 @@ namespace NActors { return node->DynamicCounters; } - void TTestActorRuntimeBase::SetupMonitoring(ui16 monitoringPortOffset) { + void TTestActorRuntimeBase::SetupMonitoring(ui16 monitoringPortOffset, bool monitoringTypeAsync) { NeedMonitoring = true; MonitoringPortOffset = monitoringPortOffset; + MonitoringTypeAsync = monitoringTypeAsync; } void TTestActorRuntimeBase::SendInternal(TAutoPtr ev, ui32 nodeIndex, bool viaActorSystem) { diff --git a/ydb/library/actors/testlib/test_runtime.h b/ydb/library/actors/testlib/test_runtime.h index eef097c1b7b9..ace6a24891a9 100644 --- a/ydb/library/actors/testlib/test_runtime.h +++ b/ydb/library/actors/testlib/test_runtime.h @@ -295,7 +295,7 @@ namespace NActors { void EnableScheduleForActor(const TActorId& actorId, bool allow = true); bool IsScheduleForActorEnabled(const TActorId& actorId) const; TIntrusivePtr GetDynamicCounters(ui32 nodeIndex = 0); - void SetupMonitoring(ui16 monitoringPortOffset = 0); + void SetupMonitoring(ui16 monitoringPortOffset = 0, bool monitoringTypeAsync = false); using TEventObserverCollection = std::list& event)>>; class TEventObserverHolder { @@ -321,7 +321,7 @@ namespace NActors { if (this != &other) { Remove(); - + List = std::move(other.List); Iter = std::move(other.Iter); @@ -656,6 +656,7 @@ namespace NActors { TAutoPtr LogBackend; bool NeedMonitoring; ui16 MonitoringPortOffset = 0; + bool MonitoringTypeAsync = false; TIntrusivePtr RandomProvider; TIntrusivePtr TimeProvider;