diff --git a/ydb/core/client/server/grpc_server.cpp b/ydb/core/client/server/grpc_server.cpp index 95079d6bc7dc..e510fdfbe3ee 100644 --- a/ydb/core/client/server/grpc_server.cpp +++ b/ydb/core/client/server/grpc_server.cpp @@ -11,8 +11,6 @@ #include -#include - #include #include #include @@ -266,15 +264,8 @@ class TSimpleRequest } void Finish(const TOut& resp, ui32 status) { - auto makeResponseString = [&] { - TString x; - google::protobuf::TextFormat::Printer printer; - printer.SetSingleLineMode(true); - printer.PrintToString(resp, &x); - return x; - }; LOG_DEBUG(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] issuing response Name# %s data# %s peer# %s", this, - Name, makeResponseString().data(), GetPeerName().c_str()); + Name, NYdbGrpc::FormatMessage(resp).data(), GetPeerName().c_str()); ResponseSize = resp.ByteSize(); ResponseStatus = status; StateFunc = &TSimpleRequest::FinishDone; @@ -300,19 +291,8 @@ class TSimpleRequest bool RequestDone(bool ok) { OnAfterCall(); - auto makeRequestString = [&] { - TString resp; - if (ok) { - google::protobuf::TextFormat::Printer printer; - printer.SetSingleLineMode(true); - printer.PrintToString(Request, &resp); - } else { - resp = ""; - } - return resp; - }; LOG_DEBUG(ActorSystem, NKikimrServices::GRPC_SERVER, "[%p] received request Name# %s ok# %s data# %s peer# %s current inflight# %li", this, - Name, ok ? "true" : "false", makeRequestString().data(), GetPeerName().c_str(), Server->GetCurrentInFlight()); + Name, ok ? "true" : "false", NYdbGrpc::FormatMessage(Request, ok).data(), GetPeerName().c_str(), Server->GetCurrentInFlight()); if (Context.c_call() == nullptr) { Y_ABORT_UNLESS(!ok); diff --git a/ydb/core/grpc_streaming/grpc_streaming.h b/ydb/core/grpc_streaming/grpc_streaming.h index 872ab75bda76..6beb16ce0ca5 100644 --- a/ydb/core/grpc_streaming/grpc_streaming.h +++ b/ydb/core/grpc_streaming/grpc_streaming.h @@ -11,7 +11,6 @@ #include #include -#include #include @@ -347,22 +346,10 @@ class TGRpcStreamingRequest final } void OnReadDone(NYdbGrpc::EQueueEventStatus status) { - auto dumpResultText = [&] { - TString text; - if (status == NYdbGrpc::EQueueEventStatus::OK) { - google::protobuf::TextFormat::Printer printer; - printer.SetSingleLineMode(true); - printer.PrintToString(ReadInProgress->Record, &text); - } else { - text = ""; - } - return text; - }; - LOG_DEBUG(ActorSystem, LoggerServiceId, "[%p] read finished Name# %s ok# %s data# %s peer# %s", this, Name, status == NYdbGrpc::EQueueEventStatus::OK ? "true" : "false", - dumpResultText().c_str(), + NYdbGrpc::FormatMessage(ReadInProgress->Record, status == NYdbGrpc::EQueueEventStatus::OK).c_str(), this->GetPeerName().c_str()); // Take current in-progress read first @@ -400,25 +387,17 @@ class TGRpcStreamingRequest final } bool Write(TOut&& message, const grpc::WriteOptions& options = { }, const grpc::Status* status = nullptr) { - auto dumpMessageText = [&] { - TString text; - google::protobuf::TextFormat::Printer printer; - printer.SetSingleLineMode(true); - printer.PrintToString(message, &text); - return text; - }; - if (status) { LOG_DEBUG(ActorSystem, LoggerServiceId, "[%p] facade write Name# %s data# %s peer# %s grpc status# (%d) message# %s", this, Name, - dumpMessageText().c_str(), + NYdbGrpc::FormatMessage(message).c_str(), this->GetPeerName().c_str(), static_cast(status->error_code()), status->error_message().c_str()); } else { LOG_DEBUG(ActorSystem, LoggerServiceId, "[%p] facade write Name# %s data# %s peer# %s", this, Name, - dumpMessageText().c_str(), + NYdbGrpc::FormatMessage(message).c_str(), this->GetPeerName().c_str()); } diff --git a/ydb/core/persqueue/partition_monitoring.cpp b/ydb/core/persqueue/partition_monitoring.cpp index 4343adcbb6a1..89c7f47a869c 100644 --- a/ydb/core/persqueue/partition_monitoring.cpp +++ b/ydb/core/persqueue/partition_monitoring.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -22,6 +23,14 @@ namespace NKikimr::NPQ { +TString PrintConfig(const NKikimrPQ::TPQTabletConfig& cfg) { + TSecurityTextFormatPrinter printer; + printer.SetSingleLineMode(true); + TString string; + printer.PrintToString(cfg, &string); + return string; +} + void HtmlOutput(IOutputStream& out, const TString& line, const std::deque>& keys) { HTML(out) { TABLE() { @@ -109,7 +118,7 @@ void TPartition::HandleMonitoring(TEvPQ::TEvMonRequest::TPtr& ev, const TActorCo out << "AvgWriteSize per " << avg.GetDuration().ToString() << " is " << avg.GetValue() << " bytes"; res.push_back(out.Str()); out.Clear(); } - out << Config.DebugString(); res.push_back(out.Str()); out.Clear(); + out << PrintConfig(Config); res.push_back(out.Str()); out.Clear(); HTML(out) { DIV_CLASS_ID("tab-pane fade", Sprintf("partition_%u", ui32(Partition))) { TABLE_SORTABLE_CLASS("table") { diff --git a/ydb/core/persqueue/ya.make b/ydb/core/persqueue/ya.make index 8000bbe68ae2..919301a5d3d0 100644 --- a/ydb/core/persqueue/ya.make +++ b/ydb/core/persqueue/ya.make @@ -59,6 +59,7 @@ PEERDIR( ydb/library/logger ydb/library/persqueue/counter_time_keeper ydb/library/persqueue/topic_parser + ydb/library/protobuf_printer ydb/public/lib/base ydb/public/sdk/cpp/client/ydb_persqueue_core ) diff --git a/ydb/core/protos/flat_tx_scheme.proto b/ydb/core/protos/flat_tx_scheme.proto index 694ee37abd00..e249ec1c80cf 100644 --- a/ydb/core/protos/flat_tx_scheme.proto +++ b/ydb/core/protos/flat_tx_scheme.proto @@ -7,6 +7,7 @@ import "ydb/core/protos/bind_channel_storage_pool.proto"; import "ydb/core/protos/flat_scheme_op.proto"; import "ydb/public/api/protos/ydb_cms.proto"; import "ydb/public/api/protos/ydb_issue_message.proto"; +import "ydb/public/api/protos/annotations/sensitive.proto"; package NKikimrScheme; option java_package = "ru.yandex.kikimr.proto"; @@ -53,7 +54,7 @@ message TEvModifySchemeTransaction { optional uint64 TabletId = 3; optional string Owner = 5; optional bool FailOnExist = 6; // depricated, TModifyScheme.FailOnExist is recomended - optional string UserToken = 7; // serialized NACLib::TUserToken + optional string UserToken = 7 [(Ydb.sensitive) = true]; // serialized NACLib::TUserToken optional string PeerName = 8; } diff --git a/ydb/core/protos/pqconfig.proto b/ydb/core/protos/pqconfig.proto index bb70645ab640..90f6a66969be 100644 --- a/ydb/core/protos/pqconfig.proto +++ b/ydb/core/protos/pqconfig.proto @@ -1,6 +1,8 @@ import "ydb/public/api/protos/draft/persqueue_error_codes.proto"; import "ydb/public/api/protos/draft/persqueue_common.proto"; +import "ydb/public/api/protos/annotations/sensitive.proto"; + import "ydb/core/protos/base.proto"; import "ydb/core/protos/msgbus_kv.proto"; import "ydb/core/protos/node_limits.proto"; @@ -208,11 +210,11 @@ message TMirrorPartitionConfig { message TCredentials { message IamCredentials { optional string Endpoint = 1; - optional string ServiceAccountKey = 2; + optional string ServiceAccountKey = 2 [(Ydb.sensitive) = true]; } oneof Credentials { - string OauthToken = 1; - string JwtParams = 2; + string OauthToken = 1 [(Ydb.sensitive) = true]; + string JwtParams = 2 [(Ydb.sensitive) = true]; IamCredentials Iam = 3; } } diff --git a/ydb/core/tx/schemeshard/schemeshard__operation.cpp b/ydb/core/tx/schemeshard/schemeshard__operation.cpp index 676771621098..29da42f0791f 100644 --- a/ydb/core/tx/schemeshard/schemeshard__operation.cpp +++ b/ydb/core/tx/schemeshard/schemeshard__operation.cpp @@ -12,6 +12,8 @@ #include #include +#include + #include namespace NKikimr::NSchemeShard { @@ -85,6 +87,14 @@ NKikimrScheme::TEvModifySchemeTransaction GetRecordForPrint(const NKikimrScheme: return recordForPrint; } +TString PrintSecurely(const NKikimrScheme::TEvModifySchemeTransaction& record) { + TSecurityTextFormatPrinter printer; + printer.SetSingleLineMode(true); + TString string; + printer.PrintToString(record, &string); + return string; +} + THolder TSchemeShard::IgniteOperation(TProposeRequest& request, TOperationContext& context) { THolder response = nullptr; @@ -183,7 +193,7 @@ THolder TSchemeShard::IgniteOperation(TProposeRequest& request << ", already accepted parts: " << operation->Parts.size() << ", propose result status: " << NKikimrScheme::EStatus_Name(response->Record.GetStatus()) << ", with reason: " << response->Record.GetReason() - << ", tx message: " << GetRecordForPrint(record).ShortDebugString()); + << ", tx message: " << PrintSecurely(record)); } Y_VERIFY_S(context.IsUndoChangesSafe(), @@ -194,7 +204,7 @@ THolder TSchemeShard::IgniteOperation(TProposeRequest& request << ", already accepted parts: " << operation->Parts.size() << ", propose result status: " << NKikimrScheme::EStatus_Name(response->Record.GetStatus()) << ", with reason: " << response->Record.GetReason() - << ", tx message: " << GetRecordForPrint(record).ShortDebugString()); + << ", tx message: " << PrintSecurely(record)); context.OnComplete = {}; // recreate context.DbChanges = {}; @@ -237,7 +247,7 @@ struct TSchemeShard::TTxOperationPropose: public NTabletFlatExecutor::TTransacti LOG_DEBUG_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "TTxOperationPropose Execute" - << ", message: " << GetRecordForPrint(Request->Get()->Record).ShortDebugString() + << ", message: " << PrintSecurely(Request->Get()->Record) << ", at schemeshard: " << selfId); txc.DB.NoMoreReadsForTx(); diff --git a/ydb/core/tx/schemeshard/ya.make b/ydb/core/tx/schemeshard/ya.make index 8cbf4b637d3c..b3b9e032a154 100644 --- a/ydb/core/tx/schemeshard/ya.make +++ b/ydb/core/tx/schemeshard/ya.make @@ -268,6 +268,7 @@ PEERDIR( ydb/library/aclib/protos ydb/library/login ydb/library/login/protos + ydb/library/protobuf_printer ydb/library/yql/minikql ydb/services/bg_tasks ) diff --git a/ydb/library/grpc/server/actors/ya.make b/ydb/library/grpc/server/actors/ya.make index 94a143ac3414..fa71ec9a30c5 100644 --- a/ydb/library/grpc/server/actors/ya.make +++ b/ydb/library/grpc/server/actors/ya.make @@ -6,6 +6,7 @@ SRCS( PEERDIR( ydb/library/actors/core + ydb/library/grpc/server ) END() diff --git a/ydb/library/grpc/server/logger.h b/ydb/library/grpc/server/logger.h index 1460afa09633..8cfae2b240a3 100644 --- a/ydb/library/grpc/server/logger.h +++ b/ydb/library/grpc/server/logger.h @@ -1,11 +1,17 @@ #pragma once +#include + #include #include +#include + namespace NYdbGrpc { +static bool LogBodyEnabled = "BODY" == GetEnv("YDB_GRPC_SERVER_LOGGING"); + class TLogger: public TThrRefBase { protected: TLogger() = default; @@ -40,4 +46,21 @@ using TLoggerPtr = TIntrusivePtr; logger->Write(ELogPriority::TLOG_INFO, format, __VA_ARGS__); \ } else { } +template +inline TString FormatMessage(const TMsg& message, bool ok = true) { + if (ok) { + if (LogBodyEnabled) { + TString text; + NKikimr::TSecurityTextFormatPrinter printer; + printer.SetSingleLineMode(true); + printer.PrintToString(message, &text); + return text; + } else { + return ""; + } + } else { + return ""; + } +} + } // namespace NYdbGrpc diff --git a/ydb/library/grpc/server/ya.make b/ydb/library/grpc/server/ya.make index 9ed8f30ef2e6..2e44c832ba82 100644 --- a/ydb/library/grpc/server/ya.make +++ b/ydb/library/grpc/server/ya.make @@ -10,6 +10,7 @@ SRCS( GENERATE_ENUM_SERIALIZATION(grpc_request_base.h) PEERDIR( + ydb/library/protobuf_printer contrib/libs/grpc library/cpp/monlib/dynamic_counters/percentile ) @@ -17,4 +18,3 @@ PEERDIR( END() RECURSE_FOR_TESTS(ut) - diff --git a/ydb/public/api/protos/draft/persqueue_common.proto b/ydb/public/api/protos/draft/persqueue_common.proto index 87bb3f236d9f..bad67b8e551b 100644 --- a/ydb/public/api/protos/draft/persqueue_common.proto +++ b/ydb/public/api/protos/draft/persqueue_common.proto @@ -2,6 +2,8 @@ syntax = "proto3"; import "google/protobuf/descriptor.proto"; import "ydb/public/api/protos/draft/persqueue_error_codes.proto"; +import "ydb/public/api/protos/annotations/sensitive.proto"; + package NPersQueueCommon; option java_package = "com.yandex.ydb.persqueue"; @@ -35,8 +37,7 @@ enum ECodec { message Credentials { oneof credentials { - bytes tvm_service_ticket = 1; - bytes oauth_token = 2; + bytes tvm_service_ticket = 1 [(Ydb.sensitive) = true]; + bytes oauth_token = 2 [(Ydb.sensitive) = true]; } } - diff --git a/ydb/public/api/protos/ydb_persqueue_v1.proto b/ydb/public/api/protos/ydb_persqueue_v1.proto index 5262d50f3079..e4eecdbdee2f 100644 --- a/ydb/public/api/protos/ydb_persqueue_v1.proto +++ b/ydb/public/api/protos/ydb_persqueue_v1.proto @@ -3,6 +3,7 @@ import "ydb/public/api/protos/ydb_operation.proto"; import "ydb/public/api/protos/ydb_scheme.proto"; import "ydb/public/api/protos/ydb_status_codes.proto"; import "ydb/public/api/protos/ydb_issue_message.proto"; +import "ydb/public/api/protos/annotations/sensitive.proto"; import "ydb/public/api/protos/annotations/validation.proto"; package Ydb.PersQueue.V1; @@ -38,7 +39,7 @@ message OffsetsRange { // In-session reauthentication and reauthorization, lets user increase session lifetime. You should wait for 'update_token_response' before sending next 'update_token_request'. message UpdateTokenRequest { - string token = 1; + string token = 1 [(Ydb.sensitive) = true]; } message UpdateTokenResponse { @@ -788,7 +789,7 @@ message MigrationStreamingReadClientMessage { } // User credentials if update is needed or empty string. - bytes token = 20; + bytes token = 20 [(Ydb.sensitive) = true]; } /** @@ -1073,8 +1074,8 @@ message Credentials { string service_account_key = 2; } oneof credentials { - string oauth_token = 1; - string jwt_params = 2; + string oauth_token = 1 [(Ydb.sensitive) = true]; + string jwt_params = 2 [(Ydb.sensitive) = true]; Iam iam = 3; } } diff --git a/ydb/public/api/protos/ydb_topic.proto b/ydb/public/api/protos/ydb_topic.proto index 8b5c0aac9cf4..9bcf13e6972b 100644 --- a/ydb/public/api/protos/ydb_topic.proto +++ b/ydb/public/api/protos/ydb_topic.proto @@ -3,6 +3,7 @@ import "ydb/public/api/protos/ydb_operation.proto"; import "ydb/public/api/protos/ydb_scheme.proto"; import "ydb/public/api/protos/ydb_status_codes.proto"; import "ydb/public/api/protos/ydb_issue_message.proto"; +import "ydb/public/api/protos/annotations/sensitive.proto"; import "ydb/public/api/protos/annotations/validation.proto"; import "google/protobuf/duration.proto"; @@ -43,7 +44,7 @@ message OffsetsRange { // In-session reauthentication and reauthorization, lets user increase session lifetime. // Client should wait for UpdateTokenResponse before sending next UpdateTokenRequest. message UpdateTokenRequest { - string token = 1; + string token = 1 [(Ydb.sensitive) = true]; } message UpdateTokenResponse { @@ -167,7 +168,7 @@ message StreamWriteMessage { // Explicit partition id to write to. int64 partition_id = 6; // Explicit partition location to write to. - PartitionWithGeneration partition_with_generation = 8; + PartitionWithGeneration partition_with_generation = 8; } // Message metadata. Overall size is limited to 4096 symbols (all keys and values combined). repeated MetadataItem metadata_items = 7 [(Ydb.size).le = 1000]; @@ -534,7 +535,7 @@ message StreamReadMessage { // Flag of graceful stop, used only when InitRequest.direct_read is true // Client must pass this value unchanged from the StopPartitionSessionRequest. - // Server can sent two StopPartitionSessionRequests, the first with graceful=true, the second with graceful=false. The client must answer both of them. + // Server can sent two StopPartitionSessionRequests, the first with graceful=true, the second with graceful=false. The client must answer both of them. bool graceful = 2; } @@ -563,22 +564,22 @@ message StreamReadMessage { // Messages for bidirectional streaming rpc StreamDirectRead message StreamDirectReadMessage { - // Client-server message for direct read session. + // Client-server message for direct read session. // InitDirectRead - command from client to create and start a direct read session. // StartDirectReadPartitionSession - command from client to create and start a direct read partition session. - // UpdateTokenRequest - request to update auth token + // UpdateTokenRequest - request to update auth token message FromClient { oneof client_message { InitDirectRead init_direct_read = 1; StartDirectReadPartitionSession start_direct_read_partition_session = 2; - UpdateTokenRequest update_token_request = 3; + UpdateTokenRequest update_token_request = 3; } } - // Server-client message for direct read session. + // Server-client message for direct read session. // DirectReadResponse - portion of message data. // StopDirectReadPartitionSession - command from server to stop a direct read partition session. - // UpdateTokenResponse - acknowledgment of token update. + // UpdateTokenResponse - acknowledgment of token update. message FromServer { // Server status of response. Ydb.StatusIds.StatusCode status = 1; @@ -642,13 +643,13 @@ message StreamDirectReadMessage { message DirectReadResponse { // Partition session identifier. int64 partition_session_id = 1; - + // Read request identifier. int64 direct_read_id = 2; // Messages data StreamReadMessage.ReadResponse.PartitionData partition_data = 3; - } + } } message TransactionIdentity { @@ -843,7 +844,7 @@ message CreateTopicRequest { // How long data in partition should be stored. Must be greater than 0 and less than limit for this database. // Default limit - 36 hours. google.protobuf.Duration retention_period = 4; - + // How much data in partition should be stored. Must be greater than 0 and less than limit for this database. // Zero value means infinite limit. int64 retention_storage_mb = 5 [(Ydb.value) = ">= 0"]; @@ -923,7 +924,7 @@ message DescribeTopicResult { // Settings for partitioning PartitioningSettings partitioning_settings = 2; - + // Partitions description. repeated PartitionInfo partitions = 3; @@ -932,7 +933,7 @@ message DescribeTopicResult { // // How long data in partition should be stored. google.protobuf.Duration retention_period = 4; - + // How much data in partition should be stored. // Zero value means infinite limit. int64 retention_storage_mb = 5; @@ -943,7 +944,7 @@ message DescribeTopicResult { // Writes with codec not from this list are forbidden. // If empty, codec compatibility check for the topic is disabled. SupportedCodecs supported_codecs = 7; - + // Partition write speed in bytes per second. // Zero value means default limit: 1 MB per second. int64 partition_write_speed_bytes_per_second = 8; @@ -1135,7 +1136,7 @@ message AlterTopicRequest { // How long data in partition should be stored. Must be greater than 0 and less than limit for this database. // Default limit - 36 hours. google.protobuf.Duration set_retention_period = 4; - + // How much data in partition should be stored. Must be greater than 0 and less than limit for this database. optional int64 set_retention_storage_mb = 5 [(Ydb.value) = ">= 0"]; @@ -1148,7 +1149,7 @@ message AlterTopicRequest { // Partition write speed in bytes per second. Must be less than database limit. Default limit - 1 MB/s. optional int64 set_partition_write_speed_bytes_per_second = 8 [(Ydb.value) = ">= 0"]; - + // Burst size for write in partition, in bytes. Must be less than database limit. Default limit - 1 MB. optional int64 set_partition_write_burst_bytes = 9 [(Ydb.value) = ">= 0"];