From 6a3b58bcf3ddc6ec9cc61f08b462ce6387e45125 Mon Sep 17 00:00:00 2001 From: Bulat Gayazov Date: Mon, 1 Jul 2024 18:26:05 +0000 Subject: [PATCH] Fixed fq QueryService usage --- ydb/core/fq/libs/compute/common/utils.cpp | 25 +++++++++-------- ydb/core/fq/libs/compute/common/utils.h | 7 +++-- ydb/core/fq/libs/compute/ydb/events/events.h | 14 +++++----- .../fq/libs/compute/ydb/executer_actor.cpp | 28 +++++++++---------- ydb/core/fq/libs/compute/ydb/executer_actor.h | 4 ++- .../libs/compute/ydb/result_writer_actor.cpp | 6 ++-- .../libs/compute/ydb/status_tracker_actor.cpp | 4 +-- 7 files changed, 48 insertions(+), 40 deletions(-) diff --git a/ydb/core/fq/libs/compute/common/utils.cpp b/ydb/core/fq/libs/compute/common/utils.cpp index a7fc8cd81aa1..047d08f9eb08 100644 --- a/ydb/core/fq/libs/compute/common/utils.cpp +++ b/ydb/core/fq/libs/compute/common/utils.cpp @@ -5,6 +5,8 @@ #include #include +#include + namespace NFq { using TAggregates = std::map>; @@ -1133,8 +1135,8 @@ TString SimplifiedPlan(const TString& plan) { } struct TNoneStatProcessor : IPlanStatProcessor { - Ydb::Query::StatsMode GetStatsMode() override { - return Ydb::Query::StatsMode::STATS_MODE_NONE; + NYdb::NQuery::EStatsMode GetStatsMode() override { + return NYdb::NQuery::EStatsMode::None; } TString ConvertPlan(const TString& plan) override { @@ -1160,14 +1162,14 @@ struct TNoneStatProcessor : IPlanStatProcessor { }; struct TBasicStatProcessor : TNoneStatProcessor { - Ydb::Query::StatsMode GetStatsMode() override { - return Ydb::Query::StatsMode::STATS_MODE_BASIC; + NYdb::NQuery::EStatsMode GetStatsMode() override { + return NYdb::NQuery::EStatsMode::Basic; } }; struct TPlanStatProcessor : IPlanStatProcessor { - Ydb::Query::StatsMode GetStatsMode() override { - return Ydb::Query::StatsMode::STATS_MODE_FULL; + NYdb::NQuery::EStatsMode GetStatsMode() override { + return NYdb::NQuery::EStatsMode::Full; } TString ConvertPlan(const TString& plan) override { @@ -1204,8 +1206,8 @@ struct TCostStatProcessor : TPlanStatProcessor { }; struct TProfileStatProcessor : TPlanStatProcessor { - Ydb::Query::StatsMode GetStatsMode() override { - return Ydb::Query::StatsMode::STATS_MODE_PROFILE; + NYdb::NQuery::EStatsMode GetStatsMode() override { + return NYdb::NQuery::EStatsMode::Profile; } }; @@ -1233,7 +1235,7 @@ PingTaskRequestBuilder::PingTaskRequestBuilder(const NConfig::TCommonConfig& com {} Fq::Private::PingTaskRequest PingTaskRequestBuilder::Build( - const Ydb::TableStats::QueryStats& queryStats, + const NYdb::NQuery::TExecStats& queryStats, const NYql::TIssues& issues, std::optional computeStatus, std::optional pendingStatusCode @@ -1256,8 +1258,9 @@ Fq::Private::PingTaskRequest PingTaskRequestBuilder::Build( } -Fq::Private::PingTaskRequest PingTaskRequestBuilder::Build(const Ydb::TableStats::QueryStats& queryStats) { - return Build(queryStats.query_plan(), queryStats.query_ast(), queryStats.compilation().duration_us(), queryStats.total_duration_us()); +Fq::Private::PingTaskRequest PingTaskRequestBuilder::Build(const NYdb::NQuery::TExecStats& queryStats) { + const auto& statsProto = NYdb::TProtoAccessor().GetProto(queryStats); + return Build(statsProto.query_plan(), statsProto.query_ast(), statsProto.compilation().duration_us(), statsProto.total_duration_us()); } Fq::Private::PingTaskRequest PingTaskRequestBuilder::Build(const TString& queryPlan, const TString& queryAst, int64_t compilationTimeUs, int64_t computeTimeUs) { diff --git a/ydb/core/fq/libs/compute/common/utils.h b/ydb/core/fq/libs/compute/common/utils.h index 50a468141e85..f5a3fe91da8d 100644 --- a/ydb/core/fq/libs/compute/common/utils.h +++ b/ydb/core/fq/libs/compute/common/utils.h @@ -9,6 +9,7 @@ #include #include +#include namespace NFq { @@ -53,7 +54,7 @@ TPublicStat GetPublicStat(const TString& statistics); struct IPlanStatProcessor { virtual ~IPlanStatProcessor() = default; - virtual Ydb::Query::StatsMode GetStatsMode() = 0; + virtual NYdb::NQuery::EStatsMode GetStatsMode() = 0; virtual TString ConvertPlan(const TString& plan) = 0; virtual TString GetPlanVisualization(const TString& plan) = 0; virtual TString GetQueryStat(const TString& plan, double& cpuUsage) = 0; @@ -67,12 +68,12 @@ class PingTaskRequestBuilder { public: PingTaskRequestBuilder(const NConfig::TCommonConfig& commonConfig, std::unique_ptr&& processor); Fq::Private::PingTaskRequest Build( - const Ydb::TableStats::QueryStats& queryStats, + const NYdb::NQuery::TExecStats& queryStats, const NYql::TIssues& issues, std::optional computeStatus = std::nullopt, std::optional pendingStatusCode = std::nullopt ); - Fq::Private::PingTaskRequest Build(const Ydb::TableStats::QueryStats& queryStats); + Fq::Private::PingTaskRequest Build(const NYdb::NQuery::TExecStats& queryStats); Fq::Private::PingTaskRequest Build(const TString& queryPlan, const TString& queryAst, int64_t compilationTimeUs, int64_t computeTimeUs); NYql::TIssues Issues; double CpuUsage = 0.0; diff --git a/ydb/core/fq/libs/compute/ydb/events/events.h b/ydb/core/fq/libs/compute/ydb/events/events.h index c4276ae7c66b..741b5abbcfb0 100644 --- a/ydb/core/fq/libs/compute/ydb/events/events.h +++ b/ydb/core/fq/libs/compute/ydb/events/events.h @@ -71,7 +71,7 @@ struct TEvYdbCompute { // Events struct TEvExecuteScriptRequest : public NActors::TEventLocal { - TEvExecuteScriptRequest(TString sql, TString idempotencyKey, const TDuration& resultTtl, const TDuration& operationTimeout, Ydb::Query::Syntax syntax, Ydb::Query::ExecMode execMode, Ydb::Query::StatsMode statsMode, const TString& traceId, const std::map& queryParameters) + TEvExecuteScriptRequest(TString sql, TString idempotencyKey, const TDuration& resultTtl, const TDuration& operationTimeout, NYdb::NQuery::ESyntax syntax, NYdb::NQuery::EExecMode execMode, NYdb::NQuery::EStatsMode statsMode, const TString& traceId, const std::map& queryParameters) : Sql(std::move(sql)) , IdempotencyKey(std::move(idempotencyKey)) , ResultTtl(resultTtl) @@ -87,9 +87,9 @@ struct TEvYdbCompute { TString IdempotencyKey; TDuration ResultTtl; TDuration OperationTimeout; - Ydb::Query::Syntax Syntax = Ydb::Query::SYNTAX_YQL_V1; - Ydb::Query::ExecMode ExecMode = Ydb::Query::EXEC_MODE_EXECUTE; - Ydb::Query::StatsMode StatsMode = Ydb::Query::StatsMode::STATS_MODE_FULL; + NYdb::NQuery::ESyntax Syntax = NYdb::NQuery::ESyntax::YqlV1; + NYdb::NQuery::EExecMode ExecMode = NYdb::NQuery::EExecMode::Execute; + NYdb::NQuery::EStatsMode StatsMode = NYdb::NQuery::EStatsMode::Full; TString TraceId; std::map QueryParameters; }; @@ -127,7 +127,7 @@ struct TEvYdbCompute { , Ready(ready) {} - TEvGetOperationResponse(NYdb::NQuery::EExecStatus execStatus, Ydb::StatusIds::StatusCode statusCode, const TVector& resultSetsMeta, const Ydb::TableStats::QueryStats& queryStats, NYql::TIssues issues, bool ready = true) + TEvGetOperationResponse(NYdb::NQuery::EExecStatus execStatus, Ydb::StatusIds::StatusCode statusCode, const std::vector& resultSetsMeta, const NYdb::NQuery::TExecStats& queryStats, NYql::TIssues issues, bool ready = true) : ExecStatus(execStatus) , StatusCode(statusCode) , ResultSetsMeta(resultSetsMeta) @@ -139,8 +139,8 @@ struct TEvYdbCompute { NYdb::NQuery::EExecStatus ExecStatus = NYdb::NQuery::EExecStatus::Unspecified; Ydb::StatusIds::StatusCode StatusCode = Ydb::StatusIds::STATUS_CODE_UNSPECIFIED; - TVector ResultSetsMeta; - Ydb::TableStats::QueryStats QueryStats; + std::vector ResultSetsMeta; + NYdb::NQuery::TExecStats QueryStats; NYql::TIssues Issues; NYdb::EStatus Status; bool Ready; diff --git a/ydb/core/fq/libs/compute/ydb/executer_actor.cpp b/ydb/core/fq/libs/compute/ydb/executer_actor.cpp index 0f13cfe7e863..a9d45faa010a 100644 --- a/ydb/core/fq/libs/compute/ydb/executer_actor.cpp +++ b/ydb/core/fq/libs/compute/ydb/executer_actor.cpp @@ -59,7 +59,7 @@ class TExecuterActor : public TBaseComputeActor { } }; - TExecuterActor(const TRunActorParams& params, Ydb::Query::StatsMode statsMode, const TActorId& parent, const TActorId& connector, const TActorId& pinger, const ::NYql::NCommon::TServiceCounters& queryCounters) + TExecuterActor(const TRunActorParams& params, NYdb::NQuery::EStatsMode statsMode, const TActorId& parent, const TActorId& connector, const TActorId& pinger, const ::NYql::NCommon::TServiceCounters& queryCounters) : TBaseComputeActor(queryCounters, "Executer") , Params(params) , StatsMode(statsMode) @@ -115,38 +115,38 @@ class TExecuterActor : public TBaseComputeActor { } void SendExecuteScript() { - Register(new TRetryActor>(Counters.GetCounters(ERequestType::RT_EXECUTE_SCRIPT), SelfId(), Connector, Params.Sql, Params.JobId, Params.ResultTtl, Params.ExecutionTtl, GetSyntax(), GetExecuteMode(), StatsMode, Params.JobId + "_" + ToString(Params.RestartCount), Params.QueryParameters)); + Register(new TRetryActor>(Counters.GetCounters(ERequestType::RT_EXECUTE_SCRIPT), SelfId(), Connector, Params.Sql, Params.JobId, Params.ResultTtl, Params.ExecutionTtl, GetSyntax(), GetExecuteMode(), StatsMode, Params.JobId + "_" + ToString(Params.RestartCount), Params.QueryParameters)); } - Ydb::Query::Syntax GetSyntax() const { + NYdb::NQuery::ESyntax GetSyntax() const { switch (Params.QuerySyntax) { case FederatedQuery::QueryContent::PG: - return Ydb::Query::SYNTAX_PG; + return NYdb::NQuery::ESyntax::Pg; case FederatedQuery::QueryContent::YQL_V1: - return Ydb::Query::SYNTAX_YQL_V1; + return NYdb::NQuery::ESyntax::YqlV1; case FederatedQuery::QueryContent::QUERY_SYNTAX_UNSPECIFIED: case FederatedQuery::QueryContent_QuerySyntax_QueryContent_QuerySyntax_INT_MAX_SENTINEL_DO_NOT_USE_: case FederatedQuery::QueryContent_QuerySyntax_QueryContent_QuerySyntax_INT_MIN_SENTINEL_DO_NOT_USE_: - return Ydb::Query::SYNTAX_UNSPECIFIED; + return NYdb::NQuery::ESyntax::Unspecified; } } - Ydb::Query::ExecMode GetExecuteMode() const { + NYdb::NQuery::EExecMode GetExecuteMode() const { switch (Params.ExecuteMode) { case FederatedQuery::RUN: - return Ydb::Query::ExecMode::EXEC_MODE_EXECUTE; + return NYdb::NQuery::EExecMode::Execute; case FederatedQuery::PARSE: - return Ydb::Query::ExecMode::EXEC_MODE_PARSE; + return NYdb::NQuery::EExecMode::Parse; case FederatedQuery::VALIDATE: - return Ydb::Query::ExecMode::EXEC_MODE_VALIDATE; + return NYdb::NQuery::EExecMode::Validate; case FederatedQuery::EXPLAIN: - return Ydb::Query::ExecMode::EXEC_MODE_EXPLAIN; + return NYdb::NQuery::EExecMode::Explain; case FederatedQuery::EXECUTE_MODE_UNSPECIFIED: case FederatedQuery::COMPILE: case FederatedQuery::SAVE: case FederatedQuery::ExecuteMode_INT_MAX_SENTINEL_DO_NOT_USE_: case FederatedQuery::ExecuteMode_INT_MIN_SENTINEL_DO_NOT_USE_: - return Ydb::Query::ExecMode::EXEC_MODE_UNSPECIFIED; + return NYdb::NQuery::EExecMode::Unspecified; } } @@ -163,7 +163,7 @@ class TExecuterActor : public TBaseComputeActor { private: TRunActorParams Params; - Ydb::Query::StatsMode StatsMode; + NYdb::NQuery::EStatsMode StatsMode; TActorId Parent; TActorId Connector; TActorId Pinger; @@ -174,7 +174,7 @@ class TExecuterActor : public TBaseComputeActor { }; std::unique_ptr CreateExecuterActor(const TRunActorParams& params, - Ydb::Query::StatsMode statsMode, + NYdb::NQuery::EStatsMode statsMode, const TActorId& parent, const TActorId& connector, const TActorId& pinger, diff --git a/ydb/core/fq/libs/compute/ydb/executer_actor.h b/ydb/core/fq/libs/compute/ydb/executer_actor.h index c1a6c1d6478a..683d448e0e35 100644 --- a/ydb/core/fq/libs/compute/ydb/executer_actor.h +++ b/ydb/core/fq/libs/compute/ydb/executer_actor.h @@ -6,10 +6,12 @@ #include +#include + namespace NFq { std::unique_ptr CreateExecuterActor(const TRunActorParams& params, - Ydb::Query::StatsMode statsMode, + NYdb::NQuery::EStatsMode statsMode, const NActors::TActorId& parent, const NActors::TActorId& connector, const NActors::TActorId& pinger, diff --git a/ydb/core/fq/libs/compute/ydb/result_writer_actor.cpp b/ydb/core/fq/libs/compute/ydb/result_writer_actor.cpp index cae7e6b57f65..26f9f1a22765 100644 --- a/ydb/core/fq/libs/compute/ydb/result_writer_actor.cpp +++ b/ydb/core/fq/libs/compute/ydb/result_writer_actor.cpp @@ -327,8 +327,10 @@ class TResultWriterActor : public TBaseComputeActor { for (const auto& resultSetMeta: ev.Get()->Get()->ResultSetsMeta) { auto& meta = *PingTaskRequest.add_result_set_meta(); - for (const auto& column: resultSetMeta.columns()) { - *meta.add_column() = column; + for (const auto& column: resultSetMeta.Columns) { + auto& new_column = *meta.add_column(); + new_column.set_name(column.Name); + *new_column.mutable_type() = column.Type.GetProto(); } } diff --git a/ydb/core/fq/libs/compute/ydb/status_tracker_actor.cpp b/ydb/core/fq/libs/compute/ydb/status_tracker_actor.cpp index 062cf855d83f..8dae1eba177f 100644 --- a/ydb/core/fq/libs/compute/ydb/status_tracker_actor.cpp +++ b/ydb/core/fq/libs/compute/ydb/status_tracker_actor.cpp @@ -234,7 +234,7 @@ class TStatusTrackerActor : public TBaseComputeActor { } void UpdateCpuQuota(double cpuUsage) { - TDuration duration = TDuration::MicroSeconds(QueryStats.total_duration_us()); + TDuration duration = QueryStats.GetTotalDuration(); if (cpuUsage && duration) { Send(NFq::ComputeDatabaseControlPlaneServiceActorId(), new TEvYdbCompute::TEvCpuQuotaAdjust(Params.Scope.ToString(), duration, cpuUsage)); } @@ -282,7 +282,7 @@ class TStatusTrackerActor : public TBaseComputeActor { NYdb::EStatus Status = NYdb::EStatus::SUCCESS; NYdb::NQuery::EExecStatus ExecStatus = NYdb::NQuery::EExecStatus::Unspecified; NYql::NDqProto::StatusIds::StatusCode StatusCode = NYql::NDqProto::StatusIds::StatusCode::StatusIds_StatusCode_UNSPECIFIED; - Ydb::TableStats::QueryStats QueryStats; + NYdb::NQuery::TExecStats QueryStats; NKikimr::TBackoffTimer BackoffTimer; NFq::TStatusCodeByScopeCounters::TPtr FailedStatusCodeCounters; FederatedQuery::QueryMeta::ComputeStatus ComputeStatus = FederatedQuery::QueryMeta::RUNNING;