Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow data query for olap #12404

Merged
merged 1 commit into from
Dec 9, 2024
Merged

Allow data query for olap #12404

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ydb/core/kqp/compile_service/kqp_compile_actor.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -642,6 +642,7 @@ void ApplyServiceConfig(TKikimrConfiguration& kqpConfig, const TTableServiceConf
kqpConfig.ExtractPredicateRangesLimit = serviceConfig.GetExtractPredicateRangesLimit();
kqpConfig.EnablePerStatementQueryExecution = serviceConfig.GetEnablePerStatementQueryExecution();
kqpConfig.EnableCreateTableAs = serviceConfig.GetEnableCreateTableAs();
kqpConfig.AllowOlapDataQuery = serviceConfig.GetAllowOlapDataQuery();
kqpConfig.EnableOlapSink = serviceConfig.GetEnableOlapSink();
kqpConfig.EnableOltpSink = serviceConfig.GetEnableOltpSink();
kqpConfig.EnableHtapTx = serviceConfig.GetEnableHtapTx();
Expand Down
2 changes: 2 additions & 0 deletions ydb/core/kqp/compile_service/kqp_compile_service.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,7 @@ class TKqpCompileService : public TActorBootstrapped<TKqpCompileService> {

bool enableSequences = TableServiceConfig.GetEnableSequences();
bool enableColumnsWithDefault = TableServiceConfig.GetEnableColumnsWithDefault();
bool allowOlapDataQuery = TableServiceConfig.GetAllowOlapDataQuery();
bool enableOlapSink = TableServiceConfig.GetEnableOlapSink();
bool enableOltpSink = TableServiceConfig.GetEnableOltpSink();
bool enableHtapTx = TableServiceConfig.GetEnableHtapTx();
Expand Down Expand Up @@ -329,6 +330,7 @@ class TKqpCompileService : public TActorBootstrapped<TKqpCompileService> {
TableServiceConfig.GetIndexAutoChooseMode() != indexAutoChooser ||
TableServiceConfig.GetEnableSequences() != enableSequences ||
TableServiceConfig.GetEnableColumnsWithDefault() != enableColumnsWithDefault ||
TableServiceConfig.GetAllowOlapDataQuery() != allowOlapDataQuery ||
TableServiceConfig.GetEnableOlapSink() != enableOlapSink ||
TableServiceConfig.GetEnableOltpSink() != enableOltpSink ||
TableServiceConfig.GetEnableHtapTx() != enableHtapTx ||
Expand Down
4 changes: 3 additions & 1 deletion ydb/core/kqp/executer_actor/kqp_data_executer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ class TKqpDataExecuter : public TKqpExecuterBase<TKqpDataExecuter, EExecType::Da
, FederatedQuerySetup(federatedQuerySetup)
, GUCSettings(GUCSettings)
, ShardIdToTableInfo(shardIdToTableInfo)
, AllowOlapDataQuery(tableServiceConfig.GetAllowOlapDataQuery())
, BlockTrackingMode(tableServiceConfig.GetBlockTrackingMode())
{
Target = creator;
Expand Down Expand Up @@ -1926,7 +1927,7 @@ class TKqpDataExecuter : public TKqpExecuterBase<TKqpDataExecuter, EExecType::Da
}

bool HasDmlOperationOnOlap(NKqpProto::TKqpPhyTx_EType queryType, const NKqpProto::TKqpPhyStage& stage) {
if (queryType == NKqpProto::TKqpPhyTx::TYPE_DATA) {
if (queryType == NKqpProto::TKqpPhyTx::TYPE_DATA && !AllowOlapDataQuery) {
return true;
}

Expand Down Expand Up @@ -2953,6 +2954,7 @@ class TKqpDataExecuter : public TKqpExecuterBase<TKqpDataExecuter, EExecType::Da
const std::optional<TKqpFederatedQuerySetup> FederatedQuerySetup;
const TGUCSettings::TPtr GUCSettings;
TShardIdToTableInfoPtr ShardIdToTableInfo;
const bool AllowOlapDataQuery = false;

bool HasExternalSources = false;
bool SecretSnapshotRequired = false;
Expand Down
3 changes: 2 additions & 1 deletion ydb/core/kqp/opt/kqp_opt.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,8 @@ bool IsKqpEffectsStage(const TDqStageBase& stage) {
}

bool NeedSinks(const TKikimrTableDescription& table, const TKqpOptimizeContext& kqpCtx) {
return (kqpCtx.IsGenericQuery() || (kqpCtx.IsDataQuery() && table.Metadata->Kind != EKikimrTableKind::Olap))
return (kqpCtx.IsGenericQuery()
|| (kqpCtx.IsDataQuery() && (table.Metadata->Kind != EKikimrTableKind::Olap || kqpCtx.Config->AllowOlapDataQuery)))
&& (table.Metadata->Kind != EKikimrTableKind::Olap || kqpCtx.Config->EnableOlapSink)
&& (table.Metadata->Kind != EKikimrTableKind::Datashard || kqpCtx.Config->EnableOltpSink);
}
Expand Down
1 change: 1 addition & 0 deletions ydb/core/kqp/provider/yql_kikimr_settings.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ struct TKikimrConfiguration : public TKikimrSettings, public NCommon::TSettingDi
bool EnablePerStatementQueryExecution = false;
bool EnableCreateTableAs = false;
ui64 IdxLookupJoinsPrefixPointLimit = 1;
bool AllowOlapDataQuery = false;
bool EnableOlapSink = false;
bool EnableOltpSink = false;
bool EnableHtapTx = false;
Expand Down
13 changes: 5 additions & 8 deletions ydb/core/kqp/ut/olap/kqp_olap_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1820,7 +1820,11 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
}

void TestOlapUpsert(ui32 numShards) {
NKikimrConfig::TAppConfig appConfig;
appConfig.MutableTableServiceConfig()->SetEnableOlapSink(true);
appConfig.MutableTableServiceConfig()->SetAllowOlapDataQuery(true);
auto settings = TKikimrSettings()
.SetAppConfig(appConfig)
.SetWithSampleTables(false);
TKikimrRunner kikimr(settings);

Expand Down Expand Up @@ -1868,22 +1872,15 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
TString result = StreamResultToYson(it);
Cout << result << Endl;
//CompareYson(result, R"([[0;15];[1;15]])");
CompareYson(result, R"([])"); // FIXME
CompareYson(result, R"([[15;0];[15;1]])");
}
}

Y_UNIT_TEST(OlapUpsertImmediate) {
// Should be fixed in KIKIMR-17646
return;

TestOlapUpsert(1);
}

Y_UNIT_TEST(OlapUpsert) {
// Should be fixed in KIKIMR-17646
return;

TestOlapUpsert(2);
}

Expand Down
2 changes: 2 additions & 0 deletions ydb/core/protos/table_service_config.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,4 +347,6 @@ message TTableServiceConfig {
}

optional EBlockTrackingMode BlockTrackingMode = 73 [ default = BLOCK_TRACKING_SERIALIZE ];

optional bool AllowOlapDataQuery = 74 [default = false];
};
Loading