diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000000..def54e933a86 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,18 @@ +### Changelog entry + +... + +### Changelog category + +* New feature +* Experimental feature +* Improvement +* Performance improvement +* Bugfix +* Backward incompatible change +* Documentation (changelog entry is not required) +* Not for changelog (changelog entry is not required) + +### Additional information + +... diff --git a/.github/config/muted_ya.txt b/.github/config/muted_ya.txt index 9c96a8347eba..3ab191db3a4c 100644 --- a/.github/config/muted_ya.txt +++ b/.github/config/muted_ya.txt @@ -1,3 +1,4 @@ +ydb/core/blobstorage/dsproxy/ut TBlobStorageProxySequenceTest.TestBlock42PutWithChangingSlowDisk ydb/core/blobstorage/pdisk/ut TSectorMap.* ydb/core/blobstorage/ut_blobstorage CostMetricsGetBlock4Plus2.TestGet4Plus2BlockRequests10000Inflight1BlobSize1000 ydb/core/blobstorage/ut_blobstorage Defragmentation.DoesItWork @@ -6,30 +7,43 @@ ydb/core/blobstorage/ut_blobstorage VDiskAssimilation.Test ydb/core/blobstorage/ut_blobstorage [6/10]* ydb/core/blobstorage/ut_blobstorage/ut_read_only_vdisk ReadOnlyVDisk.TestStorageLoad ydb/core/cms/ut_sentinel TSentinelTests.BSControllerCantChangeStatus +ydb/core/persqueue/ut [31/40]* ydb/core/persqueue/ut TPersQueueMirrorer.TestBasicRemote +ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient ydb/core/kqp/ut/federated_query/generic * ydb/core/kqp/ut/scan KqpRequestContext.TraceIdInErrorMessage +ydb/core/kqp/ut/scheme KqpOlapScheme.TenThousandColumns ydb/core/kqp/ut/service KqpQueryService.ExecuteQueryPgTableSelect ydb/core/tx/columnshard/ut_schema TColumnShardTestSchema.ForgetAfterFail ydb/core/tx/columnshard/ut_schema TColumnShardTestSchema.RebootForgetAfterFail +ydb/core/tx/columnshard/engines/ut * ydb/core/tx/coordinator/ut Coordinator.RestoreTenantConfiguration +ydb/core/tx/schemeshard/ut_split_merge TSchemeShardSplitBySizeTest.Merge1KShards ydb/library/yql/providers/generic/connector/tests test.py.test_select_positive_postgresql* ydb/library/yql/sql/pg/ut PgSqlParsingAutoparam.AutoParamValues_DifferentTypes +ydb/library/yql/tests/sql/dq_file/part16 test.py.test[expr-as_dict_list_key-default.txt-Analyze] +ydb/library/yql/tests/sql/dq_file/part18 test.py.test[expr-cast_type_bind-default.txt-Analyze] ydb/public/sdk/cpp/client/ydb_topic/ut BasicUsage.WriteRead ydb/services/persqueue_v1/ut TPersQueueTest.DirectRead* -ydb/services/ydb/sdk_sessions_pool_ut YdbSdkSessionsPool.StressTestSync10 +ydb/services/persqueue_v1/ut [3/10]* +ydb/services/ydb/sdk_sessions_pool_ut YdbSdkSessionsPool.StressTestSync* ydb/services/ydb/table_split_ut YdbTableSplit.SplitByLoadWithReadsMultipleSplitsWithData ydb/services/ydb/ut YdbOlapStore.LogPagingAfter-NotNull ydb/tests/fq/s3 * ydb/tests/fq/yds test_metrics_cleanup.py.TestCleanup.test_cleanup[v1] ydb/tests/functional/audit * +ydb/tests/functional/blobstorage test_replication.py.TestReplicationAfterNodesRestart.test_replication[mirror-3-dc] ydb/tests/functional/clickbench test.py.test_plans* +ydb/tests/functional/clickbench test.py.test_run_determentistic[column] ydb/tests/functional/kqp/kqp_query_session KqpQuerySession.NoLocalAttach ydb/tests/functional/postgresql test_postgres.py.TestPostgresSuite.test_postgres_suite* ydb/tests/functional/restarts test_restarts.py.* ydb/tests/functional/sqs/cloud test_yandex_cloud_mode.py.TestSqsYandexCloudMode.test_dlq_mechanics_in_cloud* +ydb/tests/functional/sqs/cloud test_yandex_cloud_queue_counters.py.TestYmqQueueCounters.test_purge_queue_counters +ydb/tests/functional/sqs/common test_queue_counters.py.TestSqsGettingCounters.test_purge_queue_counters ydb/tests/functional/tenants test_dynamic_tenants.py.* ydb/tests/functional/tenants test_storage_config.py.TestStorageConfig.* ydb/tests/functional/tenants test_tenants.py.* ydb/tests/functional/ydb_cli test_ydb_scripting.py.TestScriptingServiceHelp.test_help ydb/tests/functional/ydb_cli test_ydb_scripting.py.TestScriptingServiceHelp.test_help_ex +ydb/tests/tools/pq_read/test test_timeout.py.TestTimeout.test_timeout diff --git a/.github/workflows/allowed_dirs.yml b/.github/workflows/allowed_dirs.yml index 0f14886bc0c4..98167cb10a04 100644 --- a/.github/workflows/allowed_dirs.yml +++ b/.github/workflows/allowed_dirs.yml @@ -6,8 +6,10 @@ on: jobs: build: + concurrency: + group: check-allowed-dirs-${{ github.ref }} + cancel-in-progress: true runs-on: ubuntu-latest - steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/docs_preview.yaml b/.github/workflows/docs_preview.yaml index d57a0f7cdc06..663019f10475 100644 --- a/.github/workflows/docs_preview.yaml +++ b/.github/workflows/docs_preview.yaml @@ -10,6 +10,9 @@ on: jobs: post-build: permissions: write-all + concurrency: + group: preview-documentation-${{ github.ref }} + cancel-in-progress: true runs-on: ubuntu-latest steps: - name: Upload diff --git a/.github/workflows/docs_release.yaml b/.github/workflows/docs_release.yaml index 1d4341d343b0..0c7ae9da9f82 100644 --- a/.github/workflows/docs_release.yaml +++ b/.github/workflows/docs_release.yaml @@ -11,6 +11,9 @@ on: jobs: release: + concurrency: + group: release-documentation-${{ github.ref }} + cancel-in-progress: true runs-on: ubuntu-latest steps: - name: Checkout diff --git a/ROADMAP.md b/ROADMAP.md index f2c7948d8e36..644d33dd226d 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -23,7 +23,7 @@ We use the following symbols as abbreviations: 1. ㉔ Support for **temporary tables** 1. ㉔ Support for **VIEW** SQL clause 1. ㉔ **Data Spilling** in case there is issufient amount of RAM -1. ㉔ **TPC-H, TPC-H for 100TB** dataset +1. ㉔ **TPC-H, TPC-DS for 100TB** dataset 1. ✅ ㉓ Support for **Snapshot Readonly** transactions mode 1. 🚧 ㉓ **Better resource management** for KQP Resource Manager (share information about nodes resources, avoid OOMs) 1. ✅ ㉓ Switch to **New Engine** for OLTP queries @@ -120,13 +120,7 @@ We use the following symbols as abbreviations: 1. ✅ ㉓ **Run the first version** ## Embedded UI -1. **Support for all schema entities** - * ㉓ **YDB Topics** (add support for viewing metadata of YDB topics, its data, lag, etc) - * ㉓ **CDC Streams** - * ㉓ **Secondary Indexes** - * ㉓ **Read Replicas** - * ✅ ㉓ **Column-oriented Tables** -1. ㉓ **Basic charts for database monitoring** +Detailed roadmap could be found at [YDB Embedded UI repo](https://github.com/ydb-platform/ydb-embedded-ui/blob/main/ROADMAP.md). ## Command Line Utility 1. 🚧 ㉓ Use a **single `ydb yql`** instead of `ydb table query` or `ydb scripting` diff --git a/ydb/core/base/logoblob.cpp b/ydb/core/base/logoblob.cpp index d5deb4b75af5..385c0a3aa14d 100644 --- a/ydb/core/base/logoblob.cpp +++ b/ydb/core/base/logoblob.cpp @@ -1,35 +1,26 @@ #include "logoblob.h" #include -#include namespace NKikimr { TString TLogoBlobID::ToString() const { - return Sprintf( - "[%" PRIu64 ":%" PRIu32 ":%" PRIu32 ":%" PRIu32 ":%" PRIu32 ":%" PRIu32 ":%" PRIu32 "]", - TabletID(), - Generation(), - Step(), - Channel(), - Cookie(), - BlobSize(), - PartId()).data(); + TString str; + str.reserve(64); + TStringOutput outStr(str); + Out(outStr); + return str; } void TLogoBlobID::Out(IOutputStream &o) const { - char buf[240]; - snprintf(buf, sizeof(buf), - "[%" PRIu64 ":%" PRIu32 ":%" PRIu32 ":%" PRIu32 ":%" PRIu32 ":%" PRIu32 ":%" PRIu32 "]", - TabletID(), - Generation(), - Step(), - Channel(), - Cookie(), - BlobSize(), - PartId() - ); - - o << buf; + o << '[' + << TabletID() << ':' + << Generation() << ':' + << Step() << ':' + << Channel() << ':' + << Cookie() << ':' + << BlobSize() << ':' + << PartId() + << ']' ; } void TLogoBlobID::Out(IOutputStream &o, const TVector &vec) { diff --git a/ydb/core/base/traceid.cpp b/ydb/core/base/traceid.cpp index 23311ea0d1cc..af3cace92f25 100644 --- a/ydb/core/base/traceid.cpp +++ b/ydb/core/base/traceid.cpp @@ -20,16 +20,15 @@ TTraceID TTraceID::GenerateNew() { } TString TTraceID::ToString() const { - TString result; - TStringOutput out(result); - Out(out); - return result; + TString str; + str.reserve(128); + TStringOutput outStr(str); + Out(outStr); + return str; } void TTraceID::Out(IOutputStream &o) const { - char buf[240]; - snprintf(buf, sizeof(buf), "[ID:%" PRIu64 ", Created: %s]", RandomID, TInstant::MicroSeconds(CreationTime).ToRfc822StringLocal().data()); - o << buf; + o << "[ID: " << RandomID << ", " << "Created: " << TInstant::MicroSeconds(CreationTime).ToRfc822StringLocal() << "]"; } bool TTraceID::operator<(const TTraceID &x) const { diff --git a/ydb/core/formats/arrow/converter.cpp b/ydb/core/formats/arrow/converter.cpp index d832275dcf5d..58bcc0f8aafe 100644 --- a/ydb/core/formats/arrow/converter.cpp +++ b/ydb/core/formats/arrow/converter.cpp @@ -1,9 +1,11 @@ #include "converter.h" #include "switch_type.h" +#include #include #include +#include #include #include @@ -47,16 +49,21 @@ static bool ConvertData(TCell& cell, const NScheme::TTypeInfo& colType, TMemoryP } static bool ConvertColumn(const NScheme::TTypeInfo colType, std::shared_ptr& column, std::shared_ptr& field) { - if (colType.GetTypeId() == NScheme::NTypeIds::Decimal) { + switch (colType.GetTypeId()) { + case NScheme::NTypeIds::Decimal: return false; + case NScheme::NTypeIds::JsonDocument: { + const static TSet jsonDocArrowTypes{ arrow::Type::BINARY, arrow::Type::STRING }; + if (!jsonDocArrowTypes.contains(column->type()->id())) { + return false; + } + break; } - - if ((colType.GetTypeId() == NScheme::NTypeIds::JsonDocument) && - (column->type()->id() == arrow::Type::BINARY || column->type()->id() == arrow::Type::STRING)) - { - ; - } else if (column->type()->id() != arrow::Type::BINARY) { - return false; + default: + if (column->type()->id() != arrow::Type::BINARY) { + return false; + } + break; } auto& binaryArray = static_cast(*column); @@ -81,9 +88,16 @@ static bool ConvertColumn(const NScheme::TTypeInfo colType, std::shared_ptrData(), binaryJson->Size()).ok()) { - return false; + const TStringBuf valueBuf(value.data(), value.size()); + if (NBinaryJson::IsValidBinaryJson(valueBuf)) { + if (!builder.Append(value).ok()) { + return false; + } + } else { + const auto binaryJson = NBinaryJson::SerializeToBinaryJson(valueBuf); + if (!binaryJson.Defined() || !builder.Append(binaryJson->Data(), binaryJson->Size()).ok()) { + return false; + } } } } diff --git a/ydb/core/fq/libs/checkpointing/checkpoint_coordinator.cpp b/ydb/core/fq/libs/checkpointing/checkpoint_coordinator.cpp index 5186054fdbbc..6832f05f2dc7 100644 --- a/ydb/core/fq/libs/checkpointing/checkpoint_coordinator.cpp +++ b/ydb/core/fq/libs/checkpointing/checkpoint_coordinator.cpp @@ -103,6 +103,12 @@ void TCheckpointCoordinator::Handle(NYql::NDqs::TEvReadyState::TPtr& ev) { AllActorsSet.insert(actorId); } + CC_LOG_D("ActorsToTrigger count: " << ActorsToTrigger.size() << ", ActorsToNotify count: " << ActorsToNotify.size() << ", ActorsToWaitFor count: " << ActorsToWaitFor.size()); + + if (ActorsToTrigger.empty()) { + CC_LOG_D("No ingress tasks, coordinator was disabled"); + return; + } PendingInit = std::make_unique(AllActors.size()); CC_LOG_D("Send TEvRegisterCoordinatorRequest"); diff --git a/ydb/core/fq/libs/checkpointing/ut/checkpoint_coordinator_ut.cpp b/ydb/core/fq/libs/checkpointing/ut/checkpoint_coordinator_ut.cpp index 5ca54341612c..6cd2a11461b0 100644 --- a/ydb/core/fq/libs/checkpointing/ut/checkpoint_coordinator_ut.cpp +++ b/ydb/core/fq/libs/checkpointing/ut/checkpoint_coordinator_ut.cpp @@ -19,7 +19,7 @@ enum ETestGraphFlags : ui64 { SourceWithChannelInOneTask = 2, }; -NYql::NDqProto::TReadyState BuildTestGraph(ui64 flags = 0) { +NYql::NDqProto::TReadyState BuildTestGraph(ui64 flags, const TString& sourceType) { NYql::NDqProto::TReadyState result; @@ -29,7 +29,7 @@ NYql::NDqProto::TReadyState BuildTestGraph(ui64 flags = 0) { ingressOutput->AddChannels(); if (flags & ETestGraphFlags::InputWithSource) { auto* source = ingress->AddInputs()->MutableSource(); - source->SetType("PqSource"); + source->SetType(sourceType); } auto* map = result.AddTask(); @@ -40,7 +40,7 @@ NYql::NDqProto::TReadyState BuildTestGraph(ui64 flags = 0) { mapOutput->AddChannels(); if (flags & ETestGraphFlags::SourceWithChannelInOneTask) { auto* source = map->AddInputs()->MutableSource(); - source->SetType("PqSource"); + source->SetType(sourceType); } auto* egress = result.AddTask(); @@ -70,9 +70,9 @@ struct TTestBootstrap : public TTestActorRuntime { ::NMonitoring::TDynamicCounterPtr Counters = new ::NMonitoring::TDynamicCounters(); - explicit TTestBootstrap(ui64 graphFlags = 0) + explicit TTestBootstrap(ui64 graphFlags, const TString& sourceType) : TTestActorRuntime(true) - , GraphState(BuildTestGraph(graphFlags)) + , GraphState(BuildTestGraph(graphFlags, sourceType)) , CoordinatorId("my-graph-id", 42) , CheckpointId(CoordinatorId.Generation, 1) { @@ -281,8 +281,8 @@ Y_UNIT_TEST_SUITE(TCheckpointCoordinatorTests) { class CheckpointsTestHelper : public TTestBootstrap { public: - CheckpointsTestHelper(ui64 graphFlags) - : TTestBootstrap(graphFlags) { + CheckpointsTestHelper(ui64 graphFlags, const TString& sourceType) + : TTestBootstrap(graphFlags, sourceType) { } void InjectCheckpoint() { @@ -372,22 +372,33 @@ Y_UNIT_TEST_SUITE(TCheckpointCoordinatorTests) { }; Y_UNIT_TEST(ShouldTriggerCheckpointWithSource) { - CheckpointsTestHelper test(ETestGraphFlags::InputWithSource); + CheckpointsTestHelper test(ETestGraphFlags::InputWithSource, "PqSource"); test.InjectCheckpoint(); test.AllSavedAndCommited(); } Y_UNIT_TEST(ShouldTriggerCheckpointWithSourcesAndWithChannel) { - CheckpointsTestHelper test(ETestGraphFlags::InputWithSource | ETestGraphFlags::SourceWithChannelInOneTask); + CheckpointsTestHelper test(ETestGraphFlags::InputWithSource | ETestGraphFlags::SourceWithChannelInOneTask, "PqSource"); test.InjectCheckpoint(); test.AllSavedAndCommited(); } Y_UNIT_TEST(ShouldAbortPreviousCheckpointsIfNodeStateCantBeSaved) { - CheckpointsTestHelper test(ETestGraphFlags::InputWithSource); + CheckpointsTestHelper test(ETestGraphFlags::InputWithSource, "PqSource"); test.InjectCheckpoint(); test.SaveFailed(); } + + Y_UNIT_TEST(ShouldDoNothingIfNoIngressTasks) { + CheckpointsTestHelper test(ETestGraphFlags::InputWithSource, "S3Source"); + bool empty = false; + try { + test.GrabEdgeEvent(test.StorageProxy, TDuration::Seconds(10)); + } catch (TEmptyEventQueueException&) { + empty = true; + } + UNIT_ASSERT(empty); + } } } // namespace NFq diff --git a/ydb/core/keyvalue/keyvalue_ut_trace.cpp b/ydb/core/keyvalue/keyvalue_ut_trace.cpp new file mode 100644 index 000000000000..f918a189bdc8 --- /dev/null +++ b/ydb/core/keyvalue/keyvalue_ut_trace.cpp @@ -0,0 +1,154 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace NActors; +using namespace NKikimr; + +struct TTestEnvironment { + THolder Runtime; + const ui32 NodeCount; + TActorId Edge; + const ui64 TabletId = MakeTabletID(0, 0, 1); + const TTabletTypes::EType TabletType = TTabletTypes::KeyValue; + NWilson::TFakeWilsonUploader* WilsonUploader = nullptr; + + TTestEnvironment(ui32 nodeCount): NodeCount(nodeCount) { + } + + void Prepare() { + SetupRuntime(); + InitializeRuntime(); + + Edge = Runtime->AllocateEdgeActor(); + CreateTestBootstrapper(*Runtime, + CreateTestTabletInfo(TabletId, TabletType, TErasureType::ErasureNone), + &CreateKeyValueFlat); + SetupFakeWilson(); + + TDispatchOptions options; + options.FinalEvents.push_back(TDispatchOptions::TFinalEventCondition(TEvTablet::EvBoot)); + Runtime->DispatchEvents(options); + } + + void InitializeRuntime() { + TAppPrepare app; + app.AddDomain(TDomainsInfo::TDomain::ConstructEmptyDomain("dc-1").Release()); + SetupTabletServices(*Runtime, &app); + } + + void SetupRuntime() { + Runtime = MakeHolder(NodeCount, 1u); + + for (ui32 i = 0; i < NodeCount; ++i) { + SetupStateStorage(*Runtime, i, 0, true); + SetupTabletResolver(*Runtime, i); + } + } + + void SetupFakeWilson() { + WilsonUploader = new NWilson::TFakeWilsonUploader; + auto actorId = Runtime->Register(WilsonUploader); + Runtime->RegisterService(NWilson::MakeWilsonUploaderId(), actorId); + } + + template + auto DoKVRequest(THolder request) { + Runtime->SendToPipe(TabletId, Edge, request.Release(), 0, NTabletPipe::TClientConfig(), TActorId(), + 0, NWilson::TTraceId::NewTraceId(15, 4095)); + TAutoPtr handle; + auto response = Runtime->GrabEdgeEventRethrow(handle); + UNIT_ASSERT(response); + auto& record = response->Record; + UNIT_ASSERT_EQUAL(record.status(), NKikimrKeyValue::Statuses::RSTATUS_OK); + + return std::move(record); + } +}; + +THolder CreateWrite(TString key, TString value) { + auto request = MakeHolder(); + auto write = request->Record.add_commands()->mutable_write(); + write->set_key(std::move(key)); + write->set_value(std::move(value)); + return request; +} + +THolder CreateRead(TString key) { + auto request = MakeHolder(); + auto& record = request->Record; + record.set_key(std::move(key)); + record.set_offset(0); + record.set_size(0); + record.set_limit_bytes(0); + return request; +} + +void TestOneWrite(TString value, TString expectedTrace) { + TTestEnvironment env(8); + env.Prepare(); + + env.DoKVRequest(CreateWrite("key", std::move(value))); + + UNIT_ASSERT(env.WilsonUploader->BuildTraceTrees()); + UNIT_ASSERT_EQUAL(env.WilsonUploader->Traces.size(), 1); + auto& trace = env.WilsonUploader->Traces.begin()->second; + + UNIT_ASSERT_EQUAL(trace.ToString(), expectedTrace); +} + +void TestOneRead(TString value, TString expectedTrace) { + TTestEnvironment env(8); + env.Prepare(); + + env.DoKVRequest(CreateWrite("key", value)); + env.WilsonUploader->Clear(); + + auto response = env.DoKVRequest(CreateRead("key")); + UNIT_ASSERT_EQUAL(response.value(), value); + + UNIT_ASSERT(env.WilsonUploader->BuildTraceTrees()); + UNIT_ASSERT_EQUAL(env.WilsonUploader->Traces.size(), 1); + auto& trace = env.WilsonUploader->Traces.begin()->second; + + UNIT_ASSERT_EQUAL(trace.ToString(), expectedTrace); +} + +Y_UNIT_TEST_SUITE(TKeyValueTracingTest) { + const TString SmallValue = "value"; + const TString HugeValue = TString(1 << 20, 'v'); + +Y_UNIT_TEST(WriteSmall) { + TString canon = "(KeyValue.Intermediate -> [(KeyValue.StorageRequest -> [(DSProxy.Put -> [(Backpressure.InFlight " + "-> [(VDisk.Log.Put)])])]) , (Tablet.Transaction -> [(Tablet.Transaction.Execute) , (Tablet.WriteLog -> " + "[(Tablet.WriteLog.LogEntry -> [(DSProxy.Put -> [(Backpressure.InFlight -> [(VDisk.Log.Put)])])])])])])"; + TestOneWrite(SmallValue, std::move(canon)); +} + +Y_UNIT_TEST(WriteHuge) { + TString canon = "(KeyValue.Intermediate -> [(KeyValue.StorageRequest -> [(DSProxy.Put -> [(Backpressure.InFlight " + "-> [(VDisk.HugeBlobKeeper.Write -> [(VDisk.Log.PutHuge)])])])]) , (Tablet.Transaction -> " + "[(Tablet.Transaction.Execute) , (Tablet.WriteLog -> [(Tablet.WriteLog.LogEntry -> [(DSProxy.Put -> " + "[(Backpressure.InFlight -> [(VDisk.Log.Put)])])])])])])"; + TestOneWrite(HugeValue, std::move(canon)); +} + +Y_UNIT_TEST(ReadSmall) { + TString canon = "(KeyValue.Intermediate -> [(KeyValue.StorageReadRequest -> [(DSProxy.Get -> [(Backpressure.InFlight -> " + "[(VDisk.LevelIndexExtremeQueryViaBatcherMergeData)])])])])"; + TestOneRead(SmallValue, std::move(canon)); +} + +Y_UNIT_TEST(ReadHuge) { + TString canon = "(KeyValue.Intermediate -> [(KeyValue.StorageReadRequest -> [(DSProxy.Get -> [(Backpressure.InFlight -> " + "[(VDisk.LevelIndexExtremeQueryViaBatcherMergeData -> [(VDisk.Query.ReadBatcher)])])])])])"; + TestOneRead(HugeValue, std::move(canon)); +} + +} diff --git a/ydb/core/keyvalue/ut_trace/ya.make b/ydb/core/keyvalue/ut_trace/ya.make new file mode 100644 index 000000000000..a6c8e0b9d692 --- /dev/null +++ b/ydb/core/keyvalue/ut_trace/ya.make @@ -0,0 +1,26 @@ +UNITTEST_FOR(ydb/core/keyvalue) + +FORK_SUBTESTS() + +SPLIT_FACTOR(5) + +IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) + TIMEOUT(1800) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +PEERDIR( + ydb/core/testlib/default +) + +SRCS( + keyvalue_ut_trace.cpp +) + +REQUIREMENTS(ram:16) + +END() diff --git a/ydb/core/keyvalue/ya.make b/ydb/core/keyvalue/ya.make index f8cfaab1866f..3a014e31316d 100644 --- a/ydb/core/keyvalue/ya.make +++ b/ydb/core/keyvalue/ya.make @@ -62,4 +62,5 @@ RECURSE( RECURSE_FOR_TESTS( ut + ut_trace ) diff --git a/ydb/core/kqp/executer_actor/kqp_executer_stats.cpp b/ydb/core/kqp/executer_actor/kqp_executer_stats.cpp index 6b8b194b27d0..34633516825a 100644 --- a/ydb/core/kqp/executer_actor/kqp_executer_stats.cpp +++ b/ydb/core/kqp/executer_actor/kqp_executer_stats.cpp @@ -330,7 +330,7 @@ bool CollectProfileStats(Ydb::Table::QueryStatsCollection::Mode statsMode) { void TQueryExecutionStats::FillStageDurationUs(NYql::NDqProto::TDqStageStats& stats) { if (stats.HasStartTimeMs() && stats.HasFinishTimeMs()) { auto startTimeMs = stats.GetStartTimeMs().GetMin(); - auto finishTimeMs = stats.GetFinishTimeMs().GetMin(); + auto finishTimeMs = stats.GetFinishTimeMs().GetMax(); if (startTimeMs && finishTimeMs > startTimeMs) { stats.SetStageDurationUs((finishTimeMs - startTimeMs) * 1'000); } diff --git a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json index df0a9834dfde..970c0d6589a0 100644 --- a/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json +++ b/ydb/core/kqp/expr_nodes/kqp_expr_nodes.json @@ -277,7 +277,8 @@ "Match": {"Type": "Callable", "Name": "KqlUpsertRowsIndex"}, "Children": [ {"Index": 3, "Name": "ReturningColumns", "Type": "TCoAtomList"}, - {"Index": 4, "Name": "Settings", "Type": "TCoNameValueTupleList", "Optional": true} + {"Index": 4, "Name": "GenerateColumnsIfInsert", "Type": "TCoAtomList"}, + {"Index": 5, "Name": "Settings", "Type": "TCoNameValueTupleList", "Optional": true} ] }, { @@ -569,6 +570,15 @@ "VarArgBase": "TExprBase", "Match": {"Type": "Callable", "Name": "KqpOlapXor"} }, + { + "Name": "TKqpOlapFilterUnaryOp", + "Base": "TExprBase", + "Match": {"Type": "Tuple", "Name": "KqpOlapFilterUnaryOp"}, + "Children": [ + {"Index": 0, "Name": "Operator", "Type": "TCoAtom"}, + {"Index": 1, "Name": "Arg", "Type": "TExprBase"} + ] + }, { "Name": "TKqpOlapFilterBinaryOp", "Base": "TExprBase", diff --git a/ydb/core/kqp/host/kqp_type_ann.cpp b/ydb/core/kqp/host/kqp_type_ann.cpp index 59162574e048..2796aa882c4f 100644 --- a/ydb/core/kqp/host/kqp_type_ann.cpp +++ b/ydb/core/kqp/host/kqp_type_ann.cpp @@ -869,10 +869,19 @@ bool ValidateOlapFilterConditions(const TExprNode* node, const TStructExprType* } } return res; - } else if (TKqpOlapFilterBinaryOp::Match(node)) { - if (!EnsureArgsCount(*node, 3, ctx)) { + } else if (TKqpOlapFilterUnaryOp::Match(node)) { + const auto op = node->Child(TKqpOlapFilterUnaryOp::idx_Operator); + if (!EnsureAtom(*op, ctx)) { return false; } + if (!op->IsAtom({"minus", "abs", "not", "size", "exists", "empty"})) { + ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), + TStringBuilder() << "Unexpected OLAP unary operation: " << op->Content() + )); + return false; + } + return ValidateOlapFilterConditions(node->Child(TKqpOlapFilterUnaryOp::idx_Arg), itemType, ctx); + } else if (TKqpOlapFilterBinaryOp::Match(node)) { const auto op = node->Child(TKqpOlapFilterBinaryOp::idx_Operator); if (!EnsureAtom(*op, ctx)) { return false; diff --git a/ydb/core/kqp/opt/kqp_opt_kql.cpp b/ydb/core/kqp/opt/kqp_opt_kql.cpp index 50a02406d19f..90db2d10ccc9 100644 --- a/ydb/core/kqp/opt/kqp_opt_kql.cpp +++ b/ydb/core/kqp/opt/kqp_opt_kql.cpp @@ -313,11 +313,16 @@ TExprBase BuildUpsertTableWithIndex(const TKiWriteTable& write, const TCoAtomLis const TKikimrTableDescription& table, TExprContext& ctx) { const auto [input, columns] = BuildWriteInput(write, table, inputColumns, autoincrement, write.Pos(), ctx); + auto generateColumnsIfInsertNode = GetSetting(write.Settings().Ref(), "generate_columns_if_insert"); + YQL_ENSURE(generateColumnsIfInsertNode); + TCoAtomList generateColumnsIfInsert = TCoNameValueTuple(generateColumnsIfInsertNode).Value().Cast(); + auto effect = Build(ctx, write.Pos()) .Table(BuildTableMeta(table, write.Pos(), ctx)) .Input(input.Ptr()) .Columns(columns.Ptr()) .ReturningColumns(write.ReturningColumns()) + .GenerateColumnsIfInsert(generateColumnsIfInsert) .Done(); return effect; @@ -348,6 +353,7 @@ TExprBase BuildReplaceTableWithIndex(const TKiWriteTable& write, const TCoAtomLi .Input(input.Ptr()) .Columns(columns.Ptr()) .ReturningColumns(write.ReturningColumns()) + .GenerateColumnsIfInsert().Build() .Done(); return effect; diff --git a/ydb/core/kqp/opt/kqp_query_plan.cpp b/ydb/core/kqp/opt/kqp_query_plan.cpp index 141cbcabee2e..26a791b44932 100644 --- a/ydb/core/kqp/opt/kqp_query_plan.cpp +++ b/ydb/core/kqp/opt/kqp_query_plan.cpp @@ -267,7 +267,7 @@ class TxPlanSerializer { } return res; } - }; + }; }; struct TOperator { @@ -345,7 +345,7 @@ class TxPlanSerializer { else { TArgContext c = std::get(input); writer.BeginObject(); - + auto input = LambdaInputs.find(c); if (input != LambdaInputs.end()){ if (std::holds_alternative(input->second)) { @@ -814,10 +814,13 @@ class TxPlanSerializer { } // Common settings that can be overwritten by provider - op.Properties["Name"] = "Read from external data source"; op.Properties["SourceType"] = dataSourceCategory; if (auto cluster = TryGetCluster(dataSource)) { - op.Properties["ExternalDataSource"] = RemovePathPrefix(std::move(*cluster)); + TString dataSource = RemovePathPrefix(std::move(*cluster)); + op.Properties["ExternalDataSource"] = dataSource; + op.Properties["Name"] = TStringBuilder() << "Read " << dataSource; + } else { + op.Properties["Name"] = "Read from external data source"; } if (dqIntegration) { @@ -842,10 +845,13 @@ class TxPlanSerializer { } // Common settings that can be overwritten by provider - op.Properties["Name"] = "Write to external data source"; op.Properties["SinkType"] = dataSinkCategory; if (auto cluster = TryGetCluster(dataSink)) { - op.Properties["ExternalDataSource"] = RemovePathPrefix(std::move(*cluster)); + TString dataSource = RemovePathPrefix(std::move(*cluster)); + op.Properties["ExternalDataSource"] = dataSource; + op.Properties["Name"] = TStringBuilder() << "Write " << dataSource; + } else { + op.Properties["Name"] = "Write to external data source"; } if (dqIntegration) { @@ -1036,7 +1042,7 @@ class TxPlanSerializer { inputIds.insert(inputIds.end(), flatMapLambdaInputs.begin(), flatMapLambdaInputs.end()); } - else { + else { for (const auto& child : node->Children()) { if(!child->IsLambda()) { auto ids = Visit(child, planNode); @@ -1278,7 +1284,7 @@ class TxPlanSerializer { auto operatorId = AddOperator(planNode, name, std::move(op)); Visit(flatMap, planNode); - + return operatorId; } @@ -1333,7 +1339,7 @@ class TxPlanSerializer { AddOptimizerEstimates(op, join); Visit(flatMap, planNode); - + return operatorId; } @@ -2101,7 +2107,7 @@ TString AddExecStatsToTxPlan(const TString& txPlanJson, const NYql::NDqProto::TD } stats["Tasks"] = (*stat)->GetTotalTasksCount(); - + stats["StageDurationUs"] = (*stat)->GetStageDurationUs(); if ((*stat)->HasDurationUs()) { diff --git a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects_impl.h b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects_impl.h index e2be88de4c86..3765e156ec80 100644 --- a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects_impl.h +++ b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_effects_impl.h @@ -80,6 +80,7 @@ NYql::NNodes::TMaybeNode KqpPhyUpsertIndexEffectsImpl(T const NYql::NNodes::TExprBase& inputRows, const NYql::NNodes::TCoAtomList& inputColumns, const NYql::NNodes::TCoAtomList& returningColumns, + const NYql::NNodes::TCoAtomList& columnsWithDefaults, const NYql::TKikimrTableDescription& table, const NYql::NNodes::TMaybeNode& settings, NYql::TPositionHandle pos, NYql::TExprContext& ctx); diff --git a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_update_index.cpp b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_update_index.cpp index fb5b6046a441..e00577ec7c9e 100644 --- a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_update_index.cpp +++ b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_update_index.cpp @@ -15,8 +15,10 @@ TExprBase KqpBuildUpdateIndexStages(TExprBase node, TExprContext& ctx, const TKq auto update = node.Cast(); const auto& table = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, update.Table().Path()); + TCoAtomList empty = Build(ctx, node.Pos()).Done(); + auto effects = KqpPhyUpsertIndexEffectsImpl(TKqpPhyUpsertIndexMode::UpdateOn, update.Input(), - update.Columns(), update.ReturningColumns(), table, update.Settings(), update.Pos(), ctx); + update.Columns(), update.ReturningColumns(), empty, table, update.Settings(), update.Pos(), ctx); if (!effects) { return node; diff --git a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_defaults.cpp b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_defaults.cpp index 48e25ba53013..d2f890cb3dc9 100644 --- a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_defaults.cpp +++ b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_defaults.cpp @@ -24,59 +24,8 @@ TMaybeNode PrecomputeCurrentDefaultsForKeys(const TDqPhyPrecom lookupColumns.push_back(atom); } - auto lookupColumnsList = Build(ctx, pos) - .Add(lookupColumns) - .Done(); - - auto lookupStage = Build(ctx, pos) - .Inputs() - .Add(lookupKeys) - .Build() - .Program() - .Args({"keys_list"}) - .Body() - .Table(BuildTableMeta(table, pos, ctx)) - .LookupKeys() - .List("keys_list") - .Build() - .Columns(lookupColumnsList) - .Build() - .Build() - .Settings().Build() - .Done(); - - auto lookup = Build(ctx, pos) - .Output() - .Stage(lookupStage) - .Index().Build("0") - .Build() - .Done(); - - auto lookupPayloadSelector = MakeRowsPayloadSelector(lookupColumnsList, table, lookupKeys.Pos(), ctx); - auto condenseLookupResult = CondenseInputToDictByPk(lookup, table, lookupPayloadSelector, ctx); - if (!condenseLookupResult) { - return {}; - } - - auto computeDictStage = Build(ctx, pos) - .Inputs() - .Add(condenseLookupResult->StageInputs) - .Build() - .Program() - .Args(condenseLookupResult->StageArgs) - .Body(condenseLookupResult->Stream) - .Build() - .Settings().Build() - .Done(); - - return Build(ctx, pos) - .Connection() - .Output() - .Stage(computeDictStage) - .Index().Build("0") - .Build() - .Build() - .Done(); + return PrecomputeTableLookupDict( + lookupKeys, table, lookupColumns, pos, ctx, false); } TCoAtomList BuildNonDefaultColumns( @@ -115,34 +64,15 @@ TCoAtomList BuildNonDefaultColumns( .Done(); } -TExprBase KqpRewriteGenerateIfInsert(TExprBase node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { - auto maybeInsertOnConlictUpdate = node.Maybe(); - if (!maybeInsertOnConlictUpdate) { - return node; - } - - auto insertOnConlictUpdate = maybeInsertOnConlictUpdate.Cast(); - YQL_ENSURE(insertOnConlictUpdate.GenerateColumnsIfInsert().Ref().ChildrenSize() > 0); - TCoAtomList columnsWithDefault = insertOnConlictUpdate.GenerateColumnsIfInsert(); - - auto input = insertOnConlictUpdate.Input(); - auto pos = insertOnConlictUpdate.Input().Pos(); - - const auto& tableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, insertOnConlictUpdate.Table().Path()); - - auto payloadSelector = MakeRowsPayloadSelector(insertOnConlictUpdate.Columns(), tableDesc, pos, ctx); - auto condenseResult = CondenseInputToDictByPk(input, tableDesc, payloadSelector, ctx); - if (!condenseResult) { - return node; - } - - auto inputDictAndKeys = PrecomputeDictAndKeys(*condenseResult, pos, ctx); - auto lookupDict = PrecomputeCurrentDefaultsForKeys(inputDictAndKeys.KeysPrecompute, columnsWithDefault, tableDesc, pos, ctx); - if (!lookupDict) { - return node; - } - - auto nonDefaultColumns = BuildNonDefaultColumns(tableDesc, insertOnConlictUpdate.Columns(), columnsWithDefault, pos, ctx); +TDqStage BuildInsertOnConflictUpdateInputStage( + const TKikimrTableDescription& table, + const TCoAtomList& upsertColumns, + const TCoAtomList& columnsWithDefault, + const TDictAndKeysResult& inputDictAndKeys, + const TDqPhyPrecompute& lookupDict, + TPositionHandle pos, TExprContext& ctx) +{ + auto nonDefaultColumns = BuildNonDefaultColumns(table, upsertColumns, columnsWithDefault, pos, ctx); auto inputKeysArg = TCoArgument(ctx.NewArgument(pos, "input_keys")); auto inputDictArg = TCoArgument(ctx.NewArgument(pos, "input_dict")); @@ -150,11 +80,11 @@ TExprBase KqpRewriteGenerateIfInsert(TExprBase node, TExprContext& ctx, const TK auto lookupDictArg = TCoArgument(ctx.NewArgument(pos, "lookup_dict")); auto presetHandlerPayload = TCoArgument(ctx.NewArgument(pos, "payload")); - auto filterStage = Build(ctx, pos) + return Build(ctx, pos) .Inputs() .Add(inputDictAndKeys.KeysPrecompute) .Add(inputDictAndKeys.DictPrecompute) - .Add(lookupDict.Cast()) + .Add(lookupDict) .Build() .Program() .Args({inputKeysArg, inputDictArg, lookupDictArg}) @@ -215,10 +145,38 @@ TExprBase KqpRewriteGenerateIfInsert(TExprBase node, TExprContext& ctx, const TK .Build() .Settings().Build() .Done(); +} + +TExprBase KqpRewriteGenerateIfInsert(TExprBase node, TExprContext& ctx, const TKqpOptimizeContext& kqpCtx) { + auto maybeInsertOnConlictUpdate = node.Maybe(); + if (!maybeInsertOnConlictUpdate) { + return node; + } + + auto insertOnConlictUpdate = maybeInsertOnConlictUpdate.Cast(); + YQL_ENSURE(insertOnConlictUpdate.GenerateColumnsIfInsert().Ref().ChildrenSize() > 0); + TCoAtomList columnsWithDefault = insertOnConlictUpdate.GenerateColumnsIfInsert(); + + auto input = insertOnConlictUpdate.Input(); + auto pos = insertOnConlictUpdate.Input().Pos(); + + const auto& tableDesc = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, insertOnConlictUpdate.Table().Path()); + + auto payloadSelector = MakeRowsPayloadSelector(insertOnConlictUpdate.Columns(), tableDesc, pos, ctx); + auto condenseResult = CondenseInputToDictByPk(input, tableDesc, payloadSelector, ctx); + if (!condenseResult) { + return node; + } + + auto inputDictAndKeys = PrecomputeDictAndKeys(*condenseResult, pos, ctx); + auto lookupDict = PrecomputeCurrentDefaultsForKeys(inputDictAndKeys.KeysPrecompute, columnsWithDefault, tableDesc, pos, ctx); + if (!lookupDict) { + return node; + } auto newInput = Build(ctx, pos) .Output() - .Stage(filterStage) + .Stage(BuildInsertOnConflictUpdateInputStage(tableDesc, insertOnConlictUpdate.Columns(), columnsWithDefault, inputDictAndKeys, lookupDict.Cast(), pos, ctx)) .Index().Build("0") .Build() .Done(); diff --git a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_index.cpp b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_index.cpp index c9e11842f47a..c868d065cef2 100644 --- a/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_index.cpp +++ b/ydb/core/kqp/opt/physical/effects/kqp_opt_phy_upsert_index.cpp @@ -344,7 +344,8 @@ TExprBase MakeUpsertIndexRows(TKqpPhyUpsertIndexMode mode, const TDqPhyPrecomput .Done(); } -TMaybe CheckUniqueConstraint(const TExprBase& inputRows, const THashSet inputColumns, +TMaybe RewriteInputForConstraint(const TExprBase& inputRows, const THashSet inputColumns, + const THashSet& checkDefaults, const TKikimrTableDescription& table, const TSecondaryIndexes& indexes, TPositionHandle pos, TExprContext& ctx) { @@ -379,28 +380,37 @@ TMaybe CheckUniqueConstraint(const TExprBase& inputRows, c } } - if (missedKeyInput && hasUniqIndex) { + if (!hasUniqIndex) { + missedKeyInput.clear(); + } + + if (!missedKeyInput.empty() || !checkDefaults.empty()) { TVector columns; TCoArgument inLambdaArg(ctx.NewArgument(pos, "in_lambda_arg")); - auto missedFromMain = TCoArgument(ctx.NewArgument(pos, "missed_from_main")); + auto lookupRowArgument = TCoArgument(ctx.NewArgument(pos, "lookup_row")); - TVector resCol; + TVector existingRow, nonExistingRow; + auto getterFromValue = [&ctx, &pos] (const TStringBuf& x, const TCoArgument& arg) -> TExprBase { + return Build(ctx, pos) + .Name().Build(x) + .Value() + .Struct(arg) + .Name().Build(x) + .Build() + .Done(); + }; for (const auto& x : inputColumns) { - if (!missedKeyInput.contains(x)) { - resCol.emplace_back( - Build(ctx, pos) - .Name().Build(x) - .Value() - .Struct(inLambdaArg) - .Name().Build(x) - .Build() - .Done()); + bool hasDefaultToCheck = checkDefaults.contains(x); + if (hasDefaultToCheck) { + existingRow.push_back(getterFromValue(x, lookupRowArgument)); + } else { + existingRow.emplace_back(getterFromValue(x, inLambdaArg)); } - } - TVector resNullCol = resCol; + nonExistingRow.push_back(getterFromValue(x, inLambdaArg)); + } for (const auto& x : missedKeyInput) { auto atom = Build(ctx, pos) @@ -411,16 +421,8 @@ TMaybe CheckUniqueConstraint(const TExprBase& inputRows, c auto columnType = table.GetColumnType(TString(x)); YQL_ENSURE(columnType); - resCol.emplace_back( - Build(ctx, pos) - .Name().Build(x) - .Value() - .Struct(missedFromMain) - .Name().Build(x) - .Build() - .Done()); - - resNullCol.emplace_back( + existingRow.emplace_back(getterFromValue(x, lookupRowArgument)); + nonExistingRow.emplace_back( Build(ctx, pos) .Name().Build(x) .Value() @@ -436,8 +438,11 @@ TMaybe CheckUniqueConstraint(const TExprBase& inputRows, c columns.emplace_back(atom); } - auto inPrecompute = PrecomputeCondenseInputResult(*condenseResult, pos, ctx); + for(const auto& x: checkDefaults) { + columns.push_back(Build(ctx, pos).Value(x).Done()); + } + auto inPrecompute = PrecomputeCondenseInputResult(*condenseResult, pos, ctx); auto precomputeTableLookupDict = PrecomputeTableLookupDict(inPrecompute, table, columns, pos, ctx, true); TVector keyLookupTuples; @@ -475,13 +480,13 @@ TMaybe CheckUniqueConstraint(const TExprBase& inputRows, c .Build() .Build() .PresentHandler() - .Args({missedFromMain}) + .Args({lookupRowArgument}) .Body() - .Add(resCol) + .Add(existingRow) .Build() .Build() .MissingValue() - .Add(resNullCol) + .Add(nonExistingRow) .Build() .Build() .Build() @@ -595,7 +600,7 @@ TMaybe CheckUniqueConstraint(const TExprBase& inputRows, c } // namespace TMaybeNode KqpPhyUpsertIndexEffectsImpl(TKqpPhyUpsertIndexMode mode, const TExprBase& inputRows, - const TCoAtomList& inputColumns, const TCoAtomList& returningColumns, const TKikimrTableDescription& table, + const TCoAtomList& inputColumns, const TCoAtomList& returningColumns, const TCoAtomList& columnsWithDefaults, const TKikimrTableDescription& table, const TMaybeNode& settings, TPositionHandle pos, TExprContext& ctx) { switch (mode) { @@ -614,10 +619,15 @@ TMaybeNode KqpPhyUpsertIndexEffectsImpl(TKqpPhyUpsertIndexMode mode, inputColumnsSet.emplace(column.Value()); } + THashSet columnsWithDefaultsSet; + for(const auto& column: columnsWithDefaults) { + columnsWithDefaultsSet.emplace(column.Value()); + } + auto filter = (mode == TKqpPhyUpsertIndexMode::UpdateOn) ? &inputColumnsSet : nullptr; const auto indexes = BuildSecondaryIndexVector(table, pos, ctx, filter); - auto checkedInput = CheckUniqueConstraint(inputRows, inputColumnsSet, table, indexes, pos, ctx); + auto checkedInput = RewriteInputForConstraint(inputRows, inputColumnsSet, columnsWithDefaultsSet, table, indexes, pos, ctx); if (!checkedInput) { return {}; @@ -901,7 +911,7 @@ TExprBase KqpBuildUpsertIndexStages(TExprBase node, TExprContext& ctx, const TKq const auto& table = kqpCtx.Tables->ExistingTable(kqpCtx.Cluster, upsert.Table().Path()); auto effects = KqpPhyUpsertIndexEffectsImpl(TKqpPhyUpsertIndexMode::Upsert, upsert.Input(), upsert.Columns(), - upsert.ReturningColumns(), table, upsert.Settings(), upsert.Pos(), ctx); + upsert.ReturningColumns(), upsert.GenerateColumnsIfInsert(), table, upsert.Settings(), upsert.Pos(), ctx); if (!effects) { return node; diff --git a/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp b/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp index 1ce97d75bba6..1b5bec6be4a8 100644 --- a/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp +++ b/ydb/core/kqp/opt/physical/kqp_opt_phy_olap_filter.cpp @@ -33,7 +33,7 @@ struct TPushdownSettings : public NPushdown::TSettings { Enable(EFlag::LikeOperator, NSsa::RuntimeVersion >= 2U); Enable(EFlag::LikeOperatorOnlyForUtf8, NSsa::RuntimeVersion < 3U); Enable(EFlag::JsonQueryOperators | EFlag::JsonExistsOperator, NSsa::RuntimeVersion >= 3U); - Enable(EFlag::ArithmeticalExpressions, NSsa::RuntimeVersion >= 4U); + Enable(EFlag::ArithmeticalExpressions | EFlag::UnaryOperators, NSsa::RuntimeVersion >= 4U); Enable(EFlag::LogicalXorOperator | EFlag::ParameterExpression | EFlag::CastExpression @@ -100,8 +100,43 @@ bool IsFalseLiteral(TExprBase node) { return node.Maybe() && !FromString(node.Cast().Literal().Value()); } -std::optional> ExtractBinaryFunctionParameters(const TExprBase& op, TExprContext& ctx, TPositionHandle pos); -std::vector> ExtractComparisonParameters(const TCoCompare& predicate, TExprContext& ctx, TPositionHandle pos); +std::vector ConvertComparisonNode(const TExprBase& nodeIn, TExprContext& ctx, TPositionHandle pos); + +std::optional> ExtractBinaryFunctionParameters(const TExprBase& op, TExprContext& ctx, TPositionHandle pos) +{ + const auto left = ConvertComparisonNode(TExprBase(op.Ref().HeadPtr()), ctx, pos); + if (left.size() != 1U) { + return std::nullopt; + } + + const auto right = ConvertComparisonNode(TExprBase(op.Ref().TailPtr()), ctx, pos); + if (right.size() != 1U) { + return std::nullopt; + } + + return std::make_pair(left.front(), right.front()); +} + +std::vector> ExtractComparisonParameters(const TCoCompare& predicate, TExprContext& ctx, TPositionHandle pos) +{ + std::vector> out; + auto left = ConvertComparisonNode(predicate.Left(), ctx, pos); + + if (left.empty()) { + return out; + } + + auto right = ConvertComparisonNode(predicate.Right(), ctx, pos); + if (left.size() != right.size()) { + return out; + } + + for (ui32 i = 0; i < left.size(); ++i) { + out.emplace_back(std::move(std::make_pair(left[i], right[i]))); + } + + return out; +} TMaybeNode ComparisonPushdown(const std::vector>& parameters, const TCoCompare& predicate, TExprContext& ctx, TPositionHandle pos); @@ -117,9 +152,13 @@ TMaybeNode YqlCoalescePushdown(const TCoCoalesce& coalesce, TExprCont return NullNode; } -TVector ConvertComparisonNode(const TExprBase& nodeIn, TExprContext& ctx, TPositionHandle pos) +bool IsGoodTypeForPushdown(const TTypeAnnotationNode& type) { + return NUdf::EDataTypeFeatures::IntegralType & NUdf::GetDataTypeInfo(RemoveOptionality(type).Cast()->GetSlot()).Features; +} + +std::vector ConvertComparisonNode(const TExprBase& nodeIn, TExprContext& ctx, TPositionHandle pos) { - TVector out; + std::vector out; const auto convertNode = [&ctx, &pos](const TExprBase& node) -> TMaybeNode { if (node.Maybe()) { return node; @@ -162,20 +201,35 @@ TVector ConvertComparisonNode(const TExprBase& nodeIn, TExprContext& return builder.Done(); } - if (const auto maybeArithmetic = node.Maybe()) { - if (const auto arithmetic = maybeArithmetic.Cast(); !arithmetic.Maybe()) { - if (const auto params = ExtractBinaryFunctionParameters(arithmetic, ctx, pos)) { - return Build(ctx, pos) - .Operator().Value(arithmetic.Ref().Content(), TNodeFlags::Default).Build() - .Left(params->first) - .Right(params->second) - .Done(); + if constexpr (NKikimr::NSsa::RuntimeVersion >= 4U) { + if (const auto maybeArithmetic = node.Maybe()) { + if (const auto arithmetic = maybeArithmetic.Cast(); IsGoodTypeForPushdown(*arithmetic.Ref().GetTypeAnn()) && !arithmetic.Maybe()) { + if (const auto params = ExtractBinaryFunctionParameters(arithmetic, ctx, pos)) { + return Build(ctx, pos) + .Operator().Value(arithmetic.Ref().Content(), TNodeFlags::Default).Build() + .Left(params->first) + .Right(params->second) + .Done(); + } } } - } - if (const auto maybeCoalesce = node.Maybe()) { - return YqlCoalescePushdown(maybeCoalesce.Cast(), ctx); + if (const auto maybeArithmetic = node.Maybe()) { + if (const auto arithmetic = maybeArithmetic.Cast(); IsGoodTypeForPushdown(*arithmetic.Ref().GetTypeAnn())) { + if (const auto params = ConvertComparisonNode(arithmetic.Arg(), ctx, pos); 1U == params.size()) { + TString oper(arithmetic.Ref().Content()); + YQL_ENSURE(oper.to_lower()); + return Build(ctx, pos) + .Operator().Value(oper, TNodeFlags::Default).Build() + .Arg(params.front()) + .Done(); + } + } + } + + if (const auto maybeCoalesce = node.Maybe()) { + return YqlCoalescePushdown(maybeCoalesce.Cast(), ctx); + } } if (const auto maybeCompare = node.Maybe()) { @@ -219,42 +273,6 @@ TVector ConvertComparisonNode(const TExprBase& nodeIn, TExprContext& return out; } -std::optional> ExtractBinaryFunctionParameters(const TExprBase& op, TExprContext& ctx, TPositionHandle pos) -{ - const auto left = ConvertComparisonNode(TExprBase(op.Ref().HeadPtr()), ctx, pos); - if (left.size() != 1U) { - return std::nullopt; - } - - const auto right = ConvertComparisonNode(TExprBase(op.Ref().TailPtr()), ctx, pos); - if (right.size() != 1U) { - return std::nullopt; - } - - return std::make_pair(left.front(), right.front()); -} - -std::vector> ExtractComparisonParameters(const TCoCompare& predicate, TExprContext& ctx, TPositionHandle pos) -{ - std::vector> out; - auto left = ConvertComparisonNode(predicate.Left(), ctx, pos); - - if (left.empty()) { - return out; - } - - auto right = ConvertComparisonNode(predicate.Right(), ctx, pos); - if (left.size() != right.size()) { - return out; - } - - for (ui32 i = 0; i < left.size(); ++i) { - out.emplace_back(std::move(std::make_pair(left[i], right[i]))); - } - - return out; -} - TExprBase BuildOneElementComparison(const std::pair& parameter, const TCoCompare& predicate, TExprContext& ctx, TPositionHandle pos, bool forceStrictComparison) { @@ -425,13 +443,20 @@ TMaybeNode BuildColumnsFromLambda(const TCoLambda& lambda, TExprCon } #endif +template TMaybeNode ExistsPushdown(const TCoExists& exists, TExprContext& ctx, TPositionHandle pos) { - auto columnName = exists.Optional().Cast().Name(); - - return Build(ctx, pos) - .Column(columnName) - .Done(); + const auto columnName = exists.Optional().Cast().Name(); + if constexpr (NSsa::RuntimeVersion >= 4U) { + return Build(ctx, pos) + .Operator().Value(Empty ? "empty" : "exists", TNodeFlags::Default).Build() + .Arg(columnName) + .Done(); + } else { + return Build(ctx, pos) + .Column(columnName) + .Done(); + } } TMaybeNode JsonExistsPushdown(const TCoJsonExists& jsonExists, TExprContext& ctx, TPositionHandle pos) @@ -505,32 +530,33 @@ TMaybeNode CoalescePushdown(const TCoCoalesce& coalesce, TExprContext TFilterOpsLevels PredicatePushdown(const TExprBase& predicate, TExprContext& ctx, TPositionHandle pos) { - auto maybeCoalesce = predicate.Maybe(); - if (maybeCoalesce.IsValid()) { + if (const auto maybeCoalesce = predicate.Maybe()) { auto coalescePred = CoalescePushdown(maybeCoalesce.Cast(), ctx, pos); return TFilterOpsLevels(coalescePred); } - auto maybeExists = predicate.Maybe(); - if (maybeExists.IsValid()) { - auto existsPred = ExistsPushdown(maybeExists.Cast(), ctx, pos); + if (const auto maybeExists = predicate.Maybe()) { + auto existsPred = ExistsPushdown(maybeExists.Cast(), ctx, pos); return TFilterOpsLevels(existsPred); } - auto maybeJsonExists = predicate.Maybe(); - if (maybeJsonExists.IsValid()) { + if (const auto maybeJsonExists = predicate.Maybe()) { auto jsonExistsPred = JsonExistsPushdown(maybeJsonExists.Cast(), ctx, pos); return TFilterOpsLevels(jsonExistsPred); } - auto maybePredicate = predicate.Maybe(); - if (maybePredicate.IsValid()) { + if (const auto maybePredicate = predicate.Maybe()) { auto pred = SimplePredicatePushdown(maybePredicate.Cast(), ctx, pos); return TFilterOpsLevels(pred); } - if (predicate.Maybe()) { - auto notNode = predicate.Cast(); + if (const auto maybeNot = predicate.Maybe()) { + const auto notNode = maybeNot.Cast(); + if constexpr (NSsa::RuntimeVersion >= 4U) { + if (const auto maybeExists = notNode.Value().Maybe()) { + return TFilterOpsLevels(ExistsPushdown(maybeExists.Cast(), ctx, pos)); + } + } auto pushedFilters = PredicatePushdown(notNode.Value(), ctx, pos); pushedFilters.WrapToNotOp(ctx, pos); return pushedFilters; diff --git a/ydb/core/kqp/query_compiler/kqp_olap_compiler.cpp b/ydb/core/kqp/query_compiler/kqp_olap_compiler.cpp index 51b75b7349ed..0eecc157fd75 100644 --- a/ydb/core/kqp/query_compiler/kqp_olap_compiler.cpp +++ b/ydb/core/kqp/query_compiler/kqp_olap_compiler.cpp @@ -94,6 +94,20 @@ class TKqpOlapCompileContext { return Program.AddCommand()->MutableProjection(); } + const TTypeAnnotationNode* GetReturnType(const TTypeAnnotationNode& argument, const TTypeAnnotationNode* resultItemType) const { + bool isScalar; + const auto itemType = GetBlockItemType(argument, isScalar); + + if (ETypeAnnotationKind::Optional == itemType->GetKind()) { + resultItemType = ExprContext.MakeType(resultItemType); + } + + if (isScalar) + return ExprContext.MakeType(resultItemType); + else + return ExprContext.MakeType(resultItemType); + } + const TTypeAnnotationNode* GetReturnType(const TTypeAnnotationNode& left, const TTypeAnnotationNode& right, const TTypeAnnotationNode* resultItemType) const { bool isScalarLeft, isScalarRight; const auto leftItemType = GetBlockItemType(left, isScalarLeft); @@ -174,15 +188,8 @@ class TKqpOlapCompileContext { return ExprContext; } - bool CheckYqlCompatibleArgsTypes(const TKqpOlapFilterBinaryOp& operation) const { - if (const auto maybe = operation.Left().Maybe()) { - if (const auto type = GetColumnTypeByName(maybe.Cast().Value()); type->GetKind() == ETypeAnnotationKind::Data) { - if (const auto info = GetDataTypeInfo(type->Cast()->GetSlot()); !(info.Features & (NUdf::EDataTypeFeatures::StringType | NUdf::EDataTypeFeatures::NumericType))) { - return false; - } - } - } - if (const auto maybe = operation.Right().Maybe()) { + bool CheckYqlCompatibleArgType(const TExprBase& expression) const { + if (const auto maybe = expression.Maybe()) { if (const auto type = GetColumnTypeByName(maybe.Cast().Value()); type->GetKind() == ETypeAnnotationKind::Data) { if (const auto info = GetDataTypeInfo(type->Cast()->GetSlot()); !(info.Features & (NUdf::EDataTypeFeatures::StringType | NUdf::EDataTypeFeatures::NumericType))) { return false; @@ -192,6 +199,10 @@ class TKqpOlapCompileContext { return true; } + bool CheckYqlCompatibleArgsTypes(const TKqpOlapFilterBinaryOp& operation) const { + return CheckYqlCompatibleArgType(operation.Left()) && CheckYqlCompatibleArgType(operation.Right()); + } + TKernelRequestBuilder& GetKernelRequestBuilder() const { return *YqlKernelRequestBuilder; } @@ -547,8 +558,69 @@ ui64 CompileYqlKernelComparison(const TKqpOlapFilterBinaryOp& comparison, TKqpOl return command->GetColumn().GetId(); } + + +const TProgram::TAssignment* InvertResult(TProgram::TAssignment* command, TKqpOlapCompileContext& ctx) +{ + auto *const notCommand = ctx.CreateAssignCmd(); + auto *const notFunc = notCommand->MutableFunction(); + + notFunc->SetId(TProgram::TAssignment::FUNC_BINARY_NOT); + notFunc->AddArguments()->SetId(command->GetColumn().GetId()); + return notCommand; +} + TTypedColumn GetOrCreateColumnIdAndType(const TExprBase& node, TKqpOlapCompileContext& ctx); +template +const TTypedColumn CompileExists(const TExprBase& arg, TKqpOlapCompileContext& ctx) +{ + const auto type = ctx.ExprCtx().MakeType(EDataSlot::Bool); + const auto column = GetOrCreateColumnIdAndType(arg, ctx); + auto *const command = ctx.CreateAssignCmd(); + auto *const isNullFunc = command->MutableFunction(); + + isNullFunc->SetId(TProgram::TAssignment::FUNC_IS_NULL); + isNullFunc->AddArguments()->SetId(column.Id); + + if constexpr (Empty) { + if constexpr (NSsa::RuntimeVersion >= 4U) { + return {ConvertSafeCastToColumn(command->GetColumn().GetId(), "Uint8", ctx), ctx.ConvertToBlockType(type)}; + } else { + return {command->GetColumn().GetId(), type}; + } + } + + auto *const notCommand = InvertResult(command, ctx); + if constexpr (NSsa::RuntimeVersion >= 4U) { + return {ConvertSafeCastToColumn(notCommand->GetColumn().GetId(), "Uint8", ctx), ctx.ConvertToBlockType(type)}; + } else { + return {notCommand->GetColumn().GetId(), type}; + } +} + +TTypedColumn CompileYqlKernelUnaryOperation(const TKqpOlapFilterUnaryOp& operation, TKqpOlapCompileContext& ctx) +{ + auto oper = operation.Operator().StringValue(); + if (oper == "exists"sv) { + return CompileExists(operation.Arg(), ctx); + } else if (oper == "empty"sv) { + return CompileExists(operation.Arg(), ctx); + } + + const auto argument = GetOrCreateColumnIdAndType(operation.Arg(), ctx); + auto *const command = ctx.CreateAssignCmd(); + auto *const function = command->MutableFunction(); + YQL_ENSURE(oper.to_title()); + const auto op = FromString(oper); + const auto resultType = TKernelRequestBuilder::EUnaryOp::Size == op ? ctx.GetReturnType(*argument.Type, ctx.ExprCtx().MakeType(EDataSlot::Uint32)) : argument.Type; + const auto idx = ctx.GetKernelRequestBuilder().AddUnaryOp(op, argument.Type, resultType); + function->AddArguments()->SetId(argument.Id); + function->SetKernelIdx(idx); + function->SetFunctionType(TProgram::YQL_KERNEL); + return {command->GetColumn().GetId(), resultType}; +} + TTypedColumn CompileYqlKernelBinaryOperation(const TKqpOlapFilterBinaryOp& operation, TKqpOlapCompileContext& ctx) { // Columns should be created before operation, otherwise operation fail to find columns @@ -641,46 +713,9 @@ const TTypedColumn BuildLogicalProgram(const TExprNode::TChildrenType& args, con return {logicalOp->GetColumn().GetId(), block}; } -const TProgram::TAssignment* InvertResult(TProgram::TAssignment* command, TKqpOlapCompileContext& ctx) -{ - auto *const notCommand = ctx.CreateAssignCmd(); - auto *const notFunc = notCommand->MutableFunction(); - - notFunc->SetId(TProgram::TAssignment::FUNC_BINARY_NOT); - notFunc->AddArguments()->SetId(command->GetColumn().GetId()); - return notCommand; -} - -template -const TTypedColumn CompileExists(const TKqpOlapFilterExists& exists, TKqpOlapCompileContext& ctx) -{ - const auto type = ctx.ExprCtx().MakeType(EDataSlot::Bool); - const auto columnId = GetOrCreateColumnId(exists.Column(), ctx); - auto *const command = ctx.CreateAssignCmd(); - auto *const isNullFunc = command->MutableFunction(); - - isNullFunc->SetId(TProgram::TAssignment::FUNC_IS_NULL); - isNullFunc->AddArguments()->SetId(columnId); - - if constexpr (Empty) { - if constexpr (NSsa::RuntimeVersion >= 4U) { - return {ConvertSafeCastToColumn(command->GetColumn().GetId(), "Uint8", ctx), ctx.ConvertToBlockType(type)}; - } else { - return {command->GetColumn().GetId(), type}; - } - } - - auto *const notCommand = InvertResult(command, ctx); - if constexpr (NSsa::RuntimeVersion >= 4U) { - return {ConvertSafeCastToColumn(notCommand->GetColumn().GetId(), "Uint8", ctx), ctx.ConvertToBlockType(type)}; - } else { - return {notCommand->GetColumn().GetId(), type}; - } -} - const TTypedColumn BuildLogicalNot(const TExprBase& arg, TKqpOlapCompileContext& ctx) { if (const auto maybeExists = arg.Maybe()) { - return CompileExists(maybeExists.Cast(), ctx); + return CompileExists(maybeExists.Cast().Column(), ctx); } // Not is a special way in case it has only one child @@ -711,6 +746,8 @@ TTypedColumn GetOrCreateColumnIdAndType(const TExprBase& node, TKqpOlapCompileCo ctx.ExprCtx().MakeType(ctx.ExprCtx().MakeType(EDataSlot::Uint8)) }; } + } else if (const auto& maybeUnaryOp = node.Maybe()) { + return CompileYqlKernelUnaryOperation(maybeUnaryOp.Cast(), ctx); } else if (const auto& maybeAnd = node.Maybe()) { return BuildLogicalProgram(maybeAnd.Ref().Children(), TKernelRequestBuilder::EBinaryOp::And, ctx); } else if (const auto& maybeOr = node.Maybe()) { @@ -747,12 +784,18 @@ ui64 CompileComparison(const TKqpOlapFilterBinaryOp& comparison, TKqpOlapCompile } ui64 CompileCondition(const TExprBase& condition, TKqpOlapCompileContext& ctx) { + if constexpr (NKikimr::NSsa::RuntimeVersion >= 4U) { + if (const auto maybeCompare = condition.Maybe()) { + return CompileYqlKernelUnaryOperation(maybeCompare.Cast(), ctx).Id; + } + } + if (const auto maybeCompare = condition.Maybe()) { return CompileComparison(maybeCompare.Cast(), ctx); } if (const auto maybeExists = condition.Maybe()) { - return CompileExists(maybeExists.Cast(), ctx).Id; + return CompileExists(maybeExists.Cast().Column(), ctx).Id; } if (const auto maybeJsonExists = condition.Maybe()) { diff --git a/ydb/core/kqp/ut/common/columnshard.cpp b/ydb/core/kqp/ut/common/columnshard.cpp index 212d49076d02..99a10e26ecad 100644 --- a/ydb/core/kqp/ut/common/columnshard.cpp +++ b/ydb/core/kqp/ut/common/columnshard.cpp @@ -9,7 +9,6 @@ namespace NKqp { TTestHelper::TTestHelper(const TKikimrSettings& settings) : Kikimr(settings) , TableClient(Kikimr.GetTableClient()) - , LongTxClient(Kikimr.GetDriver()) , Session(TableClient.CreateSession().GetValueSync().GetSession()) {} @@ -31,26 +30,6 @@ namespace NKqp { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); } - void TTestHelper::InsertData(const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const std::function onBeforeCommit /*= {}*/, const EStatus opStatus /*= EStatus::SUCCESS*/) { - NLongTx::TLongTxBeginResult resBeginTx = LongTxClient.BeginWriteTx().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resBeginTx.Status().GetStatus(), EStatus::SUCCESS, resBeginTx.Status().GetIssues().ToString()); - - auto txId = resBeginTx.GetResult().tx_id(); - auto batch = updates.BuildArrow(); - TString data = NArrow::NSerialization::TFullDataSerializer(arrow::ipc::IpcWriteOptions::Defaults()).Serialize(batch); - - NLongTx::TLongTxWriteResult resWrite = - LongTxClient.Write(txId, table.GetName(), txId, data, Ydb::LongTx::Data::APACHE_ARROW).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resWrite.Status().GetStatus(), opStatus, resWrite.Status().GetIssues().ToString()); - - if (onBeforeCommit) { - onBeforeCommit(); - } - - NLongTx::TLongTxCommitResult resCommitTx = LongTxClient.CommitTx(txId).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resCommitTx.Status().GetStatus(), EStatus::SUCCESS, resCommitTx.Status().GetIssues().ToString()); - } - void TTestHelper::BulkUpsert(const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const Ydb::StatusIds_StatusCode& opStatus /*= Ydb::StatusIds::SUCCESS*/) { Y_UNUSED(opStatus); NKikimr::Tests::NCS::THelper helper(Kikimr.GetTestServer()); diff --git a/ydb/core/kqp/ut/common/columnshard.h b/ydb/core/kqp/ut/common/columnshard.h index 92b45eb60759..adc2ea16989a 100644 --- a/ydb/core/kqp/ut/common/columnshard.h +++ b/ydb/core/kqp/ut/common/columnshard.h @@ -3,7 +3,6 @@ #include "kqp_ut_common.h" #include #include -#include #include #include #include @@ -64,7 +63,6 @@ namespace NKqp { private: TKikimrRunner Kikimr; NYdb::NTable::TTableClient TableClient; - NYdb::NLongTx::TClient LongTxClient; NYdb::NTable::TSession Session; public: @@ -73,7 +71,6 @@ namespace NKqp { TTestActorRuntime& GetRuntime(); NYdb::NTable::TSession& GetSession(); void CreateTable(const TColumnTableBase& table); - void InsertData(const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const std::function onBeforeCommit = {}, const NYdb::EStatus opStatus = NYdb::EStatus::SUCCESS); void BulkUpsert(const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const Ydb::StatusIds_StatusCode& opStatus = Ydb::StatusIds::SUCCESS); void BulkUpsert(const TColumnTable& table, std::shared_ptr batch, const Ydb::StatusIds_StatusCode& opStatus = Ydb::StatusIds::SUCCESS); void ReadData(const TString& query, const TString& expected, const NYdb::EStatus opStatus = NYdb::EStatus::SUCCESS); diff --git a/ydb/core/kqp/ut/federated_query/generic/kqp_generic_plan_ut.cpp b/ydb/core/kqp/ut/federated_query/generic/kqp_generic_plan_ut.cpp index f32177a2f377..e849c7251c04 100644 --- a/ydb/core/kqp/ut/federated_query/generic/kqp_generic_plan_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/generic/kqp_generic_plan_ut.cpp @@ -84,14 +84,14 @@ Y_UNIT_TEST_SUITE(KqpGenericPlanTest) { NJson::TJsonValue plan; UNIT_ASSERT(NJson::ReadJsonTree(*queryResult.GetStats()->GetPlan(), &plan)); - const auto& stagePlan = plan["Plan"]["Plans"][0]["Plans"][0]["Plans"][0]["Plans"][0]; - UNIT_ASSERT_VALUES_EQUAL(stagePlan["Node Type"].GetStringSafe(), "Filter-Source"); - const auto& sourceOp = stagePlan["Operators"].GetArraySafe()[1]; + const auto& stagePlan = plan["Plan"]["Plans"][0]["Plans"][0]["Plans"][0]["Plans"][0]["Plans"][0]; + UNIT_ASSERT_VALUES_EQUAL(stagePlan["Node Type"].GetStringSafe(), "Source"); + const auto& sourceOp = stagePlan["Operators"].GetArraySafe()[0]; UNIT_ASSERT_VALUES_EQUAL(sourceOp["ExternalDataSource"].GetStringSafe(), "pg_data_source"); UNIT_ASSERT_VALUES_EQUAL(sourceOp["Database"].GetStringSafe(), GetPgDatabase()); UNIT_ASSERT_VALUES_EQUAL(sourceOp["Protocol"].GetStringSafe(), "Native"); UNIT_ASSERT_VALUES_EQUAL(sourceOp["Table"].GetStringSafe(), "pg_table_plan_test"); - UNIT_ASSERT_VALUES_EQUAL(sourceOp["Name"].GetStringSafe(), "Read from external data source"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Name"].GetStringSafe(), "Read pg_data_source"); UNIT_ASSERT_VALUES_EQUAL(sourceOp["SourceType"].GetStringSafe(), "PostgreSql"); UNIT_ASSERT_VALUES_EQUAL(sourceOp["ReadColumns"].GetArraySafe()[0].GetStringSafe(), "key"); UNIT_ASSERT_VALUES_EQUAL(sourceOp["ReadColumns"].GetArraySafe()[1].GetStringSafe(), "name"); @@ -163,14 +163,14 @@ Y_UNIT_TEST_SUITE(KqpGenericPlanTest) { NJson::TJsonValue plan; UNIT_ASSERT(NJson::ReadJsonTree(*queryResult.GetStats()->GetPlan(), &plan)); - const auto& stagePlan = plan["Plan"]["Plans"][0]["Plans"][0]["Plans"][0]["Plans"][0]; - UNIT_ASSERT_VALUES_EQUAL(stagePlan["Node Type"].GetStringSafe(), "Filter-Source"); - const auto& sourceOp = stagePlan["Operators"].GetArraySafe()[1]; + const auto& stagePlan = plan["Plan"]["Plans"][0]["Plans"][0]["Plans"][0]["Plans"][0]["Plans"][0]; + UNIT_ASSERT_VALUES_EQUAL(stagePlan["Node Type"].GetStringSafe(), "Source"); + const auto& sourceOp = stagePlan["Operators"].GetArraySafe()[0]; UNIT_ASSERT_VALUES_EQUAL(sourceOp["ExternalDataSource"].GetStringSafe(), "ch_data_source"); UNIT_ASSERT_VALUES_EQUAL(sourceOp["Database"].GetStringSafe(), GetChDatabase()); UNIT_ASSERT_VALUES_EQUAL(sourceOp["Protocol"].GetStringSafe(), "Native"); UNIT_ASSERT_VALUES_EQUAL(sourceOp["Table"].GetStringSafe(), "ch_table_plan_test"); - UNIT_ASSERT_VALUES_EQUAL(sourceOp["Name"].GetStringSafe(), "Read from external data source"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Name"].GetStringSafe(), "Read ch_data_source"); UNIT_ASSERT_VALUES_EQUAL(sourceOp["SourceType"].GetStringSafe(), "ClickHouse"); UNIT_ASSERT_VALUES_EQUAL(sourceOp["ReadColumns"].GetArraySafe()[0].GetStringSafe(), "key"); UNIT_ASSERT_VALUES_EQUAL(sourceOp["ReadColumns"].GetArraySafe()[1].GetStringSafe(), "name"); diff --git a/ydb/core/kqp/ut/federated_query/s3/kqp_s3_plan_ut.cpp b/ydb/core/kqp/ut/federated_query/s3/kqp_s3_plan_ut.cpp index c29075f12c73..7ee61daae5b2 100644 --- a/ydb/core/kqp/ut/federated_query/s3/kqp_s3_plan_ut.cpp +++ b/ydb/core/kqp/ut/federated_query/s3/kqp_s3_plan_ut.cpp @@ -70,7 +70,7 @@ Y_UNIT_TEST_SUITE(KqpS3PlanTest) { const auto& sourceOp = stagePlan["Plans"][0]["Operators"].GetArraySafe()[0]; UNIT_ASSERT_VALUES_EQUAL(sourceOp["ExternalDataSource"].GetStringSafe(), "external_data_source"); UNIT_ASSERT_VALUES_EQUAL(sourceOp["Format"].GetStringSafe(), "json_each_row"); - UNIT_ASSERT_VALUES_EQUAL(sourceOp["Name"].GetStringSafe(), "Parse from external data source"); + UNIT_ASSERT_VALUES_EQUAL(sourceOp["Name"].GetStringSafe(), "Parse external_data_source"); UNIT_ASSERT_VALUES_EQUAL(sourceOp["SourceType"].GetStringSafe(), "s3"); UNIT_ASSERT(!IsIn(sourceOp.GetMap(), "RowsLimitHint")); UNIT_ASSERT_VALUES_EQUAL(sourceOp["ReadColumns"].GetArraySafe()[0].GetStringSafe(), "key"); diff --git a/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp index 2d34fb23496a..04496966fe66 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_stats_ut.cpp @@ -48,7 +48,7 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) { tableInserter.AddRow().Add(i).Add("test_res_" + std::to_string(i)).AddNull(); } - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } Sleep(TDuration::Seconds(1)); @@ -88,7 +88,7 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) { for (size_t i = 0; i < inserted_rows; i++) { tableInserter.AddRow().Add(i).Add("test_res_" + std::to_string(i)).AddNull(); } - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } Sleep(TDuration::Seconds(1)); @@ -135,7 +135,7 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) { .Add("test_res_" + std::to_string(i + t * tables_in_store)) .AddNull(); } - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } Sleep(TDuration::Seconds(20)); @@ -152,7 +152,52 @@ Y_UNIT_TEST_SUITE(KqpOlapStats) { UNIT_ASSERT_VALUES_EQUAL(t + inserted_rows, description.GetTableRows()); } } + + Y_UNIT_TEST(DescibeTableStore) { + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + + TKikimrSettings runnerSettings; + runnerSettings.WithSampleTables = false; + + TTestHelper testHelper(runnerSettings); + + TTestHelper::TColumnTableStore testTableStore; + + testTableStore.SetName("/Root/TableStoreTest").SetPrimaryKey({"id"}).SetSchema(schema); + testHelper.CreateTable(testTableStore); + + Tests::NCommon::TLoggerInit(testHelper.GetKikimr()).SetPriority(NActors::NLog::PRI_DEBUG).Initialize(); + + for (size_t t = 0; t < 2; t++) { + TTestHelper::TColumnTable testTable; + testTable.SetName("/Root/TableStoreTest/ColumnTableTest_" + std::to_string(t)) + .SetPrimaryKey({"id"}) + .SetSharding({"id"}) + .SetSchema(schema); + testHelper.CreateTable(testTable); + + TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); + for (size_t i = 0; i < inserted_rows; i++) { + tableInserter.AddRow() + .Add(i + t * tables_in_store) + .Add("test_res_" + std::to_string(i + t * tables_in_store)) + .AddNull(); + } + testHelper.InsertData(testTable, tableInserter); + } + + Sleep(TDuration::Seconds(10)); + + auto settings = TDescribeTableSettings().WithTableStatistics(true); + + auto describeStoreResult = + testHelper.GetSession().DescribeTable("/Root/TableStoreTest/", settings).GetValueSync(); + UNIT_ASSERT_C(describeStoreResult.IsSuccess(), describeStoreResult.GetIssues().ToString()); + const auto& storeDescription = describeStoreResult.GetTableDescription(); + + UNIT_ASSERT_VALUES_EQUAL(2000, storeDescription.GetTableRows()); + } } } // namespace NKqp -} // namespace NKikimr \ No newline at end of file +} // namespace NKikimr diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index a91b40588cf0..12408595b220 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -534,67 +533,25 @@ Y_UNIT_TEST_SUITE(KqpOlap) { void WriteTestData(TKikimrRunner& kikimr, TString testTable, ui64 pathIdBegin, ui64 tsBegin, size_t rowCount, bool withSomeNulls = false) { UNIT_ASSERT(testTable != "/Root/benchTable"); // TODO: check schema instead - TLocalHelper lHelper(kikimr); if (withSomeNulls) lHelper.WithSomeNulls(); - NYdb::NLongTx::TClient client(kikimr.GetDriver()); - - NLongTx::TLongTxBeginResult resBeginTx = client.BeginWriteTx().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resBeginTx.Status().GetStatus(), EStatus::SUCCESS, resBeginTx.Status().GetIssues().ToString()); - - auto txId = resBeginTx.GetResult().tx_id(); auto batch = lHelper.TestArrowBatch(pathIdBegin, tsBegin, rowCount); - - TString data = NArrow::NSerialization::TFullDataSerializer(arrow::ipc::IpcWriteOptions::Defaults()).Serialize(batch); - - NLongTx::TLongTxWriteResult resWrite = - client.Write(txId, testTable, txId, data, Ydb::LongTx::Data::APACHE_ARROW).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resWrite.Status().GetStatus(), EStatus::SUCCESS, resWrite.Status().GetIssues().ToString()); - - NLongTx::TLongTxCommitResult resCommitTx = client.CommitTx(txId).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resCommitTx.Status().GetStatus(), EStatus::SUCCESS, resCommitTx.Status().GetIssues().ToString()); + lHelper.SendDataViaActorSystem(testTable, batch); } void WriteTestDataForClickBench(TKikimrRunner& kikimr, TString testTable, ui64 pathIdBegin, ui64 tsBegin, size_t rowCount) { UNIT_ASSERT(testTable == "/Root/benchTable"); // TODO: check schema instead - TClickHelper lHelper(kikimr.GetTestServer()); - NYdb::NLongTx::TClient client(kikimr.GetDriver()); - - NLongTx::TLongTxBeginResult resBeginTx = client.BeginWriteTx().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resBeginTx.Status().GetStatus(), EStatus::SUCCESS, resBeginTx.Status().GetIssues().ToString()); - - auto txId = resBeginTx.GetResult().tx_id(); auto batch = lHelper.TestArrowBatch(pathIdBegin, tsBegin, rowCount); - TString data = NArrow::NSerialization::TFullDataSerializer(arrow::ipc::IpcWriteOptions::Defaults()).Serialize(batch); - - NLongTx::TLongTxWriteResult resWrite = - client.Write(txId, testTable, txId, data, Ydb::LongTx::Data::APACHE_ARROW).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resWrite.Status().GetStatus(), EStatus::SUCCESS, resWrite.Status().GetIssues().ToString()); - - NLongTx::TLongTxCommitResult resCommitTx = client.CommitTx(txId).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resCommitTx.Status().GetStatus(), EStatus::SUCCESS, resCommitTx.Status().GetIssues().ToString()); + lHelper.SendDataViaActorSystem(testTable, batch); } void WriteTestDataForTableWithNulls(TKikimrRunner& kikimr, TString testTable) { UNIT_ASSERT(testTable == "/Root/tableWithNulls"); // TODO: check schema instead TTableWithNullsHelper lHelper(kikimr.GetTestServer()); - NYdb::NLongTx::TClient client(kikimr.GetDriver()); - - NLongTx::TLongTxBeginResult resBeginTx = client.BeginWriteTx().GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resBeginTx.Status().GetStatus(), EStatus::SUCCESS, resBeginTx.Status().GetIssues().ToString()); - - auto txId = resBeginTx.GetResult().tx_id(); auto batch = lHelper.TestArrowBatch(); - TString data = NArrow::NSerialization::TFullDataSerializer(arrow::ipc::IpcWriteOptions::Defaults()).Serialize(batch); - - NLongTx::TLongTxWriteResult resWrite = - client.Write(txId, testTable, txId, data, Ydb::LongTx::Data::APACHE_ARROW).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resWrite.Status().GetStatus(), EStatus::SUCCESS, resWrite.Status().GetIssues().ToString()); - - NLongTx::TLongTxCommitResult resCommitTx = client.CommitTx(txId).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(resCommitTx.Status().GetStatus(), EStatus::SUCCESS, resCommitTx.Status().GetIssues().ToString()); + lHelper.SendDataViaActorSystem(testTable, batch); } void CreateTableOfAllTypes(TKikimrRunner& kikimr) { @@ -1635,6 +1592,20 @@ Y_UNIT_TEST_SUITE(KqpOlap) { R"(`level` * 3 > 4)", R"(`level` / 2 <= 1)", R"(`level` % 3 != 1)", + R"(-`level` < -2)", + R"(Abs(`level` - 3) >= 1)", + R"(LENGTH(`message`) > 1037U)", + R"(LENGTH(`uid`) > 1U OR `resource_id` = "10001")", + R"((LENGTH(`uid`) > 2U AND `resource_id` = "10001") OR `resource_id` = "10002")", + R"((LENGTH(`uid`) > 3U OR `resource_id` = "10002") AND (LENGTH(`uid`) < 15 OR `resource_id` = "10001"))", + R"(NOT(LENGTH(`uid`) > 0U AND `resource_id` = "10001"))", + R"(NOT(LENGTH(`uid`) > 0U OR `resource_id` = "10001"))", + R"(`level` IS NULL OR `message` IS NULL)", + R"(`level` IS NOT NULL AND `message` IS NULL)", + R"(`level` IS NULL AND `message` IS NOT NULL)", + R"(`level` IS NOT NULL AND `message` IS NOT NULL)", + R"(`level` IS NULL XOR `message` IS NOT NULL)", + R"(`level` IS NULL XOR `message` IS NULL)", #endif }; @@ -1703,6 +1674,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) { R"(Unwrap(`level`/1) = `level` AND `resource_id` = "10001")", // We can handle this case in future R"(NOT(LENGTH(`uid`) > 0 OR `resource_id` = "10001"))", + R"(`level` * 3.14 > 4)", #if SSA_RUNTIME_VERSION < 2U R"(`uid` LIKE "%30000%")", R"(`uid` LIKE "uid%")", @@ -1818,7 +1790,11 @@ Y_UNIT_TEST_SUITE(KqpOlap) { { R"(`uid` NOT LIKE "%30000%")", "TableFullScan" }, { R"(`uid` LIKE "uid%")", "TableFullScan" }, { R"(`uid` LIKE "%001")", "TableFullScan" }, +#if SSA_RUNTIME_VERSION >= 4U + { R"(`uid` LIKE "uid%001")", "TableFullScan" }, +#else { R"(`uid` LIKE "uid%001")", "Filter-TableFullScan" }, // We have filter (Size >= 6) +#endif }; std::string query = R"( SELECT `timestamp` FROM `/Root/olapStore/olapTable` WHERE @@ -5307,7 +5283,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) { tableInserter.AddRow().Add(2).Add("test_res_2").Add("val1").AddNull(); tableInserter.AddRow().Add(3).Add("test_res_3").Add("val3").AddNull(); tableInserter.AddRow().Add(2).Add("test_res_2").Add("val2").AddNull(); - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } while (csController->GetIndexations().Val() == 0) { Cout << "Wait indexation..." << Endl; diff --git a/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp index 249a8a205d49..b954da884d1d 100644 --- a/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_constraints_ut.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -466,6 +465,123 @@ Y_UNIT_TEST_SUITE(KqpConstraints) { } } + Y_UNIT_TEST(AlterTableAddNotNullWithDefaultIndexed) { + + NKikimrConfig::TAppConfig appConfig; + appConfig.MutableTableServiceConfig()->SetEnableSequences(false); + appConfig.MutableTableServiceConfig()->SetEnableColumnsWithDefault(true); + + TKikimrRunner kikimr(TKikimrSettings().SetPQConfig(DefaultPQConfig()).SetAppConfig(appConfig)); + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { + auto query = R"( + --!syntax_v1 + CREATE TABLE `/Root/AlterTableAddNotNullColumn` ( + Key Uint32, + Value String, + Value2 Int32 NOT NULL DEFAULT 1, + PRIMARY KEY (Key), + INDEX ByValue GLOBAL ON (Value) COVER (Value2) + ); + )"; + + auto result = session.ExecuteSchemeQuery(query).GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, + result.GetIssues().ToString()); + } + + auto fQuery = [&](TString query) -> TString { + NYdb::NTable::TExecDataQuerySettings execSettings; + execSettings.KeepInQueryCache(true); + execSettings.CollectQueryStats(ECollectQueryStatsMode::Basic); + + auto result = + session + .ExecuteDataQuery(query, TTxControl::BeginTx().CommitTx(), + execSettings) + .ExtractValueSync(); + + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, + result.GetIssues().ToString()); + if (result.GetResultSets().size() > 0) + return NYdb::FormatResultSetYson(result.GetResultSet(0)); + return ""; + }; + + fQuery(R"( + UPSERT INTO `/Root/AlterTableAddNotNullColumn` (Key, Value) VALUES (1, "Old"); + )"); + + auto fCompareTable = [&](TString expected) { + TString query = R"( + SELECT * FROM `/Root/AlterTableAddNotNullColumn` ORDER BY Key; + )"; + CompareYson(expected, fQuery(query)); + }; + + fCompareTable(R"( + [ + [[1u];["Old"];1] + ] + )"); + + fQuery(R"( + INSERT INTO `/Root/AlterTableAddNotNullColumn` (Key, Value) VALUES (2, "New"); + )"); + + fCompareTable(R"( + [ + [[1u];["Old"];1];[[2u];["New"];1] + ] + )"); + + + fQuery(R"( + UPSERT INTO `/Root/AlterTableAddNotNullColumn` (Key, Value, Value2) VALUES (2, "New", 2); + )"); + + fCompareTable(R"( + [ + [[1u];["Old"];1];[[2u];["New"];2] + ] + )"); + + fQuery(R"( + UPSERT INTO `/Root/AlterTableAddNotNullColumn` (Key, Value) VALUES (2, "OldNew"); + )"); + + fQuery(R"( + UPSERT INTO `/Root/AlterTableAddNotNullColumn` (Key, Value) VALUES (3, "BrandNew"); + )"); + + fCompareTable(R"( + [ + [[1u];["Old"];1];[[2u];["OldNew"];2];[[3u];["BrandNew"];1] + ] + )"); + + CompareYson( + fQuery("SELECT Value, Value2 FROM `/Root/AlterTableAddNotNullColumn` VIEW ByValue WHERE Value = \"OldNew\""), + R"( + [ + [["OldNew"];2] + ] + )" + ); + + CompareYson( + fQuery("SELECT Value, Value2 FROM `/Root/AlterTableAddNotNullColumn` VIEW ByValue WHERE Value = \"BrandNew\""), + R"( + [ + [["BrandNew"];1] + ] + )" + ); + + } + Y_UNIT_TEST(AlterTableAddNotNullWithDefault) { NKikimrConfig::TAppConfig appConfig; diff --git a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp index 536595ccf922..0f30a84aca39 100644 --- a/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp +++ b/ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp @@ -5,11 +5,11 @@ #include #include #include -#include #include #include #include #include +#include #include @@ -5378,7 +5378,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); tableInserter.AddRow().Add(1).Add("test_res_1").AddNull(); tableInserter.AddRow().Add(2).Add("test_res_2").Add(123); - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#;[\"test_res_1\"]]]"); @@ -5407,7 +5407,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); tableInserter.AddRow().Add(3).Add("test_res_3").Add(123).Add(200); - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=3", "[[3;[123];[200u];[\"test_res_3\"]]]"); @@ -5444,7 +5444,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#;#;[\"test_res_1\"]]]"); } - +/* Y_UNIT_TEST(AddColumnOnSchemeChange) { TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; @@ -5471,7 +5471,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#;#;[\"test_res_1\"]]]"); } - +*/ Y_UNIT_TEST(AddColumnWithStore) { TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; @@ -5494,7 +5494,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); tableInserter.AddRow().Add(1).Add("test_res_1").AddNull(); tableInserter.AddRow().Add(2).Add("test_res_2").Add(123); - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } testHelper.ReadData("SELECT * FROM `/Root/TableStoreTest/ColumnTableTest` WHERE id=1", "[[1;#;[\"test_res_1\"]]]"); @@ -5524,7 +5524,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); tableInserter.AddRow().Add(3).Add("test_res_3").Add(123).Add(200); - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } testHelper.ReadData("SELECT * FROM `/Root/TableStoreTest/ColumnTableTest` WHERE id=3", "[[3;[123];[200u];[\"test_res_3\"]]]"); @@ -5580,7 +5580,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schemaWithNull)); tableInserter.AddRow().Add(1).Add("test_res_1").AddNull(); tableInserter.AddRow().Add(2).Add("test_res_2").Add(123); - testHelper.InsertData(testTable, tableInserter, {}, EStatus::GENERIC_ERROR); + testHelper.BulkUpsert(testTable, tableInserter, Ydb::StatusIds::GENERIC_ERROR); } { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schemaWithNull)); @@ -5611,7 +5611,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); tableInserter.AddRow().Add(1).Add("test_res_1").AddNull(); tableInserter.AddRow().Add(2).Add("test_res_2").Add(123); - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#;[\"test_res_1\"]]]"); { @@ -5628,7 +5628,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` ", "[[1];[2]]"); } - +/* Y_UNIT_TEST(DropColumnOnSchemeChange) { TKikimrSettings runnerSettings; runnerSettings.WithSampleTables = false; @@ -5654,34 +5654,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#]]"); } - - Y_UNIT_TEST(DropColumnOldScheme) { - TKikimrSettings runnerSettings; - runnerSettings.WithSampleTables = false; - TTestHelper testHelper(runnerSettings); - - TVector schema = { - TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int32).SetNullable(false), - TTestHelper::TColumnSchema().SetName("resource_id").SetType(NScheme::NTypeIds::Utf8), - TTestHelper::TColumnSchema().SetName("level").SetType(NScheme::NTypeIds::Int32) - }; - - TTestHelper::TColumnTable testTable; - - testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({"id"}).SetSharding({"id"}).SetSchema(schema); - testHelper.CreateTable(testTable); - { - auto alterQuery = TStringBuilder() << "ALTER TABLE `" << testTable.GetName() << "`DROP COLUMN resource_id;"; - auto alterResult = testHelper.GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync(); - UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), EStatus::SUCCESS, alterResult.GetIssues().ToString()); - } - { - TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); - tableInserter.AddRow().Add(1).Add("test_res_1").AddNull(); - testHelper.InsertData(testTable, tableInserter, {}, EStatus::SUCCESS); - } - // testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#]]"); - } +*/ Y_UNIT_TEST(DropColumnOldSchemeBulkUpsert) { TKikimrSettings runnerSettings; @@ -5729,7 +5702,7 @@ Y_UNIT_TEST_SUITE(KqpOlapScheme) { TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); tableInserter.AddRow().Add(1).Add("test_res_1").AddNull(); tableInserter.AddRow().Add(2).Add("test_res_2").Add(123); - testHelper.InsertData(testTable, tableInserter); + testHelper.BulkUpsert(testTable, tableInserter); } testHelper.ReadData("SELECT * FROM `/Root/ColumnTableTest` WHERE id=1", "[[1;#;[\"test_res_1\"]]]"); { @@ -5953,8 +5926,39 @@ Y_UNIT_TEST_SUITE(KqpOlapTypes) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SCHEME_ERROR, result.GetIssues().ToString()); } } -} + Y_UNIT_TEST(JsonImport) { + TKikimrSettings runnerSettings; + runnerSettings.WithSampleTables = false; + + TTestHelper testHelper(runnerSettings); + + TVector schema = { + TTestHelper::TColumnSchema().SetName("id").SetType(NScheme::NTypeIds::Int64).SetNullable(false), + TTestHelper::TColumnSchema().SetName("json").SetType(NScheme::NTypeIds::Json).SetNullable(true), + TTestHelper::TColumnSchema().SetName("json_doc").SetType(NScheme::NTypeIds::JsonDocument).SetNullable(true), + }; + + TTestHelper::TColumnTable testTable; + testTable.SetName("/Root/ColumnTableTest").SetPrimaryKey({ "id" }).SetSharding({ "id" }).SetSchema(schema); + testHelper.CreateTable(testTable); + std::string jsonString = R"({"col1": "val1", "obj": {"obj_col2_int": 16}})"; + auto maybeJsonDoc = NBinaryJson::SerializeToBinaryJson(jsonString); + Y_ABORT_UNLESS(maybeJsonDoc.Defined()); + const std::string jsonBin(maybeJsonDoc->Data(), maybeJsonDoc->Size()); + { + TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); + tableInserter.AddRow().Add(1).AddNull().Add(jsonString); + tableInserter.AddRow().Add(2).Add(jsonString).Add(jsonBin); + testHelper.BulkUpsert(testTable, tableInserter); + } + { + TTestHelper::TUpdatesBuilder tableInserter(testTable.GetArrowSchema(schema)); + tableInserter.AddRow().Add(3).Add(jsonBin).AddNull(); + testHelper.BulkUpsert(testTable, tableInserter, Ydb::StatusIds::SCHEME_ERROR); + } + } +} } // namespace NKqp } // namespace NKikimr diff --git a/ydb/core/load_test/config_examples.cpp b/ydb/core/load_test/config_examples.cpp index 96e383e5c248..fe93d99295b0 100644 --- a/ydb/core/load_test/config_examples.cpp +++ b/ydb/core/load_test/config_examples.cpp @@ -25,6 +25,20 @@ TVector BuildExamples() { ColumnsCnt: 2 RowsCnt: 1 } +})_" + }, + TConfigTemplate{ + .LoadName = "KeyValueLoad", + .Template = R"_(KeyValueLoad: { + TargetTabletId: xxx + DurationSeconds: 120 + Workers { + KeyPrefix: "LoadTest_" + MaxInFlight: 128 + Size: 1024 + IsInline: false + LoopAtKeyCount: 0 + } })_" }, TConfigTemplate{ diff --git a/ydb/core/load_test/group_write.cpp b/ydb/core/load_test/group_write.cpp index a57e1584bb9c..26bb6a9c7145 100644 --- a/ydb/core/load_test/group_write.cpp +++ b/ydb/core/load_test/group_write.cpp @@ -223,8 +223,8 @@ class TLogWriterLoadTestActor : public TActorBootstrapped ConfirmedBlobs; diff --git a/ydb/core/mind/hive/hive_ut.cpp b/ydb/core/mind/hive/hive_ut.cpp index 984674a3989b..b28ee8db4b96 100644 --- a/ydb/core/mind/hive/hive_ut.cpp +++ b/ydb/core/mind/hive/hive_ut.cpp @@ -4163,7 +4163,9 @@ Y_UNIT_TEST_SUITE(THiveTest) { static const int NUM_TABLETS = NUM_NODES * TABLETS_PER_NODE; TTestBasicRuntime runtime(NUM_NODES, false); - Setup(runtime, true); + Setup(runtime, true, 1, [](TAppPrepare& app) { + app.HiveConfig.SetWarmUpEnabled(true); + }); const int nodeBase = runtime.GetNodeId(0); TActorId senderA = runtime.AllocateEdgeActor(); const ui64 hiveTablet = MakeDefaultHiveID(0); diff --git a/ydb/core/mind/hive/monitoring.cpp b/ydb/core/mind/hive/monitoring.cpp index b530f10c6994..f7a27621cc4b 100644 --- a/ydb/core/mind/hive/monitoring.cpp +++ b/ydb/core/mind/hive/monitoring.cpp @@ -294,7 +294,7 @@ class TTxMonEvent_MemStateTablets : public TTransactionBase { out << "" << Sprintf("%.9f", x.Weight) << ""; out << "" << GetResourceValuesText(x) << ""; out << "" << x.GetTabletAllowedMetricIds() << ""; - out << ""; + out << ""; out << ""; } out << ""; diff --git a/ydb/core/mind/hive/tx__load_everything.cpp b/ydb/core/mind/hive/tx__load_everything.cpp index 7818698a123f..9b8b05ae9b12 100644 --- a/ydb/core/mind/hive/tx__load_everything.cpp +++ b/ydb/core/mind/hive/tx__load_everything.cpp @@ -181,7 +181,11 @@ class TTxLoadEverything : public TTransactionBase { } Self->BuildCurrentConfig(); - Self->WarmUp = Self->CurrentConfig.GetWarmUpEnabled(); + if (Self->CurrentConfig.HasWarmUpEnabled()) { + Self->WarmUp = Self->CurrentConfig.GetWarmUpEnabled(); + } else { + Self->WarmUp = Self->CurrentConfig.GetWarmUpEnabled() && !Self->AreWeRootHive(); + } Self->DefaultResourceMetricsAggregates.MaximumCPU.SetWindowSize(TDuration::MilliSeconds(Self->GetMetricsWindowSize())); Self->DefaultResourceMetricsAggregates.MaximumMemory.SetWindowSize(TDuration::MilliSeconds(Self->GetMetricsWindowSize())); diff --git a/ydb/core/protos/feature_flags.proto b/ydb/core/protos/feature_flags.proto index 9cd3352402bf..bf4af84de48b 100644 --- a/ydb/core/protos/feature_flags.proto +++ b/ydb/core/protos/feature_flags.proto @@ -97,7 +97,7 @@ message TFeatureFlags { optional bool EnableAlterDatabaseCreateHiveFirst = 82 [default = false]; reserved 83; // EnableKqpDataQuerySourceRead optional bool EnableSmallDiskOptimization = 84 [default = true]; - optional bool EnableDataShardVolatileTransactions = 85 [default = false]; + optional bool EnableDataShardVolatileTransactions = 85 [default = true]; optional bool EnableTopicServiceTx = 86 [default = false]; optional bool EnableLLVMCache = 87 [default = false]; optional bool EnableExternalDataSources = 88 [default = false]; diff --git a/ydb/core/testlib/cs_helper.cpp b/ydb/core/testlib/cs_helper.cpp index 557d0cfff98f..4853290f8637 100644 --- a/ydb/core/testlib/cs_helper.cpp +++ b/ydb/core/testlib/cs_helper.cpp @@ -429,15 +429,14 @@ std::shared_ptr TTableWithNullsHelper::TestArrowBatch(ui64, Y_ABORT_UNLESS(bJsonDoc.AppendNull().ok()); } - auto maybeJsonDoc = NBinaryJson::SerializeToBinaryJson(R"({"col1": "val1", "obj": {"obj_col2_int": 16}})"); - Y_ABORT_UNLESS(maybeJsonDoc.Defined()); + const auto maybeJsonDoc = std::string(R"({"col1": "val1", "obj": {"obj_col2_int": 16}})"); for (size_t i = rowCount / 2 + 1; i <= rowCount; ++i) { Y_ABORT_UNLESS(bId.Append(i).ok()); Y_ABORT_UNLESS(bResourceId.Append(std::to_string(i)).ok()); Y_ABORT_UNLESS(bLevel.AppendNull().ok()); Y_ABORT_UNLESS(bBinaryStr.Append(std::to_string(i)).ok()); Y_ABORT_UNLESS(bJsonVal.AppendNull().ok()); - Y_ABORT_UNLESS(bJsonDoc.Append(maybeJsonDoc->Data(), maybeJsonDoc->Size()).ok()); + Y_ABORT_UNLESS(bJsonDoc.Append(maybeJsonDoc.data(), maybeJsonDoc.length()).ok()); } std::shared_ptr aId; diff --git a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp index 08e4fad95c66..5d866ae4e32e 100644 --- a/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp +++ b/ydb/core/tx/columnshard/engines/changes/general_compaction.cpp @@ -18,7 +18,6 @@ namespace NKikimr::NOlap::NCompaction { void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByFullBatches(TConstructionContext& context) noexcept { std::vector portions = TPortionInfoWithBlobs::RestorePortions(SwitchedPortions, Blobs); - Blobs.clear(); std::vector> batchResults; auto resultSchema = context.SchemaVersions.GetLastSchema(); { @@ -45,7 +44,6 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByFullBatches(TCon void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(TConstructionContext& context) noexcept { std::vector portions = TPortionInfoWithBlobs::RestorePortions(SwitchedPortions, Blobs); - Blobs.clear(); static const TString portionIdFieldName = "$$__portion_id"; static const TString portionRecordIndexFieldName = "$$__portion_record_idx"; static const std::shared_ptr portionIdField = std::make_shared(portionIdFieldName, std::make_shared()); @@ -222,7 +220,7 @@ TConclusionStatus TGeneralCompactColumnEngineChanges::DoConstructBlobs(TConstruc NChanges::TGeneralCompactionCounters::OnPortionsKind(insertedPortionsSize, compactedPortionsSize, otherPortionsSize); NChanges::TGeneralCompactionCounters::OnRepackPortions(portionsCount, portionsSize); - if (AppDataVerified().ColumnShardConfig.GetUseChunkedMergeOnCompaction()) { + if (!HasAppData() || AppDataVerified().ColumnShardConfig.GetUseChunkedMergeOnCompaction()) { BuildAppendedPortionsByChunks(context); } else { BuildAppendedPortionsByFullBatches(context); diff --git a/ydb/core/tx/columnshard/engines/portions/common.h b/ydb/core/tx/columnshard/engines/portions/common.h index b0305fa53e9a..311563918469 100644 --- a/ydb/core/tx/columnshard/engines/portions/common.h +++ b/ydb/core/tx/columnshard/engines/portions/common.h @@ -8,6 +8,14 @@ class TChunkAddress { YDB_READONLY(ui32, ColumnId, 0); YDB_READONLY(ui16, Chunk, 0); public: + ui32 GetEntityId() const { + return ColumnId; + } + + ui32 GetChunkIdx() const { + return Chunk; + } + TChunkAddress(const ui32 columnId, const ui16 chunk) : ColumnId(columnId) , Chunk(chunk) { diff --git a/ydb/core/tx/columnshard/engines/portions/portion_info.h b/ydb/core/tx/columnshard/engines/portions/portion_info.h index 19ba5e3c1df7..00e4fd205aef 100644 --- a/ydb/core/tx/columnshard/engines/portions/portion_info.h +++ b/ydb/core/tx/columnshard/engines/portions/portion_info.h @@ -97,7 +97,9 @@ class TPortionInfo { TSerializationStats GetSerializationStat(const ISnapshotSchema& schema) const { TSerializationStats result; for (auto&& i : Records) { - result.AddStat(i.GetSerializationStat(schema.GetFieldByColumnIdVerified(i.ColumnId)->name())); + if (schema.GetFieldByColumnId(i.ColumnId)) { + result.AddStat(i.GetSerializationStat(schema.GetFieldByColumnIdVerified(i.ColumnId)->name())); + } } return result; } diff --git a/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.cpp b/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.cpp index f138fc6b5f3f..13dc7a35f1e9 100644 --- a/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.cpp +++ b/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.cpp @@ -110,4 +110,10 @@ ui32 ISnapshotSchema::GetColumnId(const std::string& columnName) const { return *id; } +std::shared_ptr ISnapshotSchema::GetFieldByColumnIdVerified(const ui32 columnId) const { + auto result = GetFieldByColumnId(columnId); + AFL_VERIFY(result)("event", "unknown_column")("column_id", columnId); + return result; +} + } diff --git a/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.h b/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.h index 1ff127bee20a..9232edcf1dfe 100644 --- a/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.h +++ b/ydb/core/tx/columnshard/engines/scheme/abstract_scheme.h @@ -52,11 +52,7 @@ class ISnapshotSchema { ui32 GetColumnId(const std::string& columnName) const; std::shared_ptr GetFieldByIndex(const int index) const; std::shared_ptr GetFieldByColumnId(const ui32 columnId) const; - std::shared_ptr GetFieldByColumnIdVerified(const ui32 columnId) const { - auto result = GetFieldByColumnId(columnId); - Y_ABORT_UNLESS(result); - return result; - } + std::shared_ptr GetFieldByColumnIdVerified(const ui32 columnId) const; TString DebugString() const { return DoDebugString(); diff --git a/ydb/core/tx/columnshard/engines/ut_insert_table.cpp b/ydb/core/tx/columnshard/engines/ut_insert_table.cpp index 5f0848436ae5..ee10d7674b53 100644 --- a/ydb/core/tx/columnshard/engines/ut_insert_table.cpp +++ b/ydb/core/tx/columnshard/engines/ut_insert_table.cpp @@ -27,12 +27,12 @@ class TTestInsertTableDB : public IDbWrapper { return true; } - void WriteColumn(ui32, const TPortionInfo&, const TColumnRecord&) override {} - void EraseColumn(ui32, const TPortionInfo&, const TColumnRecord&) override {} - bool LoadColumns(ui32, const std::function&) override { return true; } + void WriteColumn(const TPortionInfo&, const TColumnRecord&) override {} + void EraseColumn(const TPortionInfo&, const TColumnRecord&) override {} + bool LoadColumns(const std::function&) override { return true; } - void WriteCounter(ui32, ui32, ui64) override {} - bool LoadCounters(ui32, const std::function&) override { return true; } + void WriteCounter(ui32, ui64) override {} + bool LoadCounters(const std::function&) override { return true; } }; } diff --git a/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp b/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp index 8e0c8ceef165..229100ecab5e 100644 --- a/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp +++ b/ydb/core/tx/columnshard/engines/ut_logs_engine.cpp @@ -69,14 +69,14 @@ class TTestDbWrapper : public IDbWrapper { return true; } - void WriteColumn(ui32 index, const TPortionInfo& portion, const TColumnRecord& row) override { + void WriteColumn(const TPortionInfo& portion, const TColumnRecord& row) override { auto proto = portion.GetMeta().SerializeToProto(row.ColumnId, row.Chunk); auto rowProto = row.GetMeta().SerializeToProto(); if (proto) { *rowProto.MutablePortionMeta() = std::move(*proto); } - auto& data = Indices[index].Columns[portion.GetPathId()]; + auto& data = Indices[0].Columns[portion.GetPathId()]; NOlap::TColumnChunkLoadContext loadContext(row.GetAddress(), row.BlobRange, rowProto); auto itInsertInfo = LoadContexts[portion.GetAddress()].emplace(row.GetAddress(), loadContext); if (!itInsertInfo.second) { @@ -104,8 +104,8 @@ class TTestDbWrapper : public IDbWrapper { } } - void EraseColumn(ui32 index, const TPortionInfo& portion, const TColumnRecord& row) override { - auto& data = Indices[index].Columns[portion.GetPathId()]; + void EraseColumn(const TPortionInfo& portion, const TColumnRecord& row) override { + auto& data = Indices[0].Columns[portion.GetPathId()]; auto it = data.find(portion.GetPortion()); Y_ABORT_UNLESS(it != data.end()); auto& portionLocal = it->second; @@ -119,8 +119,8 @@ class TTestDbWrapper : public IDbWrapper { portionLocal.Records.swap(filtered); } - bool LoadColumns(ui32 index, const std::function& callback) override { - auto& columns = Indices[index].Columns; + bool LoadColumns(const std::function& callback) override { + auto& columns = Indices[0].Columns; for (auto& [pathId, portions] : columns) { for (auto& [portionId, portionLocal] : portions) { auto copy = portionLocal; @@ -137,13 +137,13 @@ class TTestDbWrapper : public IDbWrapper { return true; } - void WriteCounter(ui32 index, ui32 counterId, ui64 value) override { - auto& counters = Indices[index].Counters; + void WriteCounter(ui32 counterId, ui64 value) override { + auto& counters = Indices[0].Counters; counters[counterId] = value; } - bool LoadCounters(ui32 index, const std::function& callback) override { - auto& counters = Indices[index].Counters; + bool LoadCounters(const std::function& callback) override { + auto& counters = Indices[0].Counters; for (auto& [id, value] : counters) { callback(id, value); } diff --git a/ydb/core/tx/datashard/datashard_pipeline.cpp b/ydb/core/tx/datashard/datashard_pipeline.cpp index 084e62d2a9da..8d8eec8f9d09 100644 --- a/ydb/core/tx/datashard/datashard_pipeline.cpp +++ b/ydb/core/tx/datashard/datashard_pipeline.cpp @@ -1171,6 +1171,10 @@ bool TPipeline::CancelPropose(NIceDb::TNiceDb& db, const TActorContext& ctx, ui6 return false; } + if (!op->IsExecutionPlanFinished()) { + GetExecutionUnit(op->GetCurrentUnit()).RemoveOperation(op); + } + ForgetTx(txId); Self->CheckDelayedProposeQueue(ctx); MaybeActivateWaitingSchemeOps(ctx); @@ -1196,6 +1200,11 @@ ECleanupStatus TPipeline::CleanupOutdated(NIceDb::TNiceDb& db, const TActorConte } for (ui64 txId : outdatedTxs) { + auto op = Self->TransQueue.FindTxInFly(txId); + if (op && !op->IsExecutionPlanFinished()) { + GetExecutionUnit(op->GetCurrentUnit()).RemoveOperation(op); + } + ForgetTx(txId); LOG_INFO(ctx, NKikimrServices::TX_DATASHARD, "Outdated Tx %" PRIu64 " is cleaned at tablet %" PRIu64 " and outdatedStep# %" PRIu64, @@ -1211,6 +1220,11 @@ bool TPipeline::CleanupVolatile(ui64 txId, const TActorContext& ctx, std::vector>& replies) { if (Self->TransQueue.CleanupVolatile(txId, replies)) { + auto op = Self->TransQueue.FindTxInFly(txId); + if (op && !op->IsExecutionPlanFinished()) { + GetExecutionUnit(op->GetCurrentUnit()).RemoveOperation(op); + } + ForgetTx(txId); Self->CheckDelayedProposeQueue(ctx); diff --git a/ydb/core/tx/datashard/datashard_ut_locks.cpp b/ydb/core/tx/datashard/datashard_ut_locks.cpp index 7056a184645d..029af4fb409e 100644 --- a/ydb/core/tx/datashard/datashard_ut_locks.cpp +++ b/ydb/core/tx/datashard/datashard_ut_locks.cpp @@ -574,7 +574,10 @@ void CheckLocksCacheUsage(bool waitForLocksStore) { TPortManager pm; TServerSettings serverSettings(pm.GetPort(2134)); serverSettings.SetDomainName("Root") - .SetUseRealThreads(false); + .SetUseRealThreads(false) + // Note: disable volatile transactions, since they don't persist lock + // state, and this test actually checks validated lock persistence. + .SetEnableDataShardVolatileTransactions(false); Tests::TServer::TPtr server = new TServer(serverSettings); auto &runtime = *server->GetRuntime(); @@ -587,7 +590,6 @@ void CheckLocksCacheUsage(bool waitForLocksStore) { InitRoot(server, sender); - TAutoPtr handle; NTabletPipe::TClientConfig pipeConfig; pipeConfig.RetryPolicy = NTabletPipe::TClientRetryPolicy::WithRetries(); diff --git a/ydb/core/tx/datashard/datashard_ut_order.cpp b/ydb/core/tx/datashard/datashard_ut_order.cpp index 3cca28881e85..334084d1a4ab 100644 --- a/ydb/core/tx/datashard/datashard_ut_order.cpp +++ b/ydb/core/tx/datashard/datashard_ut_order.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include @@ -1247,8 +1248,9 @@ Y_UNIT_TEST(TestDelayedTxWaitsForWriteActiveTxOnly) { TVector> rss; // We want to intercept all RS to table-2. - auto captureRS = [shard2,&rss](TAutoPtr &event) -> auto { - if (event->GetTypeRewrite() == TEvTxProcessing::EvReadSet) { + bool blockReadSets = true; + auto captureRS = [shard2,&rss,&blockReadSets](TAutoPtr &event) -> auto { + if (blockReadSets && event->GetTypeRewrite() == TEvTxProcessing::EvReadSet) { auto &rec = event->Get()->Record; if (rec.GetTabletDest() == shard2) { rss.push_back(std::move(event)); @@ -1283,10 +1285,12 @@ Y_UNIT_TEST(TestDelayedTxWaitsForWriteActiveTxOnly) { // because transactions above are stuck before performing any writes. Make sure it's // forced to wait for above transactions by commiting a write that is guaranteed // to "happen" after transactions above. + blockReadSets = false; // volatile transactions exchange readsets ExecSQL(server, sender, Q_(R"( UPSERT INTO `/Root/table-1` (key, value) VALUES (4, 4); UPSERT INTO `/Root/table-2` (key, value) VALUES (5, 5); )")); + blockReadSets = true; // restore to blocking // This immediate tx should be delayed due to conflict with upserts. SendSQL(server, sender, Q_("SELECT * FROM `/Root/table-2`")); @@ -1317,7 +1321,10 @@ Y_UNIT_TEST(TestOnlyDataTxLagCausesRejects) { serverSettings.SetEnableMvccSnapshotReads(false); serverSettings.SetDomainName("Root") .SetUseRealThreads(false) - .SetAppConfig(app); + .SetAppConfig(app) + // Note: this test currently relies on distributed non-mvcc reads, + // which are deprecated. This test should probably be removed. + .SetEnableDataShardVolatileTransactions(false); Tests::TServer::TPtr server = new TServer(serverSettings); auto &runtime = *server->GetRuntime(); @@ -1329,6 +1336,8 @@ Y_UNIT_TEST(TestOnlyDataTxLagCausesRejects) { runtime.SetLogPriority(NKikimrServices::KQP_SESSION, NLog::PRI_DEBUG); runtime.SetLogPriority(NKikimrServices::MINIKQL_ENGINE, NActors::NLog::PRI_DEBUG); + const bool usesVolatileTxs = runtime.GetAppData(0).FeatureFlags.GetEnableDataShardVolatileTransactions(); + InitRoot(server, sender); CreateShardedTable(server, sender, "/Root", "table-1", 2); @@ -1370,14 +1379,15 @@ Y_UNIT_TEST(TestOnlyDataTxLagCausesRejects) { SendSQL(server, sender, Q_("UPSERT INTO `/Root/table-1` (key, value) SELECT value, key FROM `/Root/table-1`")); + const size_t expectedReadSets = usesVolatileTxs ? 4 : 2; { TDispatchOptions options; options.FinalEvents.emplace_back( [&](IEventHandle &) -> bool { - return readSets.size() >= 2; + return readSets.size() >= expectedReadSets; }); runtime.DispatchEvents(options); - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); } // Now move time forward and check we can still execute data txs. @@ -1413,6 +1423,8 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderLockLost, StreamLookup) { runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG); + const bool usesVolatileTxs = runtime.GetAppData(0).FeatureFlags.GetEnableDataShardVolatileTransactions(); + InitRoot(server, sender); CreateShardedTable(server, sender, "/Root", "table-1", 1); @@ -1454,14 +1466,15 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderLockLost, StreamLookup) { UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))"), sessionId, txId, true)); // Wait until we captured both readsets + const size_t expectedReadSets = usesVolatileTxs ? 4 : 2; { TDispatchOptions options; options.FinalEvents.emplace_back( [&](IEventHandle &) -> bool { - return readSets.size() >= 2; + return readSets.size() >= expectedReadSets; }); runtime.DispatchEvents(options); - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); } // Now send a simple request that would upsert a new value into table-1 @@ -1541,6 +1554,8 @@ Y_UNIT_TEST(TestMvccReadDoesntBlockWrites) { runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG); + const bool usesVolatileTxs = runtime.GetAppData(0).FeatureFlags.GetEnableDataShardVolatileTransactions(); + InitRoot(server, sender); CreateShardedTable(server, sender, "/Root", "table-1", 1); @@ -1582,14 +1597,15 @@ Y_UNIT_TEST(TestMvccReadDoesntBlockWrites) { UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2);)"), sessionId, txId, true)); // Wait until we captured both readsets + const size_t expectedReadSets = usesVolatileTxs ? 4 : 2; { TDispatchOptions options; options.FinalEvents.emplace_back( [&](IEventHandle &) -> bool { - return readSets.size() >= 2; + return readSets.size() >= expectedReadSets; }); runtime.DispatchEvents(options); - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); } runtime.SetObserverFunc(prevObserverFunc); @@ -1673,6 +1689,8 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderReadOnlyAllowed, StreamLookup) { runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG); + const bool usesVolatileTxs = runtime.GetAppData(0).FeatureFlags.GetEnableDataShardVolatileTransactions(); + InitRoot(server, sender); CreateShardedTable(server, sender, "/Root", "table-1", 1); @@ -1713,19 +1731,21 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderReadOnlyAllowed, StreamLookup) { UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))"), sessionId, txId, true)); // Wait until we captured both readsets + const size_t expectedReadSets = usesVolatileTxs ? 4 : 2; { TDispatchOptions options; options.FinalEvents.emplace_back( [&](IEventHandle &) -> bool { - return readSets.size() >= 2; + return readSets.size() >= expectedReadSets; }); runtime.DispatchEvents(options); - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); } // Now send a simple read request from table-1 - // Since it's readonly it cannot affect inflight transaction and shouled be allowed - { + // Since it's readonly it cannot affect inflight transaction and should be allowed + // Note: volatile transactions execute immediately and reads are blocked until resolved + if (!usesVolatileTxs) { auto result = KqpSimpleExec(runtime, Q_("SELECT key, value FROM `/Root/table-1` ORDER BY key")); UNIT_ASSERT_VALUES_EQUAL(result, "{ items { uint32_value: 1 } items { uint32_value: 1 } }"); } @@ -1772,6 +1792,8 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderNonConflictingWrites, StreamLookup) { runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG); + const bool usesVolatileTxs = runtime.GetAppData(0).FeatureFlags.GetEnableDataShardVolatileTransactions(); + InitRoot(server, sender); CreateShardedTable(server, sender, "/Root", "table-1", 1); @@ -1797,8 +1819,9 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderNonConflictingWrites, StreamLookup) { // Capture and block all readset messages TVector> readSets; + bool blockReadSets = true; auto captureRS = [&](TAutoPtr &event) -> auto { - if (event->GetTypeRewrite() == TEvTxProcessing::EvReadSet) { + if (blockReadSets && event->GetTypeRewrite() == TEvTxProcessing::EvReadSet) { readSets.push_back(std::move(event)); return TTestActorRuntime::EEventAction::DROP; } @@ -1812,22 +1835,25 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderNonConflictingWrites, StreamLookup) { UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))"), sessionId, txId, true)); // Wait until we captured both readsets - if (readSets.size() < 2) { + const size_t expectedReadSets = usesVolatileTxs ? 4 : 2; + if (readSets.size() < expectedReadSets) { TDispatchOptions options; options.FinalEvents.emplace_back( [&](IEventHandle &) -> bool { - return readSets.size() >= 2; + return readSets.size() >= expectedReadSets; }); runtime.DispatchEvents(options); } - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); // Now send non-conflicting upsert to both tables { + blockReadSets = false; // needed for volatile transactions auto result = KqpSimpleExec(runtime, Q_(R"( UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 3); UPSERT INTO `/Root/table-2` (key, value) VALUES (6, 3))")); UNIT_ASSERT_VALUES_EQUAL(result, ""); + blockReadSets = true; // restore to blocking } // Check that immediate non-conflicting upsert is working too @@ -1882,6 +1908,8 @@ Y_UNIT_TEST(MvccTestOutOfOrderRestartLocksSingleWithoutBarrier) { runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG); + const bool usesVolatileTxs = runtime.GetAppData(0).FeatureFlags.GetEnableDataShardVolatileTransactions(); + InitRoot(server, sender); CreateShardedTable(server, sender, "/Root", "table-1", 1); @@ -1924,35 +1952,37 @@ Y_UNIT_TEST(MvccTestOutOfOrderRestartLocksSingleWithoutBarrier) { UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))"), sessionId, txId, true)); // Wait until we captured both readsets - if (readSets.size() < 2) { + const size_t expectedReadSets = usesVolatileTxs ? 4 : 2; + if (readSets.size() < expectedReadSets) { TDispatchOptions options; options.FinalEvents.emplace_back( [&](IEventHandle &) -> bool { - return readSets.size() >= 2; + return readSets.size() >= expectedReadSets; }); runtime.DispatchEvents(options); } - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); // Reboot table-1 tablet readSets.clear(); RebootTablet(runtime, table1shards[0], sender); // Wait until we captured both readsets again - if (readSets.size() < 2) { + if (readSets.size() < expectedReadSets) { TDispatchOptions options; options.FinalEvents.emplace_back( [&](IEventHandle &) -> bool { - return readSets.size() >= 2; + return readSets.size() >= expectedReadSets; }); runtime.DispatchEvents(options); } - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); // Select keys 1 and 3, we expect this immediate tx to succeed // Note that key 3 is not written yet, but we pretend immediate tx // executes before that waiting transaction (no key 3 yet). - { + // Note: volatile transactions block reads until they are resolved, so this read is skipped + if (!usesVolatileTxs) { auto result = KqpSimpleExec(runtime, Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 1 OR key = 3;")); UNIT_ASSERT_VALUES_EQUAL(result, "{ items { uint32_value: 1 } items { uint32_value: 1 } }"); } @@ -2001,6 +2031,8 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderRestartLocksReorderedWithoutBarrier, StreamLookup runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG); + const bool usesVolatileTxs = runtime.GetAppData(0).FeatureFlags.GetEnableDataShardVolatileTransactions(); + InitRoot(server, sender); CreateShardedTable(server, sender, "/Root", "table-1", 1); @@ -2028,8 +2060,9 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderRestartLocksReorderedWithoutBarrier, StreamLookup // Capture and block all readset messages TVector> readSets; + bool blockReadSets = true; auto captureRS = [&](TAutoPtr &event) -> auto { - if (event->GetTypeRewrite() == TEvTxProcessing::EvReadSet) { + if (blockReadSets && event->GetTypeRewrite() == TEvTxProcessing::EvReadSet) { readSets.push_back(std::move(event)); return TTestActorRuntime::EEventAction::DROP; } @@ -2043,20 +2076,23 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderRestartLocksReorderedWithoutBarrier, StreamLookup UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))"), sessionId, txId, true)); // Wait until we captured both readsets - if (readSets.size() < 2) { + const size_t expectedReadSets = usesVolatileTxs ? 4 : 2; + if (readSets.size() < expectedReadSets) { TDispatchOptions options; options.FinalEvents.emplace_back( [&](IEventHandle &) -> bool { - return readSets.size() >= 2; + return readSets.size() >= expectedReadSets; }); runtime.DispatchEvents(options); } - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); // Execute some out-of-order upserts before rebooting + blockReadSets = false; // needed for volatile transactions ExecSQL(server, sender, Q_(R"( UPSERT INTO `/Root/table-1` (key, value) VALUES (5, 3); UPSERT INTO `/Root/table-2` (key, value) VALUES (6, 3))")); + blockReadSets = true; // restore back to blocking // Select key 3, we expect a timeout, because logically writes // to 3 and 5 already happened, but physically write to 3 is still waiting. @@ -2069,20 +2105,20 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderRestartLocksReorderedWithoutBarrier, StreamLookup } // Reboot table-1 tablet - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); readSets.clear(); RebootTablet(runtime, table1shards[0], sender); // Wait until we captured both readsets again - if (readSets.size() < 2) { + if (readSets.size() < expectedReadSets) { TDispatchOptions options; options.FinalEvents.emplace_back( [&](IEventHandle &) -> bool { - return readSets.size() >= 2; + return readSets.size() >= expectedReadSets; }); runtime.DispatchEvents(options); } - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + UNIT_ASSERT_C(readSets.size() >= expectedReadSets, "expected " << readSets.size() << " >= " << expectedReadSets); // Select key 3, we still expect a timeout { @@ -2131,6 +2167,8 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderNoBarrierRestartImmediateLongTail, StreamLookup) runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG); + const bool usesVolatileTxs = runtime.GetAppData(0).FeatureFlags.GetEnableDataShardVolatileTransactions(); + InitRoot(server, sender); CreateShardedTable(server, sender, "/Root", "table-1", 1); @@ -2162,6 +2200,7 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderNoBarrierRestartImmediateLongTail, StreamLookup) THashMap actorToTablet; TVector> readSets; TVector> progressEvents; + bool blockReadSets = true; bool blockProgressEvents = false; size_t bypassProgressEvents = 0; auto captureRS = [&](TAutoPtr &event) -> auto { @@ -2175,8 +2214,11 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderNoBarrierRestartImmediateLongTail, StreamLookup) break; } case TEvTxProcessing::EvReadSet: { - readSets.push_back(std::move(event)); - return TTestActorRuntime::EEventAction::DROP; + if (blockReadSets) { + readSets.push_back(std::move(event)); + return TTestActorRuntime::EEventAction::DROP; + } + break; } case EventSpaceBegin(TKikimrEvents::ES_PRIVATE) + 0 /* EvProgressTransaction */: { ui64 tabletId = actorToTablet.Value(recipient, 0); @@ -2202,15 +2244,18 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderNoBarrierRestartImmediateLongTail, StreamLookup) UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))"), sessionId, txId, true)); // Wait until we captured both readsets - if (readSets.size() < 2) { + const size_t expectedReadSets = usesVolatileTxs ? 4 : 2; + if (readSets.size() < expectedReadSets) { TDispatchOptions options; options.FinalEvents.emplace_back( [&](IEventHandle &) -> bool { - return readSets.size() >= 2; + return readSets.size() >= expectedReadSets; }); runtime.DispatchEvents(options); } - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); + + blockReadSets = false; // needed when upsert is volatile // Send some more requests that form a staircase, they would all be blocked as well auto f3 = SendRequest(runtime, MakeSimpleRequestRPC(Q_(R"( @@ -2227,17 +2272,20 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderNoBarrierRestartImmediateLongTail, StreamLookup) UPSERT INTO `/Root/table-1` (key, value) VALUES (11, 5); UPSERT INTO `/Root/table-2` (key, value) VALUES (12, 5))")); + blockReadSets = true; // restore after a volatile upsert + // Select key 7, we expect a timeout, because logically a write to it already happened + // Note: volatile upserts are not blocked however, so it will succeed { auto sender4 = CreateSessionRPC(runtime); auto req = MakeSimpleRequestRPC(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 7;"), sender4, "", true); req.mutable_operation_params()->mutable_operation_timeout()->set_seconds(1); auto response = AwaitResponse(runtime, SendRequest(runtime, std::move(req))); - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::TIMEOUT); + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), usesVolatileTxs ? Ydb::StatusIds::SUCCESS : Ydb::StatusIds::TIMEOUT); } // Reboot table-1 tablet - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); readSets.clear(); blockProgressEvents = true; bypassProgressEvents = 1; @@ -2246,34 +2294,40 @@ Y_UNIT_TEST_TWIN(TestOutOfOrderNoBarrierRestartImmediateLongTail, StreamLookup) Cerr << "... tablet rebooted" << Endl; // Wait until we captured both readsets again - if (readSets.size() < 2) { + if (readSets.size() < expectedReadSets) { TDispatchOptions options; options.FinalEvents.emplace_back( [&](IEventHandle &) -> bool { - return readSets.size() >= 2; + return readSets.size() >= expectedReadSets; }); runtime.DispatchEvents(options); } - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + UNIT_ASSERT_C(readSets.size() >= expectedReadSets, "expected " << readSets.size() << " >= " << expectedReadSets); // Wait until we have a pending progress event - if (progressEvents.size() < 1) { - TDispatchOptions options; - options.FinalEvents.emplace_back( - [&](IEventHandle &) -> bool { - return progressEvents.size() >= 1; - }); - runtime.DispatchEvents(options); + if (usesVolatileTxs) { + // Transaction does not restart when volatile (it's already executed) + SimulateSleep(server, TDuration::Seconds(1)); + } else { + if (progressEvents.size() < 1) { + TDispatchOptions options; + options.FinalEvents.emplace_back( + [&](IEventHandle &) -> bool { + return progressEvents.size() >= 1; + }); + runtime.DispatchEvents(options); + } + UNIT_ASSERT_VALUES_EQUAL(progressEvents.size(), 1u); } - UNIT_ASSERT_VALUES_EQUAL(progressEvents.size(), 1u); // Select key 7 again, we still expect a timeout, because logically a write to it already happened + // Note: volatile upserts are not blocked however, so it will succeed { auto sender5 = CreateSessionRPC(runtime); auto req = MakeSimpleRequestRPC(Q_("SELECT key, value FROM `/Root/table-1` WHERE key = 7;"), sender5, "", true); req.mutable_operation_params()->mutable_operation_timeout()->set_seconds(1); auto response = AwaitResponse(runtime, SendRequest(runtime, std::move(req))); - UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::TIMEOUT); + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), usesVolatileTxs ? Ydb::StatusIds::SUCCESS : Ydb::StatusIds::TIMEOUT); } // Stop blocking readsets and unblock progress @@ -2309,6 +2363,8 @@ Y_UNIT_TEST(TestCopyTableNoDeadlock) { runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::KQP_EXECUTER, NLog::PRI_DEBUG); + const bool usesVolatileTxs = runtime.GetAppData(0).FeatureFlags.GetEnableDataShardVolatileTransactions(); + InitRoot(server, sender); CreateShardedTable(server, sender, "/Root", "table-1", 1); @@ -2372,15 +2428,16 @@ Y_UNIT_TEST(TestCopyTableNoDeadlock) { UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))"), sessionId, txId, true)); // Wait until we captured both readsets - if (readSets.size() < 2) { + const size_t expectedReadSets = usesVolatileTxs ? 4 : 2; + if (readSets.size() < expectedReadSets) { TDispatchOptions options; options.FinalEvents.emplace_back( [&](IEventHandle &) -> bool { - return readSets.size() >= 2; + return readSets.size() >= expectedReadSets; }); runtime.DispatchEvents(options); } - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); captureTxProposes = true; @@ -2475,6 +2532,31 @@ Y_UNIT_TEST(TestCopyTableNoDeadlock) { auto response = AwaitResponse(runtime, fRead); UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); } + + // Verify copied table has data committed + UNIT_ASSERT_VALUES_EQUAL( + KqpSimpleExec(runtime, Q_(R"( + SELECT * FROM `/Root/table-3` + ORDER BY key)")), + "{ items { uint32_value: 2 } items { uint32_value: 1 } }, " + "{ items { uint32_value: 4 } items { uint32_value: 2 } }"); +} + +void VerifyNoActiveTransactions() { + using namespace NActors::NMemory::NPrivate; + const auto& instance = TMemoryTracker::Instance(); + instance->Initialize(); + std::vector metrics; + TMemoryTracker::Instance()->GatherMetrics(metrics); + + for (size_t i : xrange(metrics.size())) { + if (instance->GetName(i) == TString(MemoryLabelActiveTransactionBody)) { + UNIT_ASSERT_VALUES_EQUAL(metrics[i].GetCount(), 0); + return; + } + } + + UNIT_ASSERT_C(false, "Datashard/TActiveTransaction/TxBody metric not found"); } Y_UNIT_TEST(TestPlannedCancelSplit) { @@ -2665,6 +2747,8 @@ Y_UNIT_TEST(TestPlannedCancelSplit) { TDuration elapsed = TInstant::Now() - splitStart; UNIT_ASSERT_C(elapsed < TDuration::Seconds(NValgrind::PlainOrUnderValgrind(2, 10)), "Split needed " << elapsed.ToString() << " to complete, which is too long"); + + VerifyNoActiveTransactions(); } Y_UNIT_TEST(TestPlannedTimeoutSplit) { @@ -2962,6 +3046,8 @@ Y_UNIT_TEST(TestReadTableWriteConflict) { runtime.SetLogPriority(NKikimrServices::TX_DATASHARD, NLog::PRI_TRACE); runtime.SetLogPriority(NKikimrServices::TX_PROXY, NLog::PRI_DEBUG); + const bool usesVolatileTxs = runtime.GetAppData(0).FeatureFlags.GetEnableDataShardVolatileTransactions(); + InitRoot(server, sender); // NOTE: table-1 has 2 shards so ReadTable is not immediate @@ -3019,15 +3105,16 @@ Y_UNIT_TEST(TestReadTableWriteConflict) { "UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2); "), sessionId, txId, true)); // Wait until we captured all readsets - if (readSets.size() < 4) { + const size_t expectedReadSets = usesVolatileTxs ? 8 : 4; + if (readSets.size() < expectedReadSets) { TDispatchOptions options; options.FinalEvents.emplace_back( [&](IEventHandle &) -> bool { - return readSets.size() >= 4; + return readSets.size() >= expectedReadSets; }); runtime.DispatchEvents(options); } - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 4u); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); captureReadSets = false; // Start reading table-1, wait for its plan step @@ -3730,9 +3817,19 @@ Y_UNIT_TEST_TWIN(TestShardRestartNoUndeterminedImmediate, StreamLookup) { auto &runtime = *server->GetRuntime(); auto sender = runtime.AllocateEdgeActor(); + const bool usesVolatileTxs = runtime.GetAppData(0).FeatureFlags.GetEnableDataShardVolatileTransactions(); + InitRoot(server, sender); - CreateShardedTable(server, sender, "/Root", "table-1", 1); + CreateShardedTable(server, sender, "/Root", "table-1", TShardedTableOptions() + .Indexes({ + // Note: async index forces read before write at the shard level, + // which causes immediate upsert to block until volatile transaction + // is resolved. + TShardedTableOptions::TIndex{ + "by_value", {"value"}, {}, NKikimrSchemeOp::EIndexTypeGlobalAsync + } + })); CreateShardedTable(server, sender, "/Root", "table-2", 1); auto table1shards = GetTableShards(server, sender, "/Root/table-1"); @@ -3791,8 +3888,9 @@ Y_UNIT_TEST_TWIN(TestShardRestartNoUndeterminedImmediate, StreamLookup) { UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))"), sessionId, txId, true)); // Wait until we captured both readsets - waitFor([&]{ return readSets.size() >= 2; }, "commit read sets"); - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + const size_t expectedReadSets = usesVolatileTxs ? 4 : 2; + waitFor([&]{ return readSets.size() >= expectedReadSets; }, "commit read sets"); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); // Now send an upsert to table-1, it should be blocked by our in-progress tx delayedProposeCount = 0; @@ -3831,12 +3929,17 @@ Y_UNIT_TEST_TWIN(TestShardRestartPlannedCommitShouldSucceed, StreamLookup) { TServerSettings serverSettings(pm.GetPort(2134)); serverSettings.SetDomainName("Root") .SetUseRealThreads(false) - .SetAppConfig(app); + .SetAppConfig(app) + // Note: currently volatile transactions don't survive tablet reboots, + // and reply with UNDETERMINED similar to immediate transactions. + .SetEnableDataShardVolatileTransactions(false); Tests::TServer::TPtr server = new TServer(serverSettings); auto &runtime = *server->GetRuntime(); auto sender = runtime.AllocateEdgeActor(); + const bool usesVolatileTxs = runtime.GetAppData(0).FeatureFlags.GetEnableDataShardVolatileTransactions(); + InitRoot(server, sender); CreateShardedTable(server, sender, "/Root", "table-1", 1); @@ -3893,8 +3996,9 @@ Y_UNIT_TEST_TWIN(TestShardRestartPlannedCommitShouldSucceed, StreamLookup) { UPSERT INTO `/Root/table-2` (key, value) VALUES (4, 2))"), sessionId, txId, true)); // Wait until we captured both readsets - waitFor([&]{ return readSets.size() >= 2; }, "commit read sets"); - UNIT_ASSERT_VALUES_EQUAL(readSets.size(), 2u); + const size_t expectedReadSets = usesVolatileTxs ? 4 : 2; + waitFor([&]{ return readSets.size() >= expectedReadSets; }, "commit read sets"); + UNIT_ASSERT_VALUES_EQUAL(readSets.size(), expectedReadSets); // Remove observer and gracefully restart the shard Cerr << "... restarting tablet" << Endl; diff --git a/ydb/core/tx/datashard/datashard_ut_snapshot.cpp b/ydb/core/tx/datashard/datashard_ut_snapshot.cpp index 338ab8660b0e..19ba5a426868 100644 --- a/ydb/core/tx/datashard/datashard_ut_snapshot.cpp +++ b/ydb/core/tx/datashard/datashard_ut_snapshot.cpp @@ -2617,6 +2617,10 @@ Y_UNIT_TEST_SUITE(DataShardSnapshots) { auto locks2 = observer.LastLocks; observer.Inject = {}; + // Note: disable volatile transactions, since this test verifies lock + // freezing and volatile transactions work without them. + runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + // Commit changes in tx 123 observer.BlockReadSets = true; observer.InjectClearTasks = true; @@ -2637,6 +2641,9 @@ Y_UNIT_TEST_SUITE(DataShardSnapshots) { observer.InjectClearTasks = false; observer.InjectLocks.reset(); + // Restore volatile transactions back to default + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); + auto writeFuture = SendRequest(runtime, MakeSimpleRequestRPC(Q_(R"( UPSERT INTO `/Root/table-1` (key, value) VALUES (2, 22) )"), writeSender, writeSenderTxId, true)); @@ -2762,6 +2769,11 @@ Y_UNIT_TEST_SUITE(DataShardSnapshots) { auto locks4 = observer.LastLocks; observer.Inject = {}; + // Note: disable volatile transactions, since reads to keys are blocked + // until readsets arrive, and this test relies on tx cross blocking, + // which doesn't happen with volatile transactions. + runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + // Commit changes in tx 123 (we expect locks to be ready for sending) observer.BlockReadSets = true; //observer.InjectClearTasks = true; @@ -2781,6 +2793,9 @@ Y_UNIT_TEST_SUITE(DataShardSnapshots) { //observer.InjectClearTasks = false; observer.InjectLocks.reset(); + // Restore volatile transactions back to default + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); + // Commit changes in tx 234 (we expect it to be blocked and broken by 123) observer.BlockReadSets = true; //observer.InjectClearTasks = true; @@ -3500,7 +3515,7 @@ Y_UNIT_TEST_SUITE(DataShardSnapshots) { ""); observer.InjectClearTasks = false; observer.InjectLocks.reset(); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); SimulateSleep(server, TDuration::Seconds(1)); @@ -3573,7 +3588,7 @@ Y_UNIT_TEST_SUITE(DataShardSnapshots) { )", volatileSessionId, "", true), "/Root"); SimulateSleep(runtime, TDuration::Seconds(1)); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); // Should be 2 expectations + 2 commit decisions UNIT_ASSERT(!upsertResult.HasValue()); diff --git a/ydb/core/tx/datashard/datashard_ut_trace.cpp b/ydb/core/tx/datashard/datashard_ut_trace.cpp index 50a5670865de..16719754f7a4 100644 --- a/ydb/core/tx/datashard/datashard_ut_trace.cpp +++ b/ydb/core/tx/datashard/datashard_ut_trace.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include #include @@ -18,9 +18,9 @@ using namespace NKikimr::NDataShard::NKqpHelpers; using namespace NSchemeShard; using namespace Tests; using namespace NDataShardReadTableTest; +using namespace NWilson; Y_UNIT_TEST_SUITE(TDataShardTrace) { - void ExecSQL(Tests::TServer::TPtr server, TActorId sender, const TString &sql, @@ -108,154 +108,6 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Record.GetRef().GetYdbStatus(), code); } - class FakeWilsonUploader : public TActorBootstrapped { - public: - class Span { - public: - Span(TString name, TString parentSpanId, ui64 startTime) : Name(name), ParentSpanId(parentSpanId), StartTime(startTime) {} - - std::optional> FindOne(TString targetName) { - for (const auto childRef : Children) { - if (childRef.get().Name == targetName) { - return childRef; - } - } - - return {}; - } - - std::vector> FindAll(TString targetName) { - std::vector> res; - - for (const auto childRef : Children) { - if (childRef.get().Name == targetName) { - res.emplace_back(childRef); - } - } - - return res; - } - - std::optional> BFSFindOne(TString targetName) { - std::queue> bfsQueue; - bfsQueue.push(std::ref(*this)); - - while (!bfsQueue.empty()) { - Span ¤tSpan = bfsQueue.front().get(); - bfsQueue.pop(); - - if (currentSpan.Name == targetName) { - return currentSpan; - } - - for (const auto childRef : currentSpan.Children) { - bfsQueue.push(childRef); - } - } - - return {}; - } - - static bool CompareByStartTime(const std::reference_wrapper& span1, const std::reference_wrapper& span2) { - return span1.get().StartTime < span2.get().StartTime; - } - - TString Name; - TString ParentSpanId; - ui64 StartTime; - std::set, decltype(&CompareByStartTime)> Children{&CompareByStartTime}; - }; - - class Trace { - public: - std::string ToString() const { - std::string result; - - for (const auto& spanPair : Spans) { - const Span& span = spanPair.second; - if (span.ParentSpanId.empty()) { - result += ToStringHelper(span); - } - } - - return result; - } - private: - std::string ToStringHelper(const Span& span) const { - std::string result = "(" + span.Name; - - if (!span.Children.empty()) { - result += " -> ["; - auto it = span.Children.begin(); - while (it != span.Children.end()) { - const Span& childSpan = it->get(); - result += ToStringHelper(childSpan); - ++it; - - if (it != span.Children.end()) { - result += " , "; - } - } - result += "]"; - } - - result += ")"; - - return result; - } - public: - std::unordered_map Spans; - - Span Root{"Root", "", 0}; - }; - - public: - void Bootstrap() { - Become(&TThis::StateFunc); - } - - void Handle(NWilson::TEvWilson::TPtr ev) { - auto& span = ev->Get()->Span; - const TString &traceId = span.trace_id(); - const TString &spanId = span.span_id(); - const TString &parentSpanId = span.parent_span_id(); - const TString &spanName = span.name(); - ui64 startTime = span.start_time_unix_nano(); - - Trace &trace = Traces[traceId]; - - trace.Spans.try_emplace(spanId, spanName, parentSpanId, startTime); - } - - void BuildTraceTrees() { - for (auto& tracePair : Traces) { - Trace& trace = tracePair.second; - - for (auto& spanPair : trace.Spans) { - Span& span = spanPair.second; - - const TString& parentSpanId = span.ParentSpanId; - - // Check if the span has a parent - if (!parentSpanId.empty()) { - auto parentSpanIt = trace.Spans.find(parentSpanId); - UNIT_ASSERT(parentSpanIt != trace.Spans.end()); - parentSpanIt->second.Children.insert(std::ref(span)); - } else { - trace.Root.Children.insert(std::ref(span)); - } - } - } - } - - STRICT_STFUNC(StateFunc, - hFunc(NWilson::TEvWilson, Handle); - ); - - public: - std::unordered_map Traces; - }; - void SplitTable(TTestActorRuntime &runtime, Tests::TServer::TPtr server, ui64 splitKey) { SetSplitMergePartCountLimit(server->GetRuntime(), -1); auto senderSplit = runtime.AllocateEdgeActor(); @@ -283,23 +135,23 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { return {runtime, server, sender}; } - void CheckTxHasWriteLog(std::reference_wrapper txSpan) { + void CheckTxHasWriteLog(std::reference_wrapper txSpan) { auto writeLogSpan = txSpan.get().FindOne("Tablet.WriteLog"); UNIT_ASSERT(writeLogSpan); auto writeLogEntrySpan = writeLogSpan->get().FindOne("Tablet.WriteLog.LogEntry"); UNIT_ASSERT(writeLogEntrySpan); } - void CheckTxHasDatashardUnits(std::reference_wrapper txSpan, ui8 count) { + void CheckTxHasDatashardUnits(std::reference_wrapper txSpan, ui8 count) { auto executeSpan = txSpan.get().FindOne("Tablet.Transaction.Execute"); UNIT_ASSERT(executeSpan); auto unitSpans = executeSpan->get().FindAll("Datashard.Unit"); - UNIT_ASSERT_EQUAL(count, unitSpans.size()); + UNIT_ASSERT_VALUES_EQUAL(count, unitSpans.size()); } - void CheckExecuteHasDatashardUnits(std::reference_wrapper executeSpan, ui8 count) { + void CheckExecuteHasDatashardUnits(std::reference_wrapper executeSpan, ui8 count) { auto unitSpans = executeSpan.get().FindAll("Datashard.Unit"); - UNIT_ASSERT_EQUAL(count, unitSpans.size()); + UNIT_ASSERT_VALUES_EQUAL(count, unitSpans.size()); } Y_UNIT_TEST(TestTraceDistributedUpsert) { @@ -307,11 +159,13 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { CreateShardedTable(server, sender, "/Root", "table-1", 1, false); - FakeWilsonUploader *uploader = new FakeWilsonUploader(); + TFakeWilsonUploader *uploader = new TFakeWilsonUploader(); TActorId uploaderId = runtime.Register(uploader, 0); runtime.RegisterService(NWilson::MakeWilsonUploaderId(), uploaderId, 0); runtime.SimulateSleep(TDuration::Seconds(10)); + const bool usesVolatileTxs = runtime.GetAppData(0).FeatureFlags.GetEnableDataShardVolatileTransactions(); + SplitTable(runtime, server, 5); NWilson::TTraceId traceId = NWilson::TTraceId::NewTraceId(15, 4095); @@ -323,11 +177,10 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { std::move(traceId) ); - uploader->BuildTraceTrees(); - + UNIT_ASSERT(uploader->BuildTraceTrees()); UNIT_ASSERT_EQUAL(1, uploader->Traces.size()); - FakeWilsonUploader::Trace &trace = uploader->Traces.begin()->second; + TFakeWilsonUploader::Trace &trace = uploader->Traces.begin()->second; auto deSpan = trace.Root.BFSFindOne("DataExecuter"); UNIT_ASSERT(deSpan); @@ -340,25 +193,44 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { UNIT_ASSERT_EQUAL(2, tabletTxs.size()); // Each shard executes a proposal tablet tx and a progress tablet tx. auto propose = tabletTxs[0]; - CheckTxHasWriteLog(propose); + // Note: when volatile transactions are enabled propose doesn't persist anything + if (!usesVolatileTxs) { + CheckTxHasWriteLog(propose); + } CheckTxHasDatashardUnits(propose, 3); auto progress = tabletTxs[1]; CheckTxHasWriteLog(progress); - CheckTxHasDatashardUnits(progress, 11); + CheckTxHasDatashardUnits(progress, usesVolatileTxs ? 6 : 11); } - - std::string canon = "(Session.query.QUERY_ACTION_EXECUTE -> [(CompileService -> [(CompileActor)]) , " - "(LiteralExecuter) , (DataExecuter -> [(WaitForTableResolve) , (RunTasks) , (Datashard.Transaction -> " - "[(Tablet.Transaction -> [(Tablet.Transaction.Execute -> [(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit)]) , " - "(Tablet.WriteLog -> [(Tablet.WriteLog.LogEntry)])]) , (Tablet.Transaction -> [(Tablet.Transaction.Execute -> " - "[(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , " - "(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit)]) , (Tablet.WriteLog -> " - "[(Tablet.WriteLog.LogEntry)])])]) , (Datashard.Transaction -> [(Tablet.Transaction -> [(Tablet.Transaction.Execute -> " - "[(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit)]) , (Tablet.WriteLog -> [(Tablet.WriteLog.LogEntry)])]) , " - "(Tablet.Transaction -> [(Tablet.Transaction.Execute -> [(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , " - "(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , " - "(Datashard.Unit) , (Datashard.Unit)]) , (Tablet.WriteLog -> [(Tablet.WriteLog.LogEntry)])])])])])"; + + std::string canon; + if (usesVolatileTxs) { + canon = "(Session.query.QUERY_ACTION_EXECUTE -> [(CompileService -> [(CompileActor)]) , " + "(LiteralExecuter) , (DataExecuter -> [(WaitForTableResolve) , (RunTasks) , (Datashard.Transaction -> " + "[(Tablet.Transaction -> [(Tablet.Transaction.Execute -> [(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit)])" + "]) , (Tablet.Transaction -> [(Tablet.Transaction.Execute -> " + "[(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit)" + "]) , (Tablet.WriteLog -> " + "[(Tablet.WriteLog.LogEntry)])])]) , (Datashard.Transaction -> [(Tablet.Transaction -> [(Tablet.Transaction.Execute -> " + "[(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit)])]) , " + "(Tablet.Transaction -> [(Tablet.Transaction.Execute -> [(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , " + "(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit)" + "]) , (Tablet.WriteLog -> [(Tablet.WriteLog.LogEntry)])])])])])"; + } else { + canon = "(Session.query.QUERY_ACTION_EXECUTE -> [(CompileService -> [(CompileActor)]) , " + "(LiteralExecuter) , (DataExecuter -> [(WaitForTableResolve) , (RunTasks) , (Datashard.Transaction -> " + "[(Tablet.Transaction -> [(Tablet.Transaction.Execute -> [(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit)]) , " + "(Tablet.WriteLog -> [(Tablet.WriteLog.LogEntry)])]) , (Tablet.Transaction -> [(Tablet.Transaction.Execute -> " + "[(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , " + "(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit)]) , (Tablet.WriteLog -> " + "[(Tablet.WriteLog.LogEntry)])])]) , (Datashard.Transaction -> [(Tablet.Transaction -> [(Tablet.Transaction.Execute -> " + "[(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit)]) , (Tablet.WriteLog -> [(Tablet.WriteLog.LogEntry)])]) , " + "(Tablet.Transaction -> [(Tablet.Transaction.Execute -> [(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , " + "(Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , (Datashard.Unit) , " + "(Datashard.Unit) , (Datashard.Unit)]) , (Tablet.WriteLog -> [(Tablet.WriteLog.LogEntry)])])])])])"; + } + UNIT_ASSERT_VALUES_EQUAL(canon, trace.ToString()); } @@ -367,7 +239,7 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { CreateShardedTable(server, sender, "/Root", "table-1", 1, false); - FakeWilsonUploader *uploader = new FakeWilsonUploader(); + TFakeWilsonUploader *uploader = new TFakeWilsonUploader(); TActorId uploaderId = runtime.Register(uploader, 0); runtime.RegisterService(NWilson::MakeWilsonUploaderId(), uploaderId, 0); runtime.SimulateSleep(TDuration::Seconds(10)); @@ -414,11 +286,10 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { std::move(traceId) ); - uploader->BuildTraceTrees(); - + UNIT_ASSERT(uploader->BuildTraceTrees()); UNIT_ASSERT_EQUAL(1, uploader->Traces.size()); - FakeWilsonUploader::Trace &trace = uploader->Traces.begin()->second; + TFakeWilsonUploader::Trace &trace = uploader->Traces.begin()->second; std::string canon; if (server->GetSettings().AppConfig->GetTableServiceConfig().GetEnableKqpDataQueryStreamLookup()) { @@ -491,7 +362,7 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { CreateShardedTable(server, sender, "/Root", "table-1", 1, false); - FakeWilsonUploader* uploader = new FakeWilsonUploader(); + TFakeWilsonUploader* uploader = new TFakeWilsonUploader(); TActorId uploaderId = runtime.Register(uploader, 0); runtime.RegisterService(NWilson::MakeWilsonUploaderId(), uploaderId, 0); runtime.SimulateSleep(TDuration::Seconds(10)); @@ -522,11 +393,10 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { std::move(traceId) ); - uploader->BuildTraceTrees(); - + UNIT_ASSERT(uploader->BuildTraceTrees()); UNIT_ASSERT_EQUAL(1, uploader->Traces.size()); - FakeWilsonUploader::Trace& trace = uploader->Traces.begin()->second; + TFakeWilsonUploader::Trace& trace = uploader->Traces.begin()->second; auto readActorSpan = trace.Root.BFSFindOne("ReadActor"); UNIT_ASSERT(readActorSpan); @@ -551,7 +421,7 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { auto opts = TShardedTableOptions().Columns({{"key", "Uint32", true, false}, {"value", "Uint32", false, false}}); auto [shards, tableId] = CreateShardedTable(server, sender, "/Root", "table-1", opts); - FakeWilsonUploader *uploader = new FakeWilsonUploader(); + TFakeWilsonUploader *uploader = new TFakeWilsonUploader(); TActorId uploaderId = runtime.Register(uploader, 0); runtime.RegisterService(NWilson::MakeWilsonUploaderId(), uploaderId, 0); runtime.SimulateSleep(TDuration::Seconds(10)); @@ -561,11 +431,10 @@ Y_UNIT_TEST_SUITE(TDataShardTrace) { ui64 txId = 100; Write(runtime, sender, shards[0], tableId, opts.Columns_, rowCount, txId, NKikimrDataEvents::TEvWrite::MODE_IMMEDIATE, NKikimrDataEvents::TEvWriteResult::STATUS_UNSPECIFIED, std::move(traceId)); - uploader->BuildTraceTrees(); - + UNIT_ASSERT(uploader->BuildTraceTrees()); UNIT_ASSERT_EQUAL(1, uploader->Traces.size()); - FakeWilsonUploader::Trace &trace = uploader->Traces.begin()->second; + TFakeWilsonUploader::Trace &trace = uploader->Traces.begin()->second; auto wtSpan = trace.Root.BFSFindOne("Datashard.WriteTransaction"); UNIT_ASSERT(wtSpan); diff --git a/ydb/core/tx/datashard/datashard_ut_volatile.cpp b/ydb/core/tx/datashard/datashard_ut_volatile.cpp index 91e5adec17cc..31b6456c8a5c 100644 --- a/ydb/core/tx/datashard/datashard_ut_volatile.cpp +++ b/ydb/core/tx/datashard/datashard_ut_volatile.cpp @@ -46,7 +46,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { )"); Cerr << "!!! distributed write end" << Endl; - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); UNIT_ASSERT_VALUES_EQUAL( KqpSimpleExec(runtime, R"( @@ -124,7 +124,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { )"), "ERROR: ABORTED"); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); // Verify transaction was not committed UNIT_ASSERT_VALUES_EQUAL( @@ -204,7 +204,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { "ERROR: UNDETERMINED"); Cerr << "!!! distributed write end" << Endl; - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); // Verify transaction was not committed UNIT_ASSERT_VALUES_EQUAL( @@ -292,7 +292,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { "ERROR: UNDETERMINED"); Cerr << "!!! distributed write end" << Endl; - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); // Verify transaction was not committed UNIT_ASSERT_VALUES_EQUAL( @@ -368,7 +368,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { WaitFor(runtime, [&]{ return capturedReadSets.size() >= 4; }, "captured readsets"); runtime.SetObserverFunc(prevObserverFunc); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); // Make sure snapshot transaction cannot see uncommitted changes and doesn't block on them UNIT_ASSERT_VALUES_EQUAL( @@ -436,7 +436,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { WaitFor(runtime, [&]{ return capturedReadSets.size() >= 4; }, "captured readsets"); runtime.SetObserverFunc(prevObserverFunc); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); TString sessionIdSnapshot = CreateSessionRPC(runtime, "/Root"); auto snapshotReadFuture = SendRequest(runtime, MakeSimpleRequestRPC(R"( @@ -533,7 +533,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { UNIT_ASSERT_VALUES_EQUAL(capturedPlans.size(), 1u); runtime.SetObserverFunc(prevObserverFunc); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); // Start reading from table-2 TString sessionIdSnapshot = CreateSessionRPC(runtime, "/Root"); @@ -619,7 +619,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { WaitFor(runtime, [&]{ return observedPlans >= 2; }, "observed plans"); UNIT_ASSERT_VALUES_EQUAL(capturedPlans.size(), 1u); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); // Wait until it completes at shard2 SimulateSleep(runtime, TDuration::Seconds(1)); @@ -690,7 +690,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { WaitFor(runtime, [&]{ return capturedReadSets.size() >= 4; }, "captured readsets"); UNIT_ASSERT_VALUES_EQUAL(capturedReadSets.size(), 4u); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); observedPropose = 0; ui64 txId = AsyncDropTable(server, sender, "/Root", "table-1"); @@ -763,7 +763,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { WaitFor(runtime, [&]{ return capturedReadSets.size() >= 4; }, "captured readsets"); UNIT_ASSERT_VALUES_EQUAL(capturedReadSets.size(), 4u); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); // Note: this upsert happens over the upsert into the value column ExecSQL(server, sender, "UPSERT INTO `/Root/table-1` (key, value2) VALUES (2, 51);"); @@ -850,7 +850,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { WaitFor(runtime, [&]{ return capturedReadSets.size() >= 4; }, "captured readsets"); UNIT_ASSERT_VALUES_EQUAL(capturedReadSets.size(), 4u); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); observedPropose = 0; ui64 txId = AsyncCreateCopyTable(server, sender, "/Root", "table-1-copy", "/Root/table-1"); @@ -930,7 +930,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { WaitFor(runtime, [&]{ return capturedReadSets.size() >= 4; }, "captured readsets"); UNIT_ASSERT_VALUES_EQUAL(capturedReadSets.size(), 4u); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); SetSplitMergePartCountLimit(server->GetRuntime(), -1); auto shards1before = GetTableShards(server, sender, "/Root/table-1"); @@ -1017,7 +1017,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { WaitFor(runtime, [&]{ return capturedReadSets.size() >= 4; }, "captured readsets"); UNIT_ASSERT_VALUES_EQUAL(capturedReadSets.size(), 4u); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); const auto shard1 = GetTableShards(server, sender, "/Root/table-1").at(0); const auto tableId1 = ResolveTableId(server, sender, "/Root/table-1"); @@ -1147,7 +1147,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { WaitFor(runtime, [&]{ return capturedReadSets.size() >= 4; }, "captured readsets"); UNIT_ASSERT_VALUES_EQUAL(capturedReadSets.size(), 4u); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); const auto shard1 = GetTableShards(server, sender, "/Root/table-1").at(0); const auto tableId1 = ResolveTableId(server, sender, "/Root/table-1"); @@ -1298,7 +1298,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { WaitFor(runtime, [&]{ return capturedReadSets.size() >= 4; }, "captured readsets"); UNIT_ASSERT_VALUES_EQUAL(capturedReadSets.size(), 4u); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); std::vector observedResults; TMaybe observedStatus; @@ -1392,7 +1392,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { UPSERT INTO `/Root/table-1` (key, value) VALUES (2, 3); UPSERT INTO `/Root/table-2` (key, value) VALUES (20, 30); )"); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); // Make sure changes are actually delivered SimulateSleep(runtime, TDuration::Seconds(1)); @@ -1476,7 +1476,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { WaitFor(runtime, [&]{ return capturedReadSets.size() >= 4; }, "captured readsets"); UNIT_ASSERT_VALUES_EQUAL(capturedReadSets.size(), 4u); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); // Write to key 2 using bulk upsert NThreading::TFuture bulkUpsertFuture; @@ -1638,7 +1638,7 @@ Y_UNIT_TEST_SUITE(DataShardVolatile) { WaitFor(runtime, [&]{ return capturedReadSets.size() >= 4; }, "captured readsets"); UNIT_ASSERT_VALUES_EQUAL(capturedReadSets.size(), 4u); - runtime.GetAppData(0).FeatureFlags.SetEnableDataShardVolatileTransactions(false); + runtime.GetAppData(0).FeatureFlags.ClearEnableDataShardVolatileTransactions(); // Write to key 2 using bulk upsert NThreading::TFuture bulkUpsertFuture; diff --git a/ydb/core/tx/schemeshard/schemeshard_impl.cpp b/ydb/core/tx/schemeshard/schemeshard_impl.cpp index 861d2ac453f1..55026ab3b282 100644 --- a/ydb/core/tx/schemeshard/schemeshard_impl.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_impl.cpp @@ -7052,7 +7052,18 @@ void TSchemeShard::SendBaseStatsToSA() { entry->SetBytesSize(aggregated.DataSize); ++count; } - // TODO: add column tables + auto columnTablesPathIds = ColumnTables.GetAllPathIds(); + for (const auto& pathId : columnTablesPathIds) { + const auto& tableInfo = ColumnTables.GetVerified(pathId); + const auto& aggregated = tableInfo->Stats.Aggregated; + auto* entry = record.AddEntries(); + auto* entryPathId = entry->MutablePathId(); + entryPathId->SetOwnerId(pathId.OwnerId); + entryPathId->SetLocalId(pathId.LocalPathId); + entry->SetRowCount(aggregated.RowCount); + entry->SetBytesSize(aggregated.DataSize); + ++count; + } TString stats; stats.clear(); diff --git a/ydb/core/tx/schemeshard/schemeshard_tables_storage.cpp b/ydb/core/tx/schemeshard/schemeshard_tables_storage.cpp index d8899d4dc208..63d3ed203226 100644 --- a/ydb/core/tx/schemeshard/schemeshard_tables_storage.cpp +++ b/ydb/core/tx/schemeshard/schemeshard_tables_storage.cpp @@ -148,4 +148,12 @@ bool TTablesStorage::TTableCreateOperator::InitShardingTablets(const TColumnTabl return true; } +std::unordered_set TTablesStorage::GetAllPathIds() const { + std::unordered_set result; + for (const auto& [pathId, _] : Tables) { + result.emplace(pathId); + } + return result; +} + } diff --git a/ydb/core/tx/schemeshard/schemeshard_tables_storage.h b/ydb/core/tx/schemeshard/schemeshard_tables_storage.h index 26ce7daf42f3..036051295983 100644 --- a/ydb/core/tx/schemeshard/schemeshard_tables_storage.h +++ b/ydb/core/tx/schemeshard/schemeshard_tables_storage.h @@ -159,6 +159,7 @@ class TTablesStorage { void OnRemoveObject(const TPathId& pathId, TColumnTableInfo::TPtr object); TColumnTableInfo::TPtr ExtractPtr(const TPathId& id); public: + std::unordered_set GetAllPathIds() const; TColumnTablesLayout GetTablesLayout(const std::vector& tabletIds) const; diff --git a/ydb/docs/en/core/integrations/goose.md b/ydb/docs/en/core/integrations/goose.md index 4c7367bbe6e8..2d5de55e351a 100644 --- a/ydb/docs/en/core/integrations/goose.md +++ b/ydb/docs/en/core/integrations/goose.md @@ -166,7 +166,7 @@ There are alternative options to see the applied changes: - Using YDB UI on http://localhost:8765 - ![YDB UI after the first migration](../_assets/goose-ydb-ui-after-first-migration.png) + ![YDB UI after the first migration](../_assets/goose-ydb-ui-after-first-migration.png =450x) - Using YDB CLI @@ -258,7 +258,7 @@ Let's use the same methods to see the new changes: - Using YDB UI on http://localhost:8765 - ![YDB UI after apply second migration](../_assets/goose-ydb-ui-after-second-migration.png) + ![YDB UI after apply second migration](../_assets/goose-ydb-ui-after-second-migration.png =450x) - Using YDB CLI @@ -322,7 +322,7 @@ Let's check the changes again: - Using YDB UI on http://localhost:8765 - ![YDB UI after apply first migration](../_assets/goose-ydb-ui-after-first-migration.png) + ![YDB UI after apply first migration](../_assets/goose-ydb-ui-after-first-migration.png =450x) - Using YDB CLI diff --git a/ydb/docs/en/core/postgresql/docker-connect.md b/ydb/docs/en/core/postgresql/docker-connect.md index c5ff9108d05b..e5931b75c397 100644 --- a/ydb/docs/en/core/postgresql/docker-connect.md +++ b/ydb/docs/en/core/postgresql/docker-connect.md @@ -32,7 +32,7 @@ To preserve the container's state, you need to remove the environment variable ` - "YDB_USE_IN_MEMORY_PDISKS=true" - "POSTGRES_USER=${YDB_PG_USER:-root}" - "POSTGRES_PASSWORD=${YDB_PG_PASSWORD:-1234}" - - "YDB_FEATURE_FLAGS=enable_temp_tables" + - "YDB_FEATURE_FLAGS=enable_temp_tables,enable_table_pg_types" - "YDB_TABLE_ENABLE_PREPARED_DDL=true" ``` diff --git a/ydb/docs/en/core/reference/kafka-api/constraints.md b/ydb/docs/en/core/reference/kafka-api/constraints.md new file mode 100644 index 000000000000..6d4a5a69d116 --- /dev/null +++ b/ydb/docs/en/core/reference/kafka-api/constraints.md @@ -0,0 +1,17 @@ +# Kafka API constraints + +YDB supports [Apache Kafka protocol](https://kafka.apache.org/protocol.html) version 3.4.0 with the following constraints: + +1. Only authenticated connections are allowed. + +2. Only `SASL/PLAIN` authentication method is supported. + +3. Message compression is not supported. + +4. Only Manual Partition Assignment is supported in read mode, using the [assign method](https://kafka.apache.org/35/javadoc/org/apache/kafka/clients/consumer/KafkaConsumer.html#assign(java.util.Collection)). Consumer group partitions can't be used. + +5. Transactions are not supported. + +6. DDL operations are not supported. Use the [YDB SDK](../ydb-sdk/index.md) or [YDB CLI](../ydb-cli/index.md) to perform them. + +7. Data schema validation not supported. \ No newline at end of file diff --git a/ydb/docs/en/core/reference/kafka-api/examples.md b/ydb/docs/en/core/reference/kafka-api/examples.md new file mode 100644 index 000000000000..1a338b55d79e --- /dev/null +++ b/ydb/docs/en/core/reference/kafka-api/examples.md @@ -0,0 +1,128 @@ +# Kafka API usage examples + +This article provides examples of Kafka API usage to work with [{{ ydb-short-name }} topics](../../concepts/topic.md). + + +Before executing the examples, [create a topic](../ydb-cli/topic-create.md) and [add a consumer](../ydb-cli/topic-consumer-add.md). + +## Examples of working with topics + +The examples use: + + * `ydb:9093` — host name. + * `/Root/Database` — database name. + * `/Root/Database/Topic` — topic name. + * `user@/Root/Database` — username. Includes the username and database name. + * `*****` — user password. + + +## Writing data to a topic + +### Writing via Kafka Java SDK + +This example includes a code snippet for writing data to a topic via [Kafka API](https://kafka.apache.org/documentation/). + + ```java + String HOST = "ydb:9093"; + String TOPIC = "/Root/Database/Topic"; + String USER = "user@/Root/Database"; + String PASS = "*****"; + + Properties props = new Properties(); + props.put("bootstrap.servers", HOST); + props.put("acks", "all"); + + props.put("key.serializer", StringSerializer.class.getName()); + props.put("key.deserializer", StringDeserializer.class.getName()); + props.put("value.serializer", StringSerializer.class.getName()); + props.put("value.deserializer", StringDeserializer.class.getName()); + + props.put("security.protocol", "SASL_SSL"); + props.put("sasl.mechanism", "PLAIN"); + props.put("sasl.jaas.config", PlainLoginModule.class.getName() + " required username=\"" + USER + "\" password=\"" + PASS + "\";"); + + props.put("compression.type", "none"); + + Producer producer = new KafkaProducer<>(props); + producer.send(new ProducerRecord(TOPIC, "msg-key", "msg-body")); + producer.flush(); + producer.close(); + ``` + +### Writing via Logstash + +To configure [Logstash](https://github.com/elastic/logstash), use the following parameters: + + ``` + output { + kafka { + codec => json + topic_id => "/Root/Database/Topic" + bootstrap_servers => "ydb:9093" + compression_type => none + security_protocol => SASL_SSL + sasl_mechanism => PLAIN + sasl_jaas_config => "org.apache.kafka.common.security.plain.PlainLoginModule required username='user@/Root/Database' password='*****';" + } + } + ``` + +### Writing via Fluent Bit + +To configure [Fluent Bit](https://github.com/fluent/fluent-bit), use the following parameters: + + ``` + [OUTPUT] + name kafka + match * + Brokers ydb:9093 + Topics /Root/Database/Topic + rdkafka.client.id Fluent-bit + rdkafka.request.required.acks 1 + rdkafka.log_level 7 + rdkafka.security.protocol SASL_SSL + rdkafka.sasl.mechanism PLAIN + rdkafka.sasl.username user@/Root/Database + rdkafka.sasl.password ***** + ``` + +## Reading data from a topic + +### Reading data from a topic via Kafka Java SDK + +This example includes a code snippet for reading data from a topic via Kafka Java SDK. + +```java + String HOST = "ydb:9093"; + String TOPIC = "/Root/Database/Topic"; + String USER = "user@/Root/Database"; + String PASS = "*****"; + + Properties props = new Properties(); + props.put("bootstrap.servers", HOST); + props.put("auto.offset.reset", "earliest"); // to read from start + props.put("check.crcs", false); + + props.put("key.deserializer", StringDeserializer.class.getName()); + props.put("value.deserializer", StringDeserializer.class.getName()); + + props.put("security.protocol", "SASL_SSL"); + props.put("sasl.mechanism", "PLAIN"); + props.put("sasl.jaas.config", PlainLoginModule.class.getName() + " required username=\"" + USER + "\" password=\"" + PASS + "\";"); + + Consumer consumer = new KafkaConsumer<>(props); + + List partitionInfos = consumer.partitionsFor(TOPIC); + List topicPartitions = new ArrayList<>(); + + for (PartitionInfo partitionInfo : partitionInfos) { + topicPartitions.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition())); + } + consumer.assign(topicPartitions); + + while (true) { + ConsumerRecords records = consumer.poll(1000); + for (ConsumerRecord record : records) { + System.out.println(record.key() + ":" + record.value()); + } + } \ No newline at end of file diff --git a/ydb/docs/en/core/reference/kafka-api/index.md b/ydb/docs/en/core/reference/kafka-api/index.md new file mode 100644 index 000000000000..64c9da26fc97 --- /dev/null +++ b/ydb/docs/en/core/reference/kafka-api/index.md @@ -0,0 +1,8 @@ +# Kafka API + +{{ ydb-short-name }} supports working with [topics](../../concepts/topic.md) using [the Kafka protocol version 3.4.0](https://kafka.apache.org/34/documentation.html). It allows to integrate {{ ydb-short-name }} with applications originally developed to work with [Apache Kafka](https://kafka.apache.org/). + +The Kafka API documentation contains the following sections: + +* [Usage examples](examples.md) +* [Constraints](constraints.md) diff --git a/ydb/docs/en/core/reference/kafka-api/toc_i.yaml b/ydb/docs/en/core/reference/kafka-api/toc_i.yaml new file mode 100644 index 000000000000..d0ef66df1eab --- /dev/null +++ b/ydb/docs/en/core/reference/kafka-api/toc_i.yaml @@ -0,0 +1,5 @@ +items: + - name: Usage examples + href: examples.md + - name: Constraints + href: constraints.md \ No newline at end of file diff --git a/ydb/docs/en/core/reference/kafka-api/toc_p.yaml b/ydb/docs/en/core/reference/kafka-api/toc_p.yaml new file mode 100644 index 000000000000..bb200f7ddbab --- /dev/null +++ b/ydb/docs/en/core/reference/kafka-api/toc_p.yaml @@ -0,0 +1,4 @@ +items: +- name: Overview + href: index.md +- include: { mode: link, path: toc_i.yaml } \ No newline at end of file diff --git a/ydb/docs/en/core/toc_i.yaml b/ydb/docs/en/core/toc_i.yaml index 53c6b335068c..294e668d9aaf 100644 --- a/ydb/docs/en/core/toc_i.yaml +++ b/ydb/docs/en/core/toc_i.yaml @@ -26,6 +26,7 @@ items: - { name: Integrations, include: { mode: link, path: integrations/toc_p.yaml } } - { name: Working with the YDB CLI, include: { mode: link, path: reference/ydb-cli/toc_p.yaml } } - { name: Working with the YDB SDK, include: { mode: link, path: reference/ydb-sdk/toc_p.yaml } } +- { name: Working with the Kafka API, include: { mode: link, path: reference/kafka-api/toc_p.yaml } } - { name: Development, include: { mode: link, path: development/toc_p.yaml } } # Footer - { name: Questions and answers, include: { mode: link, path: faq/toc_p.yaml } } diff --git a/ydb/docs/ru/core/integrations/goose.md b/ydb/docs/ru/core/integrations/goose.md index edc719df6563..7afeb6e98eee 100644 --- a/ydb/docs/ru/core/integrations/goose.md +++ b/ydb/docs/ru/core/integrations/goose.md @@ -157,7 +157,7 @@ $ goose ydb $YDB_CONNECTION_STRING status - Используя YDB UI по адресу http://localhost:8765 - ![YDB UI after apply first migration](../_assets/goose-ydb-ui-after-first-migration.png) + ![YDB UI after apply first migration](../_assets/goose-ydb-ui-after-first-migration.png =450x) - Используя YDB CLI @@ -247,7 +247,7 @@ $ goose ydb $YDB_CONNECTION_STRING status - Используя YDB UI по адресу http://localhost:8765 - ![YDB UI after apply second migration](../_assets/goose-ydb-ui-after-second-migration.png) + ![YDB UI after apply second migration](../_assets/goose-ydb-ui-after-second-migration.png =450x) - Используя YDB CLI @@ -309,7 +309,7 @@ $ goose ydb $YDB_CONNECTION_STRING status - Используя YDB UI по адресу http://localhost:8765 - ![YDB UI after apply first migration](../_assets/goose-ydb-ui-after-first-migration.png) + ![YDB UI after apply first migration](../_assets/goose-ydb-ui-after-first-migration.png =450x) - Используя YDB CLI diff --git a/ydb/docs/ru/core/postgresql/docker-connect.md b/ydb/docs/ru/core/postgresql/docker-connect.md index a102193dc997..1294a807af8c 100644 --- a/ydb/docs/ru/core/postgresql/docker-connect.md +++ b/ydb/docs/ru/core/postgresql/docker-connect.md @@ -32,7 +32,7 @@ - "YDB_USE_IN_MEMORY_PDISKS=true" - "POSTGRES_USER=${YDB_PG_USER:-root}" - "POSTGRES_PASSWORD=${YDB_PG_PASSWORD:-1234}" - - "YDB_FEATURE_FLAGS=enable_temp_tables" + - "YDB_FEATURE_FLAGS=enable_temp_tables,enable_table_pg_types" - "YDB_TABLE_ENABLE_PREPARED_DDL=true" ``` diff --git a/ydb/docs/ru/core/reference/kafka-api/constraints.md b/ydb/docs/ru/core/reference/kafka-api/constraints.md index ba1b1680ca0c..3e0fbed72862 100644 --- a/ydb/docs/ru/core/reference/kafka-api/constraints.md +++ b/ydb/docs/ru/core/reference/kafka-api/constraints.md @@ -1,3 +1,5 @@ +# Ограничения Kafka API + Поддержка протокола Kafka версии 3.4.0 осуществляется в ограниченном объеме: 1. Разрешены только аутентифицированные подключения. diff --git a/ydb/docs/ru/core/reference/kafka-api/examples.md b/ydb/docs/ru/core/reference/kafka-api/examples.md index 3c6ea56c9fe7..53770e24f08d 100644 --- a/ydb/docs/ru/core/reference/kafka-api/examples.md +++ b/ydb/docs/ru/core/reference/kafka-api/examples.md @@ -19,7 +19,7 @@ ### Запись через Kafka Java SDK -В этом примере приведен фрагмент кода для записи в топик через Kafka API. +В этом примере приведен фрагмент кода для записи в топик через [Kafka API](https://kafka.apache.org/documentation/). ```java String HOST = "ydb:9093"; diff --git a/ydb/docs/ru/core/reference/kafka-api/index.md b/ydb/docs/ru/core/reference/kafka-api/index.md index b55d7c372681..d265017a8885 100644 --- a/ydb/docs/ru/core/reference/kafka-api/index.md +++ b/ydb/docs/ru/core/reference/kafka-api/index.md @@ -2,6 +2,6 @@ YDB поддерживает работу с топиками по протоколу [Kafka версия 3.4.0](https://kafka.apache.org/34/documentation.html). -[Ограничения использования Kafka API](constraints.md) +[Примеры использования Kafka API](examples.md) -[Примеры использования Kafka API](examples.md) \ No newline at end of file +[Ограничения использования Kafka API](constraints.md) \ No newline at end of file diff --git a/ydb/docs/ru/core/reference/kafka-api/toc_i.yaml b/ydb/docs/ru/core/reference/kafka-api/toc_i.yaml index 978f1c136c2d..7ca3b1f722d6 100644 --- a/ydb/docs/ru/core/reference/kafka-api/toc_i.yaml +++ b/ydb/docs/ru/core/reference/kafka-api/toc_i.yaml @@ -1,5 +1,5 @@ items: - - name: Ограничения - href: constraints.md - name: Примеры использования - href: examples.md \ No newline at end of file + href: examples.md + - name: Ограничения + href: constraints.md \ No newline at end of file diff --git a/ydb/library/actors/util/memory_tracker_ut.cpp b/ydb/library/actors/util/memory_tracker_ut.cpp index 1b8eff7cc5c8..b8d8993ec0b4 100644 --- a/ydb/library/actors/util/memory_tracker_ut.cpp +++ b/ydb/library/actors/util/memory_tracker_ut.cpp @@ -190,7 +190,7 @@ struct TTracked template double MeasureAllocations() { - constexpr size_t objectsCount = 4 << 20; + constexpr size_t objectsCount = 1 << 20; std::vector objects; objects.resize(objectsCount); @@ -213,7 +213,7 @@ double MeasureAllocations() { Y_UNIT_TEST(Performance) { TMemoryTracker::Instance()->Initialize(); - constexpr size_t Runs = 16; + constexpr size_t Runs = 8; Cerr << "---- warmup" << Endl; MeasureAllocations(); diff --git a/ydb/library/actors/wilson/test_util/fake_wilson_uploader.h b/ydb/library/actors/wilson/test_util/fake_wilson_uploader.h new file mode 100644 index 000000000000..9afc2a2e68dc --- /dev/null +++ b/ydb/library/actors/wilson/test_util/fake_wilson_uploader.h @@ -0,0 +1,164 @@ +#pragma once + +#include +#include +#include + +namespace NWilson { + + class TFakeWilsonUploader : public NActors::TActorBootstrapped { + public: + class Span { + public: + Span(TString name, TString parentSpanId, ui64 startTime) : Name(name), ParentSpanId(parentSpanId), StartTime(startTime) {} + + std::optional> FindOne(TString targetName) { + for (const auto childRef : Children) { + if (childRef.get().Name == targetName) { + return childRef; + } + } + + return {}; + } + + std::vector> FindAll(TString targetName) { + std::vector> res; + + for (const auto childRef : Children) { + if (childRef.get().Name == targetName) { + res.emplace_back(childRef); + } + } + + return res; + } + + std::optional> BFSFindOne(TString targetName) { + std::queue> bfsQueue; + bfsQueue.push(std::ref(*this)); + + while (!bfsQueue.empty()) { + Span ¤tSpan = bfsQueue.front().get(); + bfsQueue.pop(); + + if (currentSpan.Name == targetName) { + return currentSpan; + } + + for (const auto childRef : currentSpan.Children) { + bfsQueue.push(childRef); + } + } + + return {}; + } + + static bool CompareByStartTime(const std::reference_wrapper& span1, const std::reference_wrapper& span2) { + return span1.get().StartTime < span2.get().StartTime; + } + + TString Name; + TString ParentSpanId; + ui64 StartTime; + std::set, decltype(&CompareByStartTime)> Children{&CompareByStartTime}; + }; + + class Trace { + public: + std::string ToString() const { + std::string result; + + for (const auto& spanPair : Spans) { + const Span& span = spanPair.second; + if (span.ParentSpanId.empty()) { + result += ToStringHelper(span); + } + } + + return result; + } + private: + std::string ToStringHelper(const Span& span) const { + std::string result = "(" + span.Name; + + if (!span.Children.empty()) { + result += " -> ["; + auto it = span.Children.begin(); + while (it != span.Children.end()) { + const Span& childSpan = it->get(); + result += ToStringHelper(childSpan); + ++it; + + if (it != span.Children.end()) { + result += " , "; + } + } + result += "]"; + } + + result += ")"; + + return result; + } + public: + std::unordered_map Spans; + + Span Root{"Root", "", 0}; + }; + + public: + void Bootstrap() { + Become(&TThis::StateFunc); + } + + void Handle(NWilson::TEvWilson::TPtr ev) { + auto& span = ev->Get()->Span; + const TString &traceId = span.trace_id(); + const TString &spanId = span.span_id(); + const TString &parentSpanId = span.parent_span_id(); + const TString &spanName = span.name(); + ui64 startTime = span.start_time_unix_nano(); + + Trace &trace = Traces[traceId]; + + trace.Spans.try_emplace(spanId, spanName, parentSpanId, startTime); + } + + [[nodiscard]] bool BuildTraceTrees() { + for (auto& tracePair : Traces) { + Trace& trace = tracePair.second; + + for (auto& spanPair : trace.Spans) { + Span& span = spanPair.second; + + const TString& parentSpanId = span.ParentSpanId; + + // Check if the span has a parent + if (!parentSpanId.empty()) { + auto parentSpanIt = trace.Spans.find(parentSpanId); + if (parentSpanIt == trace.Spans.end()) { + return false; + } + parentSpanIt->second.Children.insert(std::ref(span)); + } else { + trace.Root.Children.insert(std::ref(span)); + } + } + } + return true; + } + + void Clear() { + Traces.clear(); + } + + STRICT_STFUNC(StateFunc, + hFunc(NWilson::TEvWilson, Handle); + ); + + public: + std::unordered_map Traces; + }; + +} // NWilson diff --git a/ydb/library/actors/wilson/test_util/ya.make b/ydb/library/actors/wilson/test_util/ya.make new file mode 100644 index 000000000000..4ff8f8c26e86 --- /dev/null +++ b/ydb/library/actors/wilson/test_util/ya.make @@ -0,0 +1,7 @@ +LIBRARY() + +SRCS( + fake_wilson_uploader.h +) + +END() diff --git a/ydb/library/actors/wilson/ya.make b/ydb/library/actors/wilson/ya.make index 9786754fae78..07524c01b151 100644 --- a/ydb/library/actors/wilson/ya.make +++ b/ydb/library/actors/wilson/ya.make @@ -22,4 +22,5 @@ RECURSE( RECURSE_FOR_TESTS( ut + test_util ) diff --git a/ydb/library/yql/cfg/tests/gateways-experimental.conf b/ydb/library/yql/cfg/tests/gateways-experimental.conf index 857a7f71d4c9..e9fc38e79b79 100644 --- a/ydb/library/yql/cfg/tests/gateways-experimental.conf +++ b/ydb/library/yql/cfg/tests/gateways-experimental.conf @@ -32,3 +32,9 @@ Dq { } } +YqlCore { + Flags { + Name: "OptimizerFlags" + Args: ["FieldSubsetEnableMultiusage"] + } +} diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json index 1c65ddc67b21..724e69d859a0 100644 --- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json +++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json @@ -1304,6 +1304,11 @@ "Base": "TCoUnaryArithmetic", "Match": {"Type": "Callable", "Name": "Abs"} }, + { + "Name": "TCoSize", + "Base": "TCoUnaryArithmetic", + "Match": {"Type": "Callable", "Name": "Size"} + }, { "Name": "TCoBinaryArithmetic", "Base": "TCallable", diff --git a/ydb/library/yql/parser/pg_wrapper/arrow.cpp b/ydb/library/yql/parser/pg_wrapper/arrow.cpp index 5a8559ff8f2d..0bb21f5706c2 100644 --- a/ydb/library/yql/parser/pg_wrapper/arrow.cpp +++ b/ydb/library/yql/parser/pg_wrapper/arrow.cpp @@ -276,6 +276,10 @@ TColumnConverter BuildPgColumnConverter(const std::shared_ptr& return [](const std::shared_ptr& value) { return PgConvertFixed(value, [](auto value){ return MakePgDateFromUint16(value); }); }; + } else if (originalType->Equals(arrow::date32())) { + return [](const std::shared_ptr& value) { + return PgConvertFixed(value, [](auto value){ return MakePgDateFromUint16(value); }); + }; } else { return {}; } diff --git a/ydb/library/yql/parser/pg_wrapper/arrow_impl.h b/ydb/library/yql/parser/pg_wrapper/arrow_impl.h index 7574aada429f..62daeea279a7 100644 --- a/ydb/library/yql/parser/pg_wrapper/arrow_impl.h +++ b/ydb/library/yql/parser/pg_wrapper/arrow_impl.h @@ -2,6 +2,7 @@ #include #include +#include extern "C" { #include "utils/numeric.h" @@ -10,6 +11,7 @@ extern "C" { namespace NYql { Numeric PgFloatToNumeric(double item, ui64 scale, int digits); +TColumnConverter BuildPgColumnConverter(const std::shared_ptr& originalType, NKikimr::NMiniKQL::TPgType* targetType); template std::shared_ptr PgConvertNumeric(const std::shared_ptr& value) { diff --git a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp index 84f03a737d81..62ff4aa5c1a6 100644 --- a/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp +++ b/ydb/library/yql/parser/pg_wrapper/comp_factory.cpp @@ -462,6 +462,14 @@ class TReturnSetInfo { } ~TReturnSetInfo() { + Free(); + } + + void Free() { + if (!Ptr) { + return; + } + if (Ref().expectedDesc) { FreeTupleDesc(Ref().expectedDesc); } @@ -471,9 +479,12 @@ class TReturnSetInfo { } TWithDefaultMiniKQLAlloc::FreeWithSize(Ptr, sizeof(ReturnSetInfo)); + Ptr = nullptr; } ReturnSetInfo& Ref() { + Y_ENSURE(Ptr, "ReturnSetInfo is dead"); + return *static_cast(Ptr); } @@ -488,11 +499,21 @@ class TExprContextHolder { } ExprContext& Ref() { + Y_ENSURE(Ptr, "TExprContextHolder is dead"); + return *Ptr; } ~TExprContextHolder() { + Free(); + } + + void Free() { + if (!Ptr) { + return; + } FreeExprContext(Ptr, true); + Ptr = nullptr; } private: @@ -739,9 +760,7 @@ class TPgResolvedMultiCall : public TPgResolvedCallBase { } ~TIterator() { - if (TupleSlot) { - ExecDropSingleTupleTableSlot(TupleSlot); - } + FinishAndFree(); } private: @@ -766,7 +785,7 @@ class TPgResolvedMultiCall : public TPgResolvedCallBase { } else { YQL_ENSURE(!StructType); if (RSInfo.Ref().isDone == ExprEndResult) { - IsFinished = true; + FinishAndFree(); return false; } @@ -782,7 +801,7 @@ class TPgResolvedMultiCall : public TPgResolvedCallBase { bool CopyTuple(NUdf::TUnboxedValue& value) { if (!tuplestore_gettupleslot(RSInfo.Ref().setResult, true, false, TupleSlot)) { - IsFinished = true; + FinishAndFree(); return false; } @@ -835,6 +854,17 @@ class TPgResolvedMultiCall : public TPgResolvedCallBase { } } + void FinishAndFree() { + if (TupleSlot) { + ExecDropSingleTupleTableSlot(TupleSlot); + TupleSlot = nullptr; + } + RSInfo.Free(); + ExprContextHolder.Free(); + + IsFinished = true; + } + const std::string_view Name; TUnboxedValueVector Args; const TVector& ArgDesc; diff --git a/ydb/library/yql/parser/pg_wrapper/ut/arrow_ut.cpp b/ydb/library/yql/parser/pg_wrapper/ut/arrow_ut.cpp index c09359f40523..28caad945d52 100644 --- a/ydb/library/yql/parser/pg_wrapper/ut/arrow_ut.cpp +++ b/ydb/library/yql/parser/pg_wrapper/ut/arrow_ut.cpp @@ -106,6 +106,48 @@ Y_UNIT_TEST(PgConvertNumericInt) { } } +Y_UNIT_TEST(PgConvertDate32Date) { + TArenaMemoryContext arena; + + arrow::Date32Builder builder; + builder.Append(10227); + builder.AppendNull(); + builder.Append(11323); + builder.Append(10227); + builder.Append(10958); + builder.Append(11688); + + std::shared_ptr array; + builder.Finish(&array); + + NKikimr::NMiniKQL::TScopedAlloc alloc(__LOCATION__); + NKikimr::NMiniKQL::TTypeEnvironment typeEnv(alloc); + auto* targetType = NKikimr::NMiniKQL::TPgType::Create(DATEOID, typeEnv); + + auto converter = BuildPgColumnConverter(std::shared_ptr(new arrow::Date32Type), targetType); + auto result = converter(array); + const auto& data = result->data(); + UNIT_ASSERT_VALUES_EQUAL(result->length(), 6); + + const char* expected[] = { + "1998-01-01", nullptr, "2001-01-01", "1998-01-01", "2000-01-02", "2002-01-01" + }; + + NUdf::TFixedSizeBlockReader reader; + for (int i = 0; i < 6; i++) { + if (result->IsNull(i)) { + UNIT_ASSERT(expected[i] == nullptr); + } else { + auto item = reader.GetItem(*data, i).As(); + UNIT_ASSERT(expected[i] != nullptr); + UNIT_ASSERT_VALUES_EQUAL( + TString(DatumGetCString(DirectFunctionCall1(date_out, item))), + expected[i] + ); + } + } +} + } // Y_UNIT_TEST_SUITE(TArrowUtilsTests) } // namespace NYql diff --git a/ydb/library/yql/providers/common/pushdown/collection.cpp b/ydb/library/yql/providers/common/pushdown/collection.cpp index 03c13aa8b41b..c2bff37aa1e4 100644 --- a/ydb/library/yql/providers/common/pushdown/collection.cpp +++ b/ydb/library/yql/providers/common/pushdown/collection.cpp @@ -330,10 +330,6 @@ bool IsMemberColumn(const TExprBase& node, const TExprNode* lambdaArg) { return false; } -bool IsSupportedArithmeticalExpression(const TExprBase& node, const TSettings& settings) { - return settings.IsEnabled(TSettings::EFeatureFlag::ArithmeticalExpressions) && node.Maybe(); -} - bool CheckExpressionNodeForPushdown(const TExprBase& node, const TExprNode* lambdaArg, const TSettings& settings) { if (auto maybeSafeCast = node.Maybe()) { return IsSupportedCast(maybeSafeCast.Cast(), settings); @@ -355,11 +351,11 @@ bool CheckExpressionNodeForPushdown(const TExprBase& node, const TExprNode* lamb return true; } else if (settings.IsEnabled(TSettings::EFeatureFlag::ParameterExpression) && node.Maybe()) { return true; - } else if (IsSupportedArithmeticalExpression(node, settings)) { - TCoBinaryArithmetic op = node.Cast(); - return CheckExpressionNodeForPushdown(op.Left(), lambdaArg, settings) && CheckExpressionNodeForPushdown(op.Right(), lambdaArg, settings); + } else if (const auto op = node.Maybe(); op && settings.IsEnabled(TSettings::EFeatureFlag::UnaryOperators)) { + return CheckExpressionNodeForPushdown(op.Cast().Arg(), lambdaArg, settings); + } else if (const auto op = node.Maybe(); op && settings.IsEnabled(TSettings::EFeatureFlag::ArithmeticalExpressions)) { + return CheckExpressionNodeForPushdown(op.Cast().Left(), lambdaArg, settings) && CheckExpressionNodeForPushdown(op.Cast().Right(), lambdaArg, settings); } - return false; } diff --git a/ydb/library/yql/providers/common/pushdown/settings.h b/ydb/library/yql/providers/common/pushdown/settings.h index 920b6722dd00..caa8258ed114 100644 --- a/ydb/library/yql/providers/common/pushdown/settings.h +++ b/ydb/library/yql/providers/common/pushdown/settings.h @@ -22,6 +22,7 @@ struct TSettings { DecimalType = 1 << 12, DyNumberType = 1 << 13, ImplicitConversionToInt64 = 1 << 14, // Allow implicit conversions to 64-bits integers from other types of integers + UnaryOperators = 1 << 15, // -, Abs, Size }; explicit TSettings(NLog::EComponent logComponent) diff --git a/ydb/library/yql/providers/generic/connector/tests/docker-compose.yml b/ydb/library/yql/providers/generic/connector/tests/docker-compose.yml index 2991e0e883a1..0521e5963090 100644 --- a/ydb/library/yql/providers/generic/connector/tests/docker-compose.yml +++ b/ydb/library/yql/providers/generic/connector/tests/docker-compose.yml @@ -19,7 +19,7 @@ services: - 19000:9000 - 18123:8123 fq-connector-go: - image: ghcr.io/ydb-platform/fq-connector-go:v0.0.6-rc.8@sha256:74ebae0530d916c1842a7fddfbddc6c018763a0401f2f627a44e8829692fe41f + image: ghcr.io/ydb-platform/fq-connector-go:v0.1.1@sha256:47e24df143aee31a83d4a4cd0acc20b4cab8c03a9c63e81a6e99cb017a31f916 ports: - 50051:50051 network_mode: host diff --git a/ydb/library/yql/providers/generic/connector/tests/test_cases/select_datetime.py b/ydb/library/yql/providers/generic/connector/tests/test_cases/select_datetime.py index 71471ff5a9b7..f6d07a64cae6 100644 --- a/ydb/library/yql/providers/generic/connector/tests/test_cases/select_datetime.py +++ b/ydb/library/yql/providers/generic/connector/tests/test_cases/select_datetime.py @@ -88,7 +88,7 @@ def _make_test_clickhouse(self) -> TestCase: '1950-01-10', '1850-01-10', '1950-01-10 12:23:45', - '1850-01-10 12:23:45.678', + '1850-01-10 12:23:45.678910', ], # Value is OK for CH, but can be too early for YQL [ @@ -96,7 +96,7 @@ def _make_test_clickhouse(self) -> TestCase: '1970-01-10', '1950-01-10', '1980-01-10 12:23:45', - '1950-01-10 12:23:45.678', + '1950-01-10 12:23:45.678910', ], # Value is OK for both CH and YQL [ @@ -104,7 +104,7 @@ def _make_test_clickhouse(self) -> TestCase: '2004-01-10', '2004-01-10', '2004-01-10 12:23:45', - '2004-01-10 12:23:45.678', + '2004-01-10 12:23:45.678910', ], # Value is OK for CH, but too late for YQL [ @@ -112,7 +112,7 @@ def _make_test_clickhouse(self) -> TestCase: '2110-01-10', '2110-01-10', '2106-01-10 12:23:45', - '2110-01-10 12:23:45.678', + '2110-01-10 12:23:45.678910', ], # Value is too late for both OK for CH # In this case ClickHouse behaviour is undefined @@ -123,7 +123,7 @@ def _make_test_clickhouse(self) -> TestCase: '2150-01-10', '2300-01-10', '2107-01-10 12:23:45', - '2300-01-10 12:23:45.678', + '2300-01-10 12:23:45.678910', ], ] @@ -141,7 +141,7 @@ def _make_test_clickhouse(self) -> TestCase: datetime.date(2004, 1, 10), datetime.date(2004, 1, 10), datetime.datetime(2004, 1, 10, 12, 23, 45), - datetime.datetime(2004, 1, 10, 12, 23, 45, 678000), + datetime.datetime(2004, 1, 10, 12, 23, 45, 678910), ], [ 4, @@ -194,14 +194,14 @@ def _make_test_postgresql(self) -> TestCase: # Date is OK for CH, but too early for YQL [ 1, - datetime.datetime(1950, 5, 27, 12, 23, 45, 678000), + datetime.datetime(1950, 5, 27, 12, 23, 45, 678910), ], # Date is OK for both CH and YQL - [2, datetime.datetime(1988, 11, 20, 12, 23, 45, 678000)], + [2, datetime.datetime(1988, 11, 20, 12, 23, 45, 678910)], # Date is OK for CH, but too late for YQL [ 3, - datetime.datetime(2108, 1, 1, 12, 23, 45, 678000), + datetime.datetime(2108, 1, 1, 12, 23, 45, 678910), ], ] @@ -213,7 +213,7 @@ def _make_test_postgresql(self) -> TestCase: [ 2, # datetime.datetime(1988, 11, 20, 12, 23, 45, 678000).astimezone(ZoneInfo('UTC')).replace(tzinfo=None), - datetime.datetime(1988, 11, 20, 12, 23, 45, 678000), + datetime.datetime(1988, 11, 20, 12, 23, 45, 678910), ], [ 3, @@ -277,7 +277,7 @@ def _make_test_string_clickhouse(self) -> TestCase: '1950-01-10', '1850-01-10', '1950-01-10 12:23:45', - '1850-01-10 12:23:45.678', + '1850-01-10 12:23:45.678910', ], # Value is OK for CH, but can be too early for YQL [ @@ -285,7 +285,7 @@ def _make_test_string_clickhouse(self) -> TestCase: '1970-01-10', '1950-01-10', '1980-01-10 12:23:45', - '1950-01-10 12:23:45.678', + '1950-01-10 12:23:45.678910', ], # Value is OK for both CH and YQL [ @@ -293,7 +293,7 @@ def _make_test_string_clickhouse(self) -> TestCase: '2004-01-10', '2004-01-10', '2004-01-10 12:23:45', - '2004-01-10 12:23:45.678', + '2004-01-10 12:23:45.678910', ], # Value is OK for CH, but too late for YQL [ @@ -301,7 +301,7 @@ def _make_test_string_clickhouse(self) -> TestCase: '2110-01-10', '2110-01-10', '2106-01-10 12:23:45', - '2110-01-10 12:23:45.678', + '2110-01-10 12:23:45.678910', ], # Value is too late for both OK for CH # In this case ClickHouse behaviour is undefined @@ -312,21 +312,21 @@ def _make_test_string_clickhouse(self) -> TestCase: '2150-01-10', '2300-01-10', '2107-01-10 12:23:45', - '2300-01-10 12:23:45.678', + '2300-01-10 12:23:45.678910', ], ] data_out = [ - [1, '1970-01-01', '1900-01-01', '1970-01-01T00:00:00Z', '1900-01-01T12:23:45.678Z'], - [2, '1970-01-10', '1950-01-10', '1980-01-10T12:23:45Z', '1950-01-10T12:23:45.678Z'], - [3, '2004-01-10', '2004-01-10', '2004-01-10T12:23:45Z', '2004-01-10T12:23:45.678Z'], - [4, '2110-01-10', '2110-01-10', '2106-01-10T12:23:45Z', '2110-01-10T12:23:45.678Z'], + [1, '1970-01-01', '1900-01-01', '1970-01-01T00:00:00Z', '1900-01-01T12:23:45.67891Z'], + [2, '1970-01-10', '1950-01-10', '1980-01-10T12:23:45Z', '1950-01-10T12:23:45.67891Z'], + [3, '2004-01-10', '2004-01-10', '2004-01-10T12:23:45Z', '2004-01-10T12:23:45.67891Z'], + [4, '2110-01-10', '2110-01-10', '2106-01-10T12:23:45Z', '2110-01-10T12:23:45.67891Z'], [ 5, '2149-06-06', '2299-12-31', '1970-12-04T05:55:29Z', - '1900-01-01T00:00:00.000Z', # TODO: strange overflow under bottom bound for datetime64 + '1900-01-01T00:00:00Z', # TODO: strange overflow under bottom bound for datetime64 ], ] @@ -364,27 +364,27 @@ def _make_test_string_postgresql(self) -> TestCase: data_in = [ [ 1, - datetime.datetime(1950, 5, 27, 12, 23, 45, 678000), + datetime.datetime(1950, 5, 27, 12, 23, 45, 678910), ], - [2, datetime.datetime(1988, 11, 20, 12, 23, 45, 678000)], + [2, datetime.datetime(1988, 11, 20, 12, 23, 45, 678910)], [ 3, - datetime.datetime(2108, 1, 1, 12, 23, 45, 678000), + datetime.datetime(2108, 1, 1, 12, 23, 45, 678910), ], ] data_out = [ [ 1, - '1950-05-27T12:23:45.678Z', + '1950-05-27T12:23:45.67891Z', ], [ 2, - '1988-11-20T12:23:45.678Z', + '1988-11-20T12:23:45.67891Z', ], [ 3, - '2108-01-01T12:23:45.678Z', + '2108-01-01T12:23:45.67891Z', ], ] diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp index 032a0f61a224..cad13c9213b1 100644 --- a/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp +++ b/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp @@ -437,10 +437,17 @@ class TS3DqIntegration: public TDqIntegrationBase { return false; } + const NJson::TJsonValue* clusterNameProp = properties.FindPtr("ExternalDataSource"); + const TString clusterName = clusterNameProp && clusterNameProp->IsString() ? clusterNameProp->GetString() : TString(); + auto source = node.Cast(); if (auto maybeSettings = source.Settings().Maybe()) { const TS3SourceSettings settings = maybeSettings.Cast(); - properties["Name"] = "Raw read from external data source"; + if (clusterName) { + properties["Name"] = TStringBuilder() << "Raw read " << clusterName; + } else { + properties["Name"] = "Raw read from external data source"; + } properties["Format"] = "raw"; if (TString limit = settings.RowsLimitHint().StringValue()) { properties["RowsLimitHint"] = limit; @@ -448,7 +455,11 @@ class TS3DqIntegration: public TDqIntegrationBase { return true; } else if (auto maybeSettings = source.Settings().Maybe()) { const TS3ParseSettings settings = maybeSettings.Cast(); - properties["Name"] = "Parse from external data source"; + if (clusterName) { + properties["Name"] = TStringBuilder() << "Parse " << clusterName; + } else { + properties["Name"] = "Parse from external data source"; + } properties["Format"] = settings.Format().StringValue(); if (TString limit = settings.RowsLimitHint().StringValue()) { properties["RowsLimitHint"] = limit; diff --git a/ydb/library/yql/providers/yt/provider/yql_yt_helpers.cpp b/ydb/library/yql/providers/yt/provider/yql_yt_helpers.cpp index abb77fa1f017..1238e33203e8 100644 --- a/ydb/library/yql/providers/yt/provider/yql_yt_helpers.cpp +++ b/ydb/library/yql/providers/yt/provider/yql_yt_helpers.cpp @@ -121,7 +121,7 @@ bool IsYtIsolatedLambdaImpl(const TExprNode& lambdaBody, TSyncMap& syncList, TSt if (usedCluster && !UpdateUsedCluster(*usedCluster, TString{op.DataSink().Cluster().Value()})) { return false; } - syncList.emplace(op.Ptr(), syncList.size()); + syncList.emplace(out.Cast().Operation().Ptr(), syncList.size()); return true; } } diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in index 2e844b4646cd..f7393210bae9 100644 --- a/ydb/library/yql/sql/v1/SQLv1.g.in +++ b/ydb/library/yql/sql/v1/SQLv1.g.in @@ -683,7 +683,7 @@ table_constraint: ; table_index: INDEX an_id table_index_type - (WITH LPAREN an_id EQUALS an_id COMMA (an_id EQUALS an_id)* RPAREN)? + (WITH LPAREN an_id EQUALS an_id (COMMA an_id EQUALS an_id)* COMMA? RPAREN)? ON LPAREN an_id_schema (COMMA an_id_schema)* RPAREN (COVER LPAREN an_id_schema (COMMA an_id_schema)* RPAREN)?; diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp index 9edfb7b8d66b..6e2a13ecfb1d 100644 --- a/ydb/library/yql/sql/v1/sql_ut.cpp +++ b/ydb/library/yql/sql/v1/sql_ut.cpp @@ -2359,6 +2359,11 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) { UNIT_ASSERT(SqlToYql("USE plato; ALTER TABLE table SET (AUTO_PARTITIONING_BY_SIZE = DISABLED)").IsOk()); } + Y_UNIT_TEST(AlterTableAddIndexWithIsNotSupported) { + ExpectFailWithError("USE plato; ALTER TABLE table ADD INDEX idx LOCAL WITH (a=b, c=d, e=f) ON (col)", + "
:1:40: Error: local: alternative is not implemented yet: 692:7: local_index\n"); + } + Y_UNIT_TEST(OptionalAliases) { UNIT_ASSERT(SqlToYql("USE plato; SELECT foo FROM (SELECT key foo FROM Input);").IsOk()); UNIT_ASSERT(SqlToYql("USE plato; SELECT a.x FROM Input1 a JOIN Input2 b ON a.key = b.key;").IsOk()); @@ -6134,7 +6139,7 @@ Y_UNIT_TEST_SUITE(TViewSyntaxTest) { ); UNIT_ASSERT_C(res.Root, res.Issues.ToString()); } - + Y_UNIT_TEST(CreateViewFromTable) { constexpr const char* path = "/PathPrefix/TheView"; constexpr const char* query = R"( @@ -6178,7 +6183,7 @@ Y_UNIT_TEST_SUITE(TViewSyntaxTest) { ) ); UNIT_ASSERT_C(res.Root, res.Issues.ToString()); - + TString reconstructedQuery = ToString(Tokenize(query)); TVerifyLineFunc verifyLine = [&](const TString& word, const TString& line) { if (word == "query_text") { diff --git a/ydb/library/yql/tests/postgresql/cases/aggregates.err b/ydb/library/yql/tests/postgresql/cases/aggregates.err index 695de54f18e3..6668a65fb068 100644 --- a/ydb/library/yql/tests/postgresql/cases/aggregates.err +++ b/ydb/library/yql/tests/postgresql/cases/aggregates.err @@ -654,12 +654,7 @@ COPY bitwise_test FROM STDIN NULL 'null'; 7 7 7 3 4 B1100 \. --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "1" - - 1 1 1 1 1 B0101 - ^ +Metacommand \. is not supported SELECT BIT_AND(i2) AS "1", @@ -769,12 +764,7 @@ FALSE TRUE null null null TRUE FALSE null \. --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "TRUE" - - TRUE null FALSE null - ^ +Metacommand \. is not supported SELECT BOOL_AND(b1) AS "f", @@ -1354,4 +1344,4 @@ drop table p_t1; -- create temp table t1(f1 int, f2 bigint); -(TFileError) (File exists) util/system/file.cpp:857: can't open "/tmp/tmpbbmu_5di/t1" with mode CreateNew (0x00000003) +(TFileError) (File exists) util/system/file.cpp:918: can't open "/tmp/tmpjl487dvf/t1" with mode CreateNew (0x00000003) diff --git a/ydb/library/yql/tests/postgresql/cases/aggregates.out b/ydb/library/yql/tests/postgresql/cases/aggregates.out index 6bf57a88141a..be38f5ac4985 100644 --- a/ydb/library/yql/tests/postgresql/cases/aggregates.out +++ b/ydb/library/yql/tests/postgresql/cases/aggregates.out @@ -53,6 +53,67 @@ SELECT stddev_pop('nan'::numeric), stddev_samp('nan'::numeric); NaN | (1 row) +-- verify correct results for null and NaN inputs +select sum(null::int4) from generate_series(1,3); + sum +----- + +(1 row) + +select sum(null::int8) from generate_series(1,3); + sum +----- + +(1 row) + +select sum(null::numeric) from generate_series(1,3); + sum +----- + +(1 row) + +select sum(null::float8) from generate_series(1,3); + sum +----- + +(1 row) + +select avg(null::int4) from generate_series(1,3); + avg +----- + +(1 row) + +select avg(null::int8) from generate_series(1,3); + avg +----- + +(1 row) + +select avg(null::numeric) from generate_series(1,3); + avg +----- + +(1 row) + +select avg(null::float8) from generate_series(1,3); + avg +----- + +(1 row) + +select sum('NaN'::numeric) from generate_series(1,3); + sum +----- + NaN +(1 row) + +select avg('NaN'::numeric) from generate_series(1,3); + avg +----- + NaN +(1 row) + SELECT sum(x::numeric), avg(x::numeric), var_pop(x::numeric) FROM (VALUES ('1'), ('infinity')) v(x); sum | avg | var_pop diff --git a/ydb/library/yql/tests/postgresql/cases/aggregates.sql b/ydb/library/yql/tests/postgresql/cases/aggregates.sql index b1c7aa24889e..10ef58550ba8 100644 --- a/ydb/library/yql/tests/postgresql/cases/aggregates.sql +++ b/ydb/library/yql/tests/postgresql/cases/aggregates.sql @@ -13,6 +13,17 @@ SELECT var_pop('inf'::numeric), var_samp('inf'::numeric); SELECT stddev_pop('inf'::numeric), stddev_samp('inf'::numeric); SELECT var_pop('nan'::numeric), var_samp('nan'::numeric); SELECT stddev_pop('nan'::numeric), stddev_samp('nan'::numeric); +-- verify correct results for null and NaN inputs +select sum(null::int4) from generate_series(1,3); +select sum(null::int8) from generate_series(1,3); +select sum(null::numeric) from generate_series(1,3); +select sum(null::float8) from generate_series(1,3); +select avg(null::int4) from generate_series(1,3); +select avg(null::int8) from generate_series(1,3); +select avg(null::numeric) from generate_series(1,3); +select avg(null::float8) from generate_series(1,3); +select sum('NaN'::numeric) from generate_series(1,3); +select avg('NaN'::numeric) from generate_series(1,3); SELECT sum(x::numeric), avg(x::numeric), var_pop(x::numeric) FROM (VALUES ('1'), ('infinity')) v(x); SELECT sum(x::numeric), avg(x::numeric), var_pop(x::numeric) diff --git a/ydb/library/yql/tests/postgresql/cases/bit.err b/ydb/library/yql/tests/postgresql/cases/bit.err index 836e062b35a9..1ed31207ff8c 100644 --- a/ydb/library/yql/tests/postgresql/cases/bit.err +++ b/ydb/library/yql/tests/postgresql/cases/bit.err @@ -138,12 +138,7 @@ XFA50 X05AF X1234 XFFF5 \. --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "X0F" - - X0F X10 - ^ +Metacommand \. is not supported SELECT a, b, ~a AS "~ a", a & b AS "a & b", a | b AS "a | b", a # b AS "a # b" FROM varbit_table; @@ -179,12 +174,7 @@ XFA50 X05AF X1234 XFFF5 \. --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "X0F00" - - X0F00 X1000 - ^ +Metacommand \. is not supported SELECT a,b,~a AS "~ a",a & b AS "a & b", a|b AS "a | b", a # b AS "a # b" FROM bit_table; diff --git a/ydb/library/yql/tests/postgresql/cases/create_table.err b/ydb/library/yql/tests/postgresql/cases/create_table.err index bfc9390ba166..74172d95df30 100644 --- a/ydb/library/yql/tests/postgresql/cases/create_table.err +++ b/ydb/library/yql/tests/postgresql/cases/create_table.err @@ -434,23 +434,13 @@ DEALLOCATE select1; -- (temporarily hide query, to avoid the long CREATE TABLE stmt) \set ECHO none --stdin-:
: Error: Parse Sql - - -stdin-:
:3:1: Error: ERROR: syntax error at or near "\" - - \set ECHO none - ^ +Metacommand \set ECHO none is not supported SELECT 'CREATE TABLE extra_wide_table(firstc text, '|| array_to_string(array_agg('c'||i||' bool'),',')||', lastc text);' FROM generate_series(1, 1100) g(i) \gexec --stdin-:
: Error: Parse Sql - - -stdin-:
:3:1: Error: ERROR: syntax error at or near "\" - - \gexec - ^ +Metacommand \gexec is not supported \set ECHO all @@ -978,12 +968,7 @@ CREATE TABLE fail () INHERITS (partitioned2); -- Partition key in describe output \d partitioned --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "\" - - \d partitioned - ^ +Metacommand \d partitioned is not supported \d+ partitioned2 @@ -2274,12 +2259,7 @@ insert into parted_notnull_inh_test (b) values (null); -- note that while b's default is overriden, a's default is preserved \d parted_notnull_inh_test1 --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "\" - - \d parted_notnull_inh_test1 - ^ +Metacommand \d parted_notnull_inh_test1 is not supported drop table parted_notnull_inh_test; @@ -2399,32 +2379,17 @@ drop table test_part_coll_posix; -- Partition bound in describe output \d+ part_b --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "\" - - \d+ part_b - ^ +Metacommand \d+ part_b is not supported -- Both partition bound and partition key in describe output \d+ part_c --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "\" - - \d+ part_c - ^ +Metacommand \d+ part_c is not supported -- a level-2 partition's constraint will include the parent's expressions \d+ part_c_1_10 --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "\" - - \d+ part_c_1_10 - ^ +Metacommand \d+ part_c_1_10 is not supported -- Show partition count in the parent's describe output -- Tempted to include \d+ output listing partitions with bound info but @@ -2432,12 +2397,7 @@ drop table test_part_coll_posix; -- returned. \d parted --stdin-:
: Error: Parse Sql - - -stdin-:
:5:1: Error: ERROR: syntax error at or near "\" - - \d parted - ^ +Metacommand \d parted is not supported \d hash_parted diff --git a/ydb/library/yql/tests/postgresql/cases/functional_deps.err b/ydb/library/yql/tests/postgresql/cases/functional_deps.err index f375d08dea5b..fc6bed0daed9 100644 --- a/ydb/library/yql/tests/postgresql/cases/functional_deps.err +++ b/ydb/library/yql/tests/postgresql/cases/functional_deps.err @@ -455,14 +455,6 @@ INNER JOIN users u ON u.uid = n.uid WHERE n.type = 'blog' AND n.status = 1 GROUP BY u.uid, u.name; --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: AssumeColumnOrder, At function: PgReplaceUnknown, At function: OrderedMap, At function: UnionAll, At function: Aggregate - -- OK - ^ - -stdin-:
:1:1: Error: Expected hashable and equatable type for key column: _alias_u.name, but got: pgvarchar - -- OK - ^ -- OK SELECT u.uid, u.name FROM node n diff --git a/ydb/library/yql/tests/postgresql/cases/functional_deps.out b/ydb/library/yql/tests/postgresql/cases/functional_deps.out index 3f1fb1b9b052..60ac758a4294 100644 --- a/ydb/library/yql/tests/postgresql/cases/functional_deps.out +++ b/ydb/library/yql/tests/postgresql/cases/functional_deps.out @@ -35,3 +35,12 @@ CREATE TEMP TABLE users ( PRIMARY KEY (uid), UNIQUE (name) ); +-- OK +SELECT u.uid, u.name FROM node n +INNER JOIN users u ON u.uid = n.uid +WHERE n.type = 'blog' AND n.status = 1 +GROUP BY u.uid, u.name; + uid | name +-----+------ +(0 rows) + diff --git a/ydb/library/yql/tests/postgresql/cases/functional_deps.sql b/ydb/library/yql/tests/postgresql/cases/functional_deps.sql index 3f1fb1b9b052..4c8da83ef895 100644 --- a/ydb/library/yql/tests/postgresql/cases/functional_deps.sql +++ b/ydb/library/yql/tests/postgresql/cases/functional_deps.sql @@ -35,3 +35,8 @@ CREATE TEMP TABLE users ( PRIMARY KEY (uid), UNIQUE (name) ); +-- OK +SELECT u.uid, u.name FROM node n +INNER JOIN users u ON u.uid = n.uid +WHERE n.type = 'blog' AND n.status = 1 +GROUP BY u.uid, u.name; diff --git a/ydb/library/yql/tests/postgresql/cases/insert.err b/ydb/library/yql/tests/postgresql/cases/insert.err index f1a338b139c9..9939f85884bc 100644 --- a/ydb/library/yql/tests/postgresql/cases/insert.err +++ b/ydb/library/yql/tests/postgresql/cases/insert.err @@ -61,23 +61,15 @@ insert into inserttest (col1, col2, col3) values (DEFAULT, DEFAULT); insert into inserttest (col1, col2, col3) values (1, 2); --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: RemovePrefixMembers, At function: PgSelect, At function: PgSetItem - insert into inserttest (col1, col2, col3) values (1, 2); - ^ - -stdin-:
:1:1: Error: values and target_columns sizes do not match - insert into inserttest (col1, col2, col3) values (1, 2); - ^ insert into inserttest (col1) values (1, 2); -stdin-:
: Error: Type annotation - -stdin-:
:1:1: Error: At function: RemovePrefixMembers, At function: PgSelect, At function: PgSetItem + -stdin-:
:1:1: Error: At function: YtWriteTable! insert into inserttest (col1) values (1, 2); ^ - -stdin-:
:1:1: Error: values and target_columns sizes do not match + -stdin-:
:1:1: Error: values have 2 columns, INSERT INTO expects: 1 insert into inserttest (col1) values (1, 2); ^ @@ -1185,12 +1177,7 @@ from hash_parted order by part; -- partitions \d+ list_parted --stdin-:
: Error: Parse Sql - - -stdin-:
:3:1: Error: ERROR: syntax error at or near "\" - - \d+ list_parted - ^ +Metacommand \d+ list_parted is not supported -- cleanup drop table range_parted, list_parted; @@ -2505,12 +2492,7 @@ copy donothingbrtrig_test from stdout; 2 qux \. --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "1" - - 1 baz - ^ +Metacommand \. is not supported select tableoid::regclass, * from donothingbrtrig_test; diff --git a/ydb/library/yql/tests/postgresql/cases/int8.err b/ydb/library/yql/tests/postgresql/cases/int8.err index 5b962d292809..82decd997cc1 100644 --- a/ydb/library/yql/tests/postgresql/cases/int8.err +++ b/ydb/library/yql/tests/postgresql/cases/int8.err @@ -717,6 +717,6 @@ SELECT * FROM generate_series('+4567890123456789'::int8, '+4567890123456799'::in SELECT * FROM generate_series('+4567890123456789'::int8, '+4567890123456799'::int8, 2); -pgrun: /storage/d/arcc/mount/1/a0a05244-a249bf61-ec2999f5-6661a47b/contrib/libs/llvm12/lib/IR/LegacyPassManager.cpp:588: void llvm::PMTopLevelManager::setLastUser(ArrayRef, Pass *): Assertion `AnalysisPass && "Expected analysis pass to exist."' failed. +pgrun: /-S/ydbwork/ydb/contrib/libs/llvm12/lib/IR/LegacyPassManager.cpp:588: void llvm::PMTopLevelManager::setLastUser(ArrayRef, Pass *): Assertion `AnalysisPass && "Expected analysis pass to exist."' failed. pthread_kill at ./nptl/./nptl/pthread_kill.c:43:17 ?? at ??:0:0 diff --git a/ydb/library/yql/tests/postgresql/cases/interval.err b/ydb/library/yql/tests/postgresql/cases/interval.err index 2bf468218362..b67f8dd6a134 100644 --- a/ydb/library/yql/tests/postgresql/cases/interval.err +++ b/ydb/library/yql/tests/postgresql/cases/interval.err @@ -286,12 +286,7 @@ COPY INTERVAL_MULDIV_TBL FROM STDIN; 999 mon 999 days \. --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "41" - - 41 mon 12 days 360:00 - ^ +Metacommand \. is not supported SELECT span * 0.3 AS product FROM INTERVAL_MULDIV_TBL; diff --git a/ydb/library/yql/tests/postgresql/cases/join.err b/ydb/library/yql/tests/postgresql/cases/join.err index 50d0e21eb6b7..e468af68aea9 100644 --- a/ydb/library/yql/tests/postgresql/cases/join.err +++ b/ydb/library/yql/tests/postgresql/cases/join.err @@ -1680,4 +1680,4 @@ create temp table a ( constraint a_pk primary key (code) ); -(TFileError) (File exists) util/system/file.cpp:857: can't open "/tmp/tmpglg8to66/a" with mode CreateNew (0x00000003) +(TFileError) (File exists) util/system/file.cpp:918: can't open "/tmp/tmp3znkmfbo/a" with mode CreateNew (0x00000003) diff --git a/ydb/library/yql/tests/postgresql/cases/jsonb.err b/ydb/library/yql/tests/postgresql/cases/jsonb.err index d2cbf6610b70..c0a44f50465a 100644 --- a/ydb/library/yql/tests/postgresql/cases/jsonb.err +++ b/ydb/library/yql/tests/postgresql/cases/jsonb.err @@ -5938,12 +5938,6 @@ select jsonb_set('{"a": {"b": [1, 2, 3]}}', '{a, b, NULL}', '"new_value"'); -- jsonb_set_lax \pset null NULL --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "\" - - \pset null NULL - ^ -- pass though non nulls to jsonb_set select jsonb_set_lax('{"a":1,"b":2}','{b}','5') ; @@ -6063,7 +6057,6 @@ select jsonb_set_lax('{"a":1,"b":2}', '{b}', null, null_value_treatment => 'use_ \pset null '' -Metacommand \pset null '' is not supported -- jsonb_insert select jsonb_insert('{"a": [0,1,2]}', '{a, 1}', '"new_value"'); diff --git a/ydb/library/yql/tests/postgresql/cases/numeric.err b/ydb/library/yql/tests/postgresql/cases/numeric.err index 13bbc7245481..421fe55e56e1 100644 --- a/ydb/library/yql/tests/postgresql/cases/numeric.err +++ b/ydb/library/yql/tests/postgresql/cases/numeric.err @@ -2772,12 +2772,7 @@ COPY width_bucket_test (operand_num) FROM stdin; 10.0000000000001 \. --stdin-:
: Error: Parse Sql - - -stdin-:
:2:1: Error: ERROR: syntax error at or near "-" - - -5.2 - ^ +Metacommand \. is not supported UPDATE width_bucket_test SET operand_f8 = operand_num::float8; @@ -3617,6 +3612,6 @@ select * from generate_series(-100::numeric, 100::numeric, 0::numeric); select * from generate_series(-100::numeric, 100::numeric, 'nan'::numeric); -pgrun: /storage/d/arcc/mount/1/a0a05244-a249bf61-ec2999f5-6661a47b/contrib/libs/llvm12/lib/IR/LegacyPassManager.cpp:588: void llvm::PMTopLevelManager::setLastUser(ArrayRef, Pass *): Assertion `AnalysisPass && "Expected analysis pass to exist."' failed. +pgrun: /-S/ydbwork/ydb/contrib/libs/llvm12/lib/IR/LegacyPassManager.cpp:588: void llvm::PMTopLevelManager::setLastUser(ArrayRef, Pass *): Assertion `AnalysisPass && "Expected analysis pass to exist."' failed. pthread_kill at ./nptl/./nptl/pthread_kill.c:43:17 ?? at ??:0:0 diff --git a/ydb/library/yql/tests/postgresql/cases/subselect.err b/ydb/library/yql/tests/postgresql/cases/subselect.err index 6dfa1746d69d..566d00bef634 100644 --- a/ydb/library/yql/tests/postgresql/cases/subselect.err +++ b/ydb/library/yql/tests/postgresql/cases/subselect.err @@ -4,36 +4,12 @@ -- SELECT 1 AS one WHERE 1 IN (SELECT 1); --stdin-:
: Fatal: Optimization - - -stdin-:
:4:25: Fatal: ydb/library/yql/core/common_opt/yql_co_pgselect.cpp:2852 JoinOuter(): requirement outerInputs.size() == finalExtTypes->Tail().ChildrenSize() failed - SELECT 1 AS one WHERE 1 IN (SELECT 1); - ^ - -stdin-:
:1:1: Fatal: ydb/library/yql/core/common_opt/yql_co_pgselect.cpp:640 RewriteSubLinksPartial(): requirement status.Level != IGraphTransformer::TStatus::Error failed - -- - ^ SELECT 1 AS zero WHERE 1 NOT IN (SELECT 1); --stdin-:
: Fatal: Optimization - - -stdin-:
:1:26: Fatal: ydb/library/yql/core/common_opt/yql_co_pgselect.cpp:2852 JoinOuter(): requirement outerInputs.size() == finalExtTypes->Tail().ChildrenSize() failed - SELECT 1 AS zero WHERE 1 NOT IN (SELECT 1); - ^ - -stdin-:
:1:1: Fatal: ydb/library/yql/core/common_opt/yql_co_pgselect.cpp:640 RewriteSubLinksPartial(): requirement status.Level != IGraphTransformer::TStatus::Error failed - SELECT 1 AS zero WHERE 1 NOT IN (SELECT 1); - ^ SELECT 1 AS zero WHERE 1 IN (SELECT 2); --stdin-:
: Fatal: Optimization - - -stdin-:
:1:26: Fatal: ydb/library/yql/core/common_opt/yql_co_pgselect.cpp:2852 JoinOuter(): requirement outerInputs.size() == finalExtTypes->Tail().ChildrenSize() failed - SELECT 1 AS zero WHERE 1 IN (SELECT 2); - ^ - -stdin-:
:1:1: Fatal: ydb/library/yql/core/common_opt/yql_co_pgselect.cpp:640 RewriteSubLinksPartial(): requirement status.Level != IGraphTransformer::TStatus::Error failed - SELECT 1 AS zero WHERE 1 IN (SELECT 2); - ^ -- Check grammar's handling of extra parens in assorted contexts SELECT * FROM (SELECT 1 AS x) ss; @@ -44,5 +20,5 @@ SELECT * FROM ((SELECT 1 AS x)) ss; (SELECT 2) UNION SELECT 2; -NYql::TAggregateExpander::ExpandAggApply(TIntrusivePtr > const&) at /-S/ydb/library/yql/core/yql_aggregate_expander.cpp:119:51 +NYql::TAggregateExpander::ExpandAggApply(TIntrusivePtr > const&) at /-S/ydbwork/ydb/ydb/library/yql/core/yql_aggregate_expander.cpp:119:51 ?? at ??:0:0 diff --git a/ydb/library/yql/tests/postgresql/cases/subselect.out b/ydb/library/yql/tests/postgresql/cases/subselect.out index e5202b1d2067..9fbeb65df2d7 100644 --- a/ydb/library/yql/tests/postgresql/cases/subselect.out +++ b/ydb/library/yql/tests/postgresql/cases/subselect.out @@ -1,3 +1,22 @@ +-- +-- SUBSELECT +-- +SELECT 1 AS one WHERE 1 IN (SELECT 1); + one +----- + 1 +(1 row) + +SELECT 1 AS zero WHERE 1 NOT IN (SELECT 1); + zero +------ +(0 rows) + +SELECT 1 AS zero WHERE 1 IN (SELECT 2); + zero +------ +(0 rows) + -- Check grammar's handling of extra parens in assorted contexts SELECT * FROM (SELECT 1 AS x) ss; x diff --git a/ydb/library/yql/tests/postgresql/cases/subselect.sql b/ydb/library/yql/tests/postgresql/cases/subselect.sql index 95985320955e..cc4abd17a6a3 100644 --- a/ydb/library/yql/tests/postgresql/cases/subselect.sql +++ b/ydb/library/yql/tests/postgresql/cases/subselect.sql @@ -1,3 +1,9 @@ +-- +-- SUBSELECT +-- +SELECT 1 AS one WHERE 1 IN (SELECT 1); +SELECT 1 AS zero WHERE 1 NOT IN (SELECT 1); +SELECT 1 AS zero WHERE 1 IN (SELECT 2); -- Check grammar's handling of extra parens in assorted contexts SELECT * FROM (SELECT 1 AS x) ss; SELECT * FROM ((SELECT 1 AS x)) ss; diff --git a/ydb/library/yql/tests/postgresql/cases/timestamptz.err b/ydb/library/yql/tests/postgresql/cases/timestamptz.err index 7ba6e87295a1..263087a9d0d7 100644 --- a/ydb/library/yql/tests/postgresql/cases/timestamptz.err +++ b/ydb/library/yql/tests/postgresql/cases/timestamptz.err @@ -1142,6 +1142,6 @@ SET TimeZone to 'UTC'; SELECT '2011-03-27 00:00:00 Europe/Moscow'::timestamptz; -pgrun: /storage/d/arcc/mount/1/a0a05244-a249bf61-ec2999f5-6661a47b/contrib/libs/llvm12/lib/IR/LegacyPassManager.cpp:588: void llvm::PMTopLevelManager::setLastUser(ArrayRef, Pass *): Assertion `AnalysisPass && "Expected analysis pass to exist."' failed. +pgrun: /-S/ydbwork/ydb/contrib/libs/llvm12/lib/IR/LegacyPassManager.cpp:588: void llvm::PMTopLevelManager::setLastUser(ArrayRef, Pass *): Assertion `AnalysisPass && "Expected analysis pass to exist."' failed. pthread_kill at ./nptl/./nptl/pthread_kill.c:43:17 ?? at ??:0:0 diff --git a/ydb/library/yql/tests/postgresql/cases/union.err b/ydb/library/yql/tests/postgresql/cases/union.err index 9fccd88351e9..df1a8e64fe7b 100644 --- a/ydb/library/yql/tests/postgresql/cases/union.err +++ b/ydb/library/yql/tests/postgresql/cases/union.err @@ -5,5 +5,5 @@ -- Simple UNION constructs SELECT 1 AS two UNION SELECT 2 ORDER BY 1; -NYql::TAggregateExpander::ExpandAggApply(TIntrusivePtr > const&) at /-S/ydb/library/yql/core/yql_aggregate_expander.cpp:119:51 +NYql::TAggregateExpander::ExpandAggApply(TIntrusivePtr > const&) at /-S/ydbwork/ydb/ydb/library/yql/core/yql_aggregate_expander.cpp:119:51 ?? at ??:0:0 diff --git a/ydb/library/yql/tests/postgresql/cases/window.err b/ydb/library/yql/tests/postgresql/cases/window.err index 81c5dff5806d..e785c75f109d 100644 --- a/ydb/library/yql/tests/postgresql/cases/window.err +++ b/ydb/library/yql/tests/postgresql/cases/window.err @@ -25,25 +25,9 @@ INSERT INTO empsalary VALUES SELECT depname, empno, salary, sum(salary) OVER (PARTITION BY depname) FROM empsalary ORDER BY depname, salary; --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: AssumeColumnOrder, At function: Sort, At function: PgReplaceUnknown, At function: OrderedMap, At function: CalcOverWindow - SELECT depname, empno, salary, sum(salary) OVER (PARTITION BY depname) FROM empsalary ORDER BY depname, salary; - ^ - -stdin-:
:1:1: Error: Expected hashable and equatable type for key column: _yql_partition_key_0_0, but got: pgvarchar - SELECT depname, empno, salary, sum(salary) OVER (PARTITION BY depname) FROM empsalary ORDER BY depname, salary; - ^ SELECT depname, empno, salary, rank() OVER (PARTITION BY depname ORDER BY salary) FROM empsalary; --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: AssumeColumnOrder, At function: PgReplaceUnknown, At function: OrderedMap, At function: CalcOverWindow - SELECT depname, empno, salary, rank() OVER (PARTITION BY depname ORDER BY salary) FROM empsalary; - ^ - -stdin-:
:1:1: Error: Expected hashable and equatable type for key column: _yql_partition_key_0_0, but got: pgvarchar - SELECT depname, empno, salary, rank() OVER (PARTITION BY depname ORDER BY salary) FROM empsalary; - ^ -- with GROUP BY SELECT four, ten, SUM(SUM(four)) OVER (PARTITION BY four), AVG(ten) FROM tenk1 @@ -56,14 +40,6 @@ GROUP BY four, ten ORDER BY four, ten; SELECT depname, empno, salary, sum(salary) OVER w FROM empsalary WINDOW w AS (PARTITION BY depname); --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: AssumeColumnOrder, At function: PgReplaceUnknown, At function: OrderedMap, At function: CalcOverWindow - SELECT depname, empno, salary, sum(salary) OVER w FROM empsalary WINDOW w AS (PARTITION BY depname); - ^ - -stdin-:
:1:1: Error: Expected hashable and equatable type for key column: _yql_partition_key_0_0, but got: pgvarchar - SELECT depname, empno, salary, sum(salary) OVER w FROM empsalary WINDOW w AS (PARTITION BY depname); - ^ SELECT depname, empno, salary, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary) ORDER BY rank() OVER w; @@ -301,14 +277,6 @@ SELECT sum(salary), sum(sum(salary)) OVER (ORDER BY depname DESC) FROM empsalary GROUP BY depname; --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: AssumeColumnOrder, At function: PgReplaceUnknown, At function: OrderedMap, At function: CalcOverWindow, At function: CalcOverWindow, At function: UnionAll, At function: Aggregate - -- more than one window with GROUP BY - ^ - -stdin-:
:1:1: Error: Expected hashable and equatable type for key column: _alias_empsalary.depname, but got: pgvarchar - -- more than one window with GROUP BY - ^ -- identical windows with different names SELECT sum(salary) OVER w1, count(*) OVER w2 @@ -335,14 +303,6 @@ SELECT count(*) OVER (PARTITION BY four) FROM (SELECT * FROM tenk1 WHERE FALSE)s -- mixture of agg/wfunc in the same window SELECT sum(salary) OVER w, rank() OVER w FROM empsalary WINDOW w AS (PARTITION BY depname ORDER BY salary DESC); --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: AssumeColumnOrder, At function: PgReplaceUnknown, At function: OrderedMap, At function: CalcOverWindow - -- mixture of agg/wfunc in the same window - ^ - -stdin-:
:1:1: Error: Expected hashable and equatable type for key column: _yql_partition_key_0_0, but got: pgvarchar - -- mixture of agg/wfunc in the same window - ^ -- strict aggs SELECT empno, depname, salary, bonus, depadj, MIN(bonus) OVER (ORDER BY empno), MAX(depadj) OVER () FROM( @@ -2919,14 +2879,6 @@ SELECT * FROM FROM empsalary) emp WHERE first_emp = 1 OR last_emp = 1; --stdin-:
: Error: Type annotation - - -stdin-:
:1:1: Error: At function: PgSelect, At function: PgSetItem, At function: AssumeColumnOrder, At function: PgReplaceUnknown, At function: OrderedMap, At function: CalcOverWindow, At function: OrderedMap, At function: CalcOverWindow - SELECT * FROM - ^ - -stdin-:
:1:1: Error: Expected hashable and equatable type for key column: _yql_partition_key_0_0, but got: pgvarchar - SELECT * FROM - ^ -- cleanup DROP TABLE empsalary; @@ -3294,72 +3246,72 @@ SELECT i,AVG(v::bigint) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED SELECT i,AVG(v::int) OVER (ORDER BY i ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) FROM (VALUES(1,1),(2,2),(3,NULL),(4,NULL)) t(i,v); -VERIFY failed (2023-12-12T19:41:50.094283+0300): Value is not boxed +VERIFY failed (2024-01-12T02:19:37.139252+0300): Value is not boxed ydb/library/yql/public/udf/udf_value_inl.h:379 AsBoxed(): requirement IsBoxed() failed -BackTrace(void**, unsigned long)+29 (0x14087F6D) -FormatBackTrace(IOutputStream*)+32 (0x14088440) -PrintBackTrace()+17 (0x14088491) -NPrivate::InternalPanicImpl(int, char const*, char const*, int, int, int, TBasicStringBuf >, char const*, unsigned long)+995 (0x140CE743) -NPrivate::Panic(NPrivate::TStaticBuf const&, int, char const*, char const*, char const*, ...)+418 (0x140C18B2) -NYql::NUdf::TUnboxedValuePod::AsBoxed() const+218 (0x152AF12A) -NYql::PointerDatumFromPod(NYql::NUdf::TUnboxedValuePod const&)+29 (0x1B7A9E8D) -NYql::TPgResolvedCall::DoCalculate(NKikimr::NMiniKQL::TComputationContext&) const+771 (0x1B7E5C33) -NKikimr::NMiniKQL::TMutableComputationNode >::GetValue(NKikimr::NMiniKQL::TComputationContext&) const+420 (0x1B7E50D4) -??+0 (0x7F854D4FCC53) -??+0 (0x7F854D4FC47C) -??+0 (0x7F854D4FAE01) -??+0 (0x1FD82FD2) -NYql::NUdf::TBoxedValueAccessor::Fetch(NYql::NUdf::IBoxedValue&, NYql::NUdf::TUnboxedValue&)+244 (0x178E67A4) -NYql::NUdf::TUnboxedValuePod::Fetch(NYql::NUdf::TUnboxedValue&) const+228 (0x178E6624) -NKikimr::NMiniKQL::TForwardListValue::TIterator::Next(NYql::NUdf::TUnboxedValue&)+36 (0x17B99974) -NYql::NUdf::TBoxedValueAccessor::Next(NYql::NUdf::IBoxedValue&, NYql::NUdf::TUnboxedValue&)+244 (0x14EFDD24) -NYql::NUdf::TUnboxedValuePod::Next(NYql::NUdf::TUnboxedValue&) const+228 (0x14EFD9F4) -NYql::TYsonExecuteResOrPull::WriteValue(NYql::NUdf::TUnboxedValue const&, NKikimr::NMiniKQL::TType*)+436 (0x209F9A54) -NYql::NFile::TYtFileGateway::ExecuteResult(NYql::NFile::TSession&, NYson::TYsonWriter&, NYql::NNodes::TExprBase, NYql::TExprContext&, NYql::IYtGateway::TResOrPullOptions&&, TVector >, std::__y1::allocator > > > const&) const+2002 (0x1C34AE12) -NYql::NFile::TYtFileGateway::ResOrPull(TIntrusivePtr > const&, NYql::TExprContext&, NYql::IYtGateway::TResOrPullOptions&&)+1128 (0x1C31DEF8) -??+0 (0x1AB7ECFE) -??+0 (0x1AB82F89) -??+0 (0x1AB82EE8) -??+0 (0x1AB82E78) -??+0 (0x1AB82E28) -??+0 (0x1AB821BC) -std::__y1::__function::__value_func > const&, TIntrusivePtr >&, NYql::TExprContext&)> > > (TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&)>::operator()[abi:v15000](TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&) const+93 (0x1AB92C9D) -std::__y1::function > const&, TIntrusivePtr >&, NYql::TExprContext&)> > > (TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&)>::operator()(TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&) const+56 (0x1AB91FE8) -NYql::TExecTransformerBase::CallbackTransform(TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&)+811 (0x1AB91C6B) -NYql::TAsyncCallbackTransformer::DoTransform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+64 (0x1AB7FF50) -NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x153FF174) -??+0 (0x176F665C) -??+0 (0x176EA0A6) -NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x153FF174) -??+0 (0x1786F3EC) -??+0 (0x17861728) -??+0 (0x1786DCCC) -??+0 (0x178619D7) -??+0 (0x1786D98A) -??+0 (0x178619D7) -??+0 (0x17859F2E) -NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x153FF174) -??+0 (0x15403098) -??+0 (0x15402DC9) -??+0 (0x153FF7D5) -NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x153FF174) -NYql::AsyncTransformStepImpl(NYql::IGraphTransformer&, TIntrusivePtr >&, NYql::TExprContext&, bool, bool, TBasicStringBuf > const&)+1222 (0x153FE226) -NYql::AsyncTransform(NYql::IGraphTransformer&, TIntrusivePtr >&, NYql::TExprContext&, bool)+127 (0x153FEAAF) -NYql::TProgram::AsyncTransformWithFallback(bool)+120 (0x16C17A78) -??+0 (0x16C3EC1E) -??+0 (0x16C3E797) -??+0 (0x16C3E665) -??+0 (0x16C3D140) -??+0 (0x16C3D007) -??+0 (0x16C15F8D) -NYql::TProgram::RunAsync(TBasicString > const&, IOutputStream*, IOutputStream*, IOutputStream*, bool)+3760 (0x16C15C20) -??+0 (0x16C12804) -NYql::TProgram::Run(TBasicString > const&, IOutputStream*, IOutputStream*, IOutputStream*, bool)+301 (0x16C14CCD) -Main(int, char**)+4924 (0x13F14D4C) -main+273 (0x13F16831) -??+0 (0x7F854D229D90) -__libc_start_main+128 (0x7F854D229E40) -??+0 (0x13EA0029) +BackTrace(void**, unsigned long)+29 (0x142C329D) +FormatBackTrace(IOutputStream*)+32 (0x142C3770) +PrintBackTrace()+17 (0x142C37C1) +NPrivate::InternalPanicImpl(int, char const*, char const*, int, int, int, TBasicStringBuf >, char const*, unsigned long)+995 (0x143260F3) +NPrivate::Panic(NPrivate::TStaticBuf const&, int, char const*, char const*, char const*, ...)+418 (0x14319332) +NYql::NUdf::TUnboxedValuePod::AsBoxed() const+218 (0x154F450A) +NYql::PointerDatumFromPod(NYql::NUdf::TUnboxedValuePod const&)+29 (0x1BB15C6D) +NYql::TPgResolvedCall::DoCalculate(NKikimr::NMiniKQL::TComputationContext&) const+771 (0x1BB577E3) +NKikimr::NMiniKQL::TMutableComputationNode >::GetValue(NKikimr::NMiniKQL::TComputationContext&) const+420 (0x1BB56C84) +??+0 (0x7F17843F0C53) +??+0 (0x7F17843F047C) +??+0 (0x7F17843EEE01) +??+0 (0x2011E4F2) +NYql::NUdf::TBoxedValueAccessor::Fetch(NYql::NUdf::IBoxedValue&, NYql::NUdf::TUnboxedValue&)+244 (0x17B82E34) +NYql::NUdf::TUnboxedValuePod::Fetch(NYql::NUdf::TUnboxedValue&) const+228 (0x17B82CB4) +NKikimr::NMiniKQL::TForwardListValue::TIterator::Next(NYql::NUdf::TUnboxedValue&)+36 (0x17E35FF4) +NYql::NUdf::TBoxedValueAccessor::Next(NYql::NUdf::IBoxedValue&, NYql::NUdf::TUnboxedValue&)+244 (0x151403B4) +NYql::NUdf::TUnboxedValuePod::Next(NYql::NUdf::TUnboxedValue&) const+228 (0x15140084) +NYql::TYsonExecuteResOrPull::WriteValue(NYql::NUdf::TUnboxedValue const&, NKikimr::NMiniKQL::TType*)+436 (0x20D5B834) +NYql::NFile::TYtFileGateway::ExecuteResult(NYql::NFile::TSession&, NYson::TYsonWriter&, NYql::NNodes::TExprBase, NYql::TExprContext&, NYql::IYtGateway::TResOrPullOptions&&, TVector >, std::__y1::allocator > > > const&) const+2002 (0x1C6C1D32) +NYql::NFile::TYtFileGateway::ResOrPull(TIntrusivePtr > const&, NYql::TExprContext&, NYql::IYtGateway::TResOrPullOptions&&)+1128 (0x1C694F58) +??+0 (0x1AEE8ACE) +??+0 (0x1AEECD59) +??+0 (0x1AEECCB8) +??+0 (0x1AEECC48) +??+0 (0x1AEECBF8) +??+0 (0x1AEEBF8C) +std::__y1::__function::__value_func > const&, TIntrusivePtr >&, NYql::TExprContext&)> > > (TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&)>::operator()[abi:v15000](TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&) const+93 (0x1AEFCA0D) +std::__y1::function > const&, TIntrusivePtr >&, NYql::TExprContext&)> > > (TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&)>::operator()(TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&) const+56 (0x1AEFBD58) +NYql::TExecTransformerBase::CallbackTransform(TIntrusivePtr > const&, TIntrusivePtr >&, NYql::TExprContext&)+811 (0x1AEFB9DB) +NYql::TAsyncCallbackTransformer::DoTransform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+64 (0x1AEE9D20) +NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x15645D84) +??+0 (0x1799623C) +??+0 (0x17989C86) +NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x15645D84) +??+0 (0x17B2220C) +??+0 (0x17B14548) +??+0 (0x17B20AEC) +??+0 (0x17B147F7) +??+0 (0x17B207AA) +??+0 (0x17B147F7) +??+0 (0x17B0CD4E) +NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x15645D84) +??+0 (0x15649CA8) +??+0 (0x156499D9) +??+0 (0x156463E5) +NYql::TGraphTransformerBase::Transform(TIntrusivePtr >, TIntrusivePtr >&, NYql::TExprContext&)+100 (0x15645D84) +NYql::AsyncTransformStepImpl(NYql::IGraphTransformer&, TIntrusivePtr >&, NYql::TExprContext&, bool, bool, TBasicStringBuf > const&)+1222 (0x15644E36) +NYql::AsyncTransform(NYql::IGraphTransformer&, TIntrusivePtr >&, NYql::TExprContext&, bool)+127 (0x156456BF) +NYql::TProgram::AsyncTransformWithFallback(bool)+120 (0x16EA37F8) +??+0 (0x16ECA8FE) +??+0 (0x16ECA477) +??+0 (0x16ECA345) +??+0 (0x16EC8E20) +??+0 (0x16EC8CE7) +??+0 (0x16EA1D0D) +NYql::TProgram::RunAsync(TBasicString > const&, IOutputStream*, IOutputStream*, IOutputStream*, bool)+3760 (0x16EA19A0) +??+0 (0x16E9E584) +NYql::TProgram::Run(TBasicString > const&, IOutputStream*, IOutputStream*, IOutputStream*, bool)+301 (0x16EA0A4D) +Main(int, char**)+5378 (0x1414F2E2) +main+273 (0x14150F11) +??+0 (0x7F1784029D90) +__libc_start_main+128 (0x7F1784029E40) +??+0 (0x140DA029) pthread_kill at ./nptl/./nptl/pthread_kill.c:43:17 ?? at ??:0:0 diff --git a/ydb/library/yql/tests/postgresql/cases/xml.err b/ydb/library/yql/tests/postgresql/cases/xml.err index 7ac69d22136c..0b33ab4cb111 100644 --- a/ydb/library/yql/tests/postgresql/cases/xml.err +++ b/ydb/library/yql/tests/postgresql/cases/xml.err @@ -1358,12 +1358,7 @@ SELECT xml_is_well_formed('abc'); -- error messages, we suppress the DETAIL in this test. \set VERBOSITY terse --stdin-:
: Error: Parse Sql - - -stdin-:
:11:1: Error: ERROR: syntax error at or near "\" - - \set VERBOSITY terse - ^ +Metacommand \set VERBOSITY terse is not supported SELECT xpath('/*', ''); diff --git a/ydb/library/yql/tests/postgresql/cases/xml.err.1 b/ydb/library/yql/tests/postgresql/cases/xml.err.1 index 7ac69d22136c..0b33ab4cb111 100644 --- a/ydb/library/yql/tests/postgresql/cases/xml.err.1 +++ b/ydb/library/yql/tests/postgresql/cases/xml.err.1 @@ -1358,12 +1358,7 @@ SELECT xml_is_well_formed('abc'); -- error messages, we suppress the DETAIL in this test. \set VERBOSITY terse --stdin-:
: Error: Parse Sql - - -stdin-:
:11:1: Error: ERROR: syntax error at or near "\" - - \set VERBOSITY terse - ^ +Metacommand \set VERBOSITY terse is not supported SELECT xpath('/*', ''); diff --git a/ydb/library/yql/tests/postgresql/cases/xml.err.2 b/ydb/library/yql/tests/postgresql/cases/xml.err.2 index 7ac69d22136c..0b33ab4cb111 100644 --- a/ydb/library/yql/tests/postgresql/cases/xml.err.2 +++ b/ydb/library/yql/tests/postgresql/cases/xml.err.2 @@ -1358,12 +1358,7 @@ SELECT xml_is_well_formed('abc'); -- error messages, we suppress the DETAIL in this test. \set VERBOSITY terse --stdin-:
: Error: Parse Sql - - -stdin-:
:11:1: Error: ERROR: syntax error at or near "\" - - \set VERBOSITY terse - ^ +Metacommand \set VERBOSITY terse is not supported SELECT xpath('/*', ''); diff --git a/ydb/library/yql/tests/sql/dq_file.py b/ydb/library/yql/tests/sql/dq_file.py index 2e04321484a5..bb68cc9c174d 100644 --- a/ydb/library/yql/tests/sql/dq_file.py +++ b/ydb/library/yql/tests/sql/dq_file.py @@ -57,7 +57,7 @@ def normalize_res(res, sort): data['Truncated'] = True if 'Data' in data and len(data['Data']) == 0: del data['Data'] - # return res # TODO: uncomment it in future + return res program_sql = os.path.join(DATA_PATH, suite, '%s.sql' % case) with codecs.open(program_sql, encoding='utf-8') as program_file_descr: diff --git a/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json index 3d51e1a6a038..82099eb4f1b1 100644 --- a/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json @@ -1860,7 +1860,13 @@ "uri": "https://{canondata_backend}/1600758/aad142702907f13e911494c1a7b312bad34f692a/resource.tar.gz#test.test_pg-pg_types_orderby--Plan_/plan.txt" } ], - "test.test[pg-pg_types_orderby--Results]": [], + "test.test[pg-pg_types_orderby--Results]": [ + { + "checksum": "1cd28382e5bb763d1a10fbef380b19f4", + "size": 918, + "uri": "https://{canondata_backend}/1925842/b9dbc79de0fb09dbc2b3e197272eb35de7410096/resource.tar.gz#test.test_pg-pg_types_orderby--Results_/results.txt" + } + ], "test.test[pg-select_having-default.txt-Analyze]": [ { "checksum": "b4dd508a329723c74293d80f0278c705", diff --git a/ydb/library/yql/tests/sql/suites/pg/pg_types_orderby.cfg b/ydb/library/yql/tests/sql/suites/pg/pg_types_orderby.cfg index 9fd3037a3bcc..748ba77b3ffc 100644 --- a/ydb/library/yql/tests/sql/suites/pg/pg_types_orderby.cfg +++ b/ydb/library/yql/tests/sql/suites/pg/pg_types_orderby.cfg @@ -1 +1,2 @@ in Input4 input4.txt +providers dq diff --git a/ydb/public/lib/ydb_cli/common/format.cpp b/ydb/public/lib/ydb_cli/common/format.cpp index 08b76da9781d..dc8f37c59d8b 100644 --- a/ydb/public/lib/ydb_cli/common/format.cpp +++ b/ydb/public/lib/ydb_cli/common/format.cpp @@ -439,6 +439,22 @@ void TQueryPlanPrinter::PrintPrettyTableImpl(const NJson::TJsonValue& plan, TStr newRow.Column(2, std::move(nRows)); } + NColorizer::TColors colors = NColorizer::AutoColors(Cout); + TStringBuf color; + switch(offset.size() % 3) { + case 0: + color = colors.Red(); + break; + case 1: + color = colors.Green(); + break; + case 2: + color = colors.Blue(); + break; + default: + color = colors.Default(); + break; + } if (node.contains("Operators")) { for (const auto& op : node.at("Operators").GetArraySafe()) { @@ -458,26 +474,25 @@ void TQueryPlanPrinter::PrintPrettyTableImpl(const NJson::TJsonValue& plan, TStr TStringBuilder operation; if (info.empty()) { - operation << offset << " -> " << op.GetMapSafe().at("Name").GetString(); + operation << offset << color << " -> " << colors.Default() << op.GetMapSafe().at("Name").GetString(); } else { - operation << offset << " -> " << op.GetMapSafe().at("Name").GetString() + operation << offset << color << " -> " << colors.Default() << op.GetMapSafe().at("Name").GetString() << " (" << JoinStrings(info, ", ") << ")"; } - auto& row = table.AddRow(); - row.Column(0, std::move(operation)); + newRow.Column(0, std::move(operation)); if (AnalyzeMode) { - row.Column(3, std::move(eCost)); - row.Column(4, std::move(eRows)); + newRow.Column(3, std::move(eCost)); + newRow.Column(4, std::move(eRows)); } else { - row.Column(1, std::move(eCost)); - row.Column(2, std::move(eRows)); + newRow.Column(1, std::move(eCost)); + newRow.Column(2, std::move(eRows)); } } } else { TStringBuilder operation; - operation << offset << " -> " << node.at("Node Type").GetString(); + operation << offset << color << " -> " << colors.Default() << node.at("Node Type").GetString(); newRow.Column(0, std::move(operation)); } @@ -614,6 +629,8 @@ NJson::TJsonValue TQueryPlanPrinter::ReconstructQueryPlanRec(const NJson::TJsonV TVector planInputs; + auto opName = op.GetMapSafe().at("Name").GetStringSafe(); + for (auto opInput : op.GetMapSafe().at("Inputs").GetArraySafe()) { if (opInput.GetMapSafe().contains("ExternalPlanNodeId")) { auto inputPlanKey = opInput.GetMapSafe().at("ExternalPlanNodeId").GetIntegerSafe(); @@ -623,6 +640,10 @@ NJson::TJsonValue TQueryPlanPrinter::ReconstructQueryPlanRec(const NJson::TJsonV auto inputPlanId = opInput.GetMapSafe().at("InternalOperatorId").GetIntegerSafe(); planInputs.push_back( ReconstructQueryPlanRec(plan, inputPlanId, planIndex, precomputes, nodeCounter)); } + // temp hack + if (opName == "Filter") { + break; + } } if (op.GetMapSafe().contains("Inputs")) { @@ -642,7 +663,7 @@ NJson::TJsonValue TQueryPlanPrinter::ReconstructQueryPlanRec(const NJson::TJsonV } } - result["Node Type"] = op.GetMapSafe().at("Name").GetStringSafe(); + result["Node Type"] = opName; NJson::TJsonValue newOps; newOps.AppendValue(op); result["Operators"] = newOps; diff --git a/ydb/public/lib/ydb_cli/common/pretty_table.cpp b/ydb/public/lib/ydb_cli/common/pretty_table.cpp index 0dc07572ee6e..b32c8ae9d00f 100644 --- a/ydb/public/lib/ydb_cli/common/pretty_table.cpp +++ b/ydb/public/lib/ydb_cli/common/pretty_table.cpp @@ -15,6 +15,48 @@ TPrettyTable::TRow::TRow(size_t nColumns) { } +size_t TotalAnsiEscapeCodeLen(TStringBuf text) { + enum { + TEXT, + BEFORE_CODE, + IN_CODE, + } state = TEXT; + + size_t totalLen = 0; + size_t curLen = 0; + + for (auto it = text.begin(); it < text.end(); ++it) { + switch (state) { + case TEXT: + if (*it == '\033') { + state = BEFORE_CODE; + curLen = 1; + } + break; + case BEFORE_CODE: + if (*it == '[') { + state = IN_CODE; + curLen++; + } else { + state = TEXT; + } + break; + case IN_CODE: + if (*it == ';' || isdigit(*it)) { + curLen++; + } else { + if (*it == 'm') { + totalLen += curLen + 1; + } + state = TEXT; + } + break; + } + } + + return totalLen; +} + size_t TPrettyTable::TRow::ExtraBytes(TStringBuf data) const { // counter of previously uncounted bytes size_t extraBytes = 0; @@ -31,7 +73,9 @@ size_t TPrettyTable::TRow::ExtraBytes(TStringBuf data) const { extraBytes += n - 1; } } - + // update counter with len of color + extraBytes += TotalAnsiEscapeCodeLen(data); + return extraBytes; } @@ -70,8 +114,11 @@ bool TPrettyTable::TRow::PrintColumns(IOutputStream& o, const TVector& w for (const auto& line : column) { data = line; extraBytes = ExtraBytes(data); + if (data && l < lineNumber) { + data.Skip(extraBytes); + } while (data && l < lineNumber) { - data.Skip(width + extraBytes); + data.Skip(width); ++l; } }