diff --git a/ydb/apps/ydb/ut/epilogue.cmake b/ydb/apps/ydb/ut/epilogue.cmake deleted file mode 100644 index c14b6679de76..000000000000 --- a/ydb/apps/ydb/ut/epilogue.cmake +++ /dev/null @@ -1 +0,0 @@ -add_dependencies(ydb-apps-ydb-ut ydb_py_protos) diff --git a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp index 6a6c4700fd3d..aa14ad399e31 100644 --- a/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp +++ b/ydb/core/blobstorage/pdisk/blobstorage_pdisk_impl.cpp @@ -1518,8 +1518,17 @@ void TPDisk::WhiteboardReport(TWhiteboardReport &whiteboardReport) { TGuard guard(StateMutex); const ui64 totalSize = Format.DiskSize; const ui64 availableSize = (ui64)Format.ChunkSize * Keeper.GetFreeChunkCount(); - *Mon.FreeSpaceBytes = availableSize; - *Mon.UsedSpaceBytes = totalSize - *Mon.FreeSpaceBytes; + + if (*Mon.PDiskBriefState != TPDiskMon::TPDisk::Error) { + *Mon.FreeSpaceBytes = availableSize; + *Mon.UsedSpaceBytes = totalSize - availableSize; + } else { + // If disk is in Error State, show its total and used space as 32KiB (format size) + *Mon.FreeSpaceBytes = 0; + *Mon.UsedSpaceBytes = 32_KB; + *Mon.TotalSpaceBytes = 32_KB; + } + NKikimrWhiteboard::TPDiskStateInfo& pdiskState = reportResult->PDiskState->Record; pdiskState.SetPDiskId(PDiskId); pdiskState.SetPath(Cfg->GetDevicePath()); diff --git a/ydb/core/change_exchange/change_sender_common_ops.h b/ydb/core/change_exchange/change_sender_common_ops.h index 2e25c8aa72f4..cdaed3fe37b8 100644 --- a/ydb/core/change_exchange/change_sender_common_ops.h +++ b/ydb/core/change_exchange/change_sender_common_ops.h @@ -21,28 +21,37 @@ struct TEvChangeExchangePrivate { static_assert(EvEnd < EventSpaceEnd(TKikimrEvents::ES_PRIVATE)); - template - struct TEvWithPartitionId: public TEventLocal { + struct TEvReady: public TEventLocal { ui64 PartitionId; - explicit TEvWithPartitionId(ui64 partiionId) + explicit TEvReady(ui64 partiionId) : PartitionId(partiionId) { } TString ToString() const override { - return TStringBuilder() << TEventLocal::ToStringHeader() << " {" + return TStringBuilder() << ToStringHeader() << " {" << " PartitionId: " << PartitionId << " }"; } }; - struct TEvReady: public TEvWithPartitionId { - using TEvWithPartitionId::TEvWithPartitionId; - }; + struct TEvGone: public TEventLocal { + ui64 PartitionId; + bool HardError; + + explicit TEvGone(ui64 partitionId, bool hardError = false) + : PartitionId(partitionId) + , HardError(hardError) + { + } - struct TEvGone: public TEvWithPartitionId { - using TEvWithPartitionId::TEvWithPartitionId; + TString ToString() const override { + return TStringBuilder() << ToStringHeader() << " {" + << " PartitionId: " << PartitionId + << " HardError: " << HardError + << " }"; + } }; }; // TEvChangeExchangePrivate diff --git a/ydb/core/driver_lib/run/auto_config_initializer.cpp b/ydb/core/driver_lib/run/auto_config_initializer.cpp index 7996002983df..2b08be5e906d 100644 --- a/ydb/core/driver_lib/run/auto_config_initializer.cpp +++ b/ydb/core/driver_lib/run/auto_config_initializer.cpp @@ -31,7 +31,7 @@ namespace { i16 MaxThreadCount; }; - constexpr i16 MaxPreparedCpuCount = 31; + constexpr i16 MaxPreparedCpuCount = 30; constexpr i16 GRpcWorkerCountInMaxPreparedCpuCase = 4; constexpr i16 GrpcProxyCountInMaxPreparedCpuCase = 4; constexpr i16 CpuCountForEachGRpcWorker = MaxPreparedCpuCount / GRpcWorkerCountInMaxPreparedCpuCase; @@ -46,32 +46,31 @@ namespace { { {1, 2}, {0, 3}, {1, 1}, {0, 0}, {1, 1} }, // 3 { {1, 2}, {1, 4}, {1, 1}, {0, 0}, {1, 2} }, // 4 { {1, 3}, {2, 5}, {1, 1}, {0, 0}, {1, 2} }, // 5 - { {1, 3}, {3, 6}, {1, 1}, {0, 0}, {1, 2} }, // 6 + { {1, 3}, {3, 6}, {1, 1}, {0, 0}, {1, 3} }, // 6 { {2, 4}, {3, 7}, {1, 2}, {0, 0}, {1, 3} }, // 7 - { {2, 4}, {4, 8}, {1, 2}, {0, 0}, {1, 3} }, // 8 - { {2, 5}, {4, 9}, {2, 3}, {0, 0}, {1, 3} }, // 9 - { {2, 5}, {5, 10}, {2, 3}, {0, 0}, {1, 3} }, // 10 - { {2, 6}, {5, 11}, {2, 3}, {0, 0}, {2, 4} }, // 11 - { {2, 6}, {6, 12}, {2, 3}, {0, 0}, {2, 5} }, // 12 - { {3, 7}, {6, 13}, {2, 3}, {0, 0}, {2, 5} }, // 13 - { {3, 7}, {6, 14}, {2, 3}, {0, 0}, {3, 6} }, // 14 - { {3, 8}, {7, 15}, {2, 4}, {0, 0}, {3, 6} }, // 15 - { {3, 8}, {8, 16}, {2, 4}, {0, 0}, {3, 6} }, // 16 - { {3, 9}, {9, 17}, {2, 4}, {0, 0}, {3, 7} }, // 17 - { {3, 9}, {9, 18}, {3, 5}, {0, 0}, {3, 7} }, // 18 - { {3, 10}, {9, 19}, {3, 5}, {0, 0}, {4, 8} }, // 19 - { {4, 10}, {9, 20}, {3, 5}, {0, 0}, {4, 8} }, // 20 - { {4, 11}, {10, 21}, {3, 5}, {0, 0}, {4, 8} }, // 21 - { {4, 11}, {11, 22}, {3, 5}, {0, 0}, {4, 9} }, // 22 - { {4, 12}, {12, 23}, {3, 6}, {0, 0}, {4, 9} }, // 23 - { {4, 12}, {12, 24}, {3, 6}, {0, 0}, {5, 10} }, // 24 - { {5, 13}, {12, 25}, {3, 6}, {0, 0}, {5, 10} }, // 25 - { {5, 13}, {12, 26}, {4, 7}, {0, 0}, {5, 10} }, // 26 - { {5, 14}, {13, 27}, {4, 7}, {0, 0}, {5, 11} }, // 27 - { {5, 14}, {14, 28}, {4, 7}, {0, 0}, {5, 11} }, // 28 - { {5, 15}, {14, 29}, {4, 8}, {0, 0}, {6, 12} }, // 29 - { {5, 15}, {15, 30}, {4, 8}, {0, 0}, {6, 12} }, // 30 - { {6, 18}, {15, 31}, {4, 8}, {0, 0}, {6, 12} }, // 31 + { {2, 4}, {4, 8}, {1, 2}, {0, 0}, {1, 4} }, // 8 + { {2, 5}, {4, 9}, {2, 3}, {0, 0}, {1, 4} }, // 9 + { {2, 5}, {5, 10}, {2, 3}, {0, 0}, {1, 5} }, // 10 + { {2, 6}, {5, 11}, {2, 3}, {0, 0}, {2, 5} }, // 11 + { {2, 6}, {6, 12}, {2, 3}, {0, 0}, {2, 6} }, // 12 + { {3, 7}, {6, 13}, {2, 3}, {0, 0}, {2, 6} }, // 13 + { {3, 7}, {6, 14}, {2, 3}, {0, 0}, {3, 7} }, // 14 + { {3, 8}, {7, 15}, {2, 4}, {0, 0}, {3, 7} }, // 15 + { {3, 8}, {8, 16}, {2, 4}, {0, 0}, {3, 8} }, // 16 + { {3, 9}, {9, 17}, {2, 4}, {0, 0}, {3, 8} }, // 17 + { {3, 9}, {9, 18}, {3, 5}, {0, 0}, {3, 9} }, // 18 + { {3, 10}, {9, 19}, {3, 5}, {0, 0}, {4, 9} }, // 19 + { {4, 10}, {9, 20}, {3, 5}, {0, 0}, {4, 10} }, // 20 + { {4, 11}, {10, 21}, {3, 5}, {0, 0}, {4, 10} }, // 21 + { {4, 11}, {11, 22}, {3, 5}, {0, 0}, {4, 11} }, // 22 + { {4, 12}, {12, 23}, {3, 6}, {0, 0}, {4, 11} }, // 23 + { {4, 12}, {12, 24}, {3, 6}, {0, 0}, {5, 12} }, // 24 + { {5, 13}, {12, 25}, {3, 6}, {0, 0}, {5, 12} }, // 25 + { {5, 13}, {12, 26}, {4, 7}, {0, 0}, {5, 13} }, // 26 + { {5, 14}, {13, 27}, {4, 7}, {0, 0}, {5, 13} }, // 27 + { {5, 14}, {14, 28}, {4, 7}, {0, 0}, {5, 14} }, // 28 + { {5, 15}, {14, 29}, {4, 8}, {0, 0}, {6, 14} }, // 29 + { {5, 15}, {15, 30}, {4, 8}, {0, 0}, {6, 15} }, // 30 }; TShortPoolCfg HybridCpuTable[MaxPreparedCpuCount + 1][5] { @@ -81,32 +80,31 @@ namespace { { {1, 2}, {0, 3}, {1, 1}, {0, 0}, {1, 1} }, // 3 { {1, 2}, {1, 4}, {1, 1}, {0, 0}, {1, 2} }, // 4 { {1, 2}, {2, 5}, {1, 1}, {0, 0}, {1, 2} }, // 5 - { {1, 2}, {2, 6}, {1, 1}, {0, 0}, {2, 2} }, // 6 + { {1, 2}, {2, 6}, {1, 1}, {0, 0}, {2, 3} }, // 6 { {2, 3}, {2, 7}, {1, 2}, {0, 0}, {2, 3} }, // 7 - { {2, 3}, {3, 8}, {1, 2}, {0, 0}, {2, 3} }, // 8 + { {2, 3}, {3, 8}, {1, 2}, {0, 0}, {2, 4} }, // 8 { {2, 4}, {3, 9}, {1, 2}, {0, 0}, {3, 4} }, // 9 - { {3, 4}, {3, 10}, {1, 2}, {0, 0}, {3, 4} }, // 10 + { {3, 4}, {3, 10}, {1, 2}, {0, 0}, {3, 5} }, // 10 { {3, 5}, {4, 11}, {1, 2}, {0, 0}, {3, 5} }, // 11 - { {3, 5}, {4, 12}, {1, 3}, {0, 0}, {4, 5} }, // 12 + { {3, 5}, {4, 12}, {1, 3}, {0, 0}, {4, 6} }, // 12 { {4, 6}, {4, 13}, {1, 3}, {0, 0}, {4, 6} }, // 13 - { {4, 6}, {5, 14}, {1, 3}, {0, 0}, {4, 6} }, // 14 + { {4, 6}, {5, 14}, {1, 3}, {0, 0}, {4, 7} }, // 14 { {4, 7}, {5, 15}, {1, 3}, {0, 0}, {5, 7} }, // 15 - { {5, 7}, {5, 16}, {1, 3}, {0, 0}, {5, 7} }, // 16 + { {5, 7}, {5, 16}, {1, 3}, {0, 0}, {5, 8} }, // 16 { {5, 8}, {6, 17}, {1, 4}, {0, 0}, {5, 8} }, // 17 - { {5, 8}, {6, 18}, {1, 4}, {0, 0}, {6, 8} }, // 18 + { {5, 8}, {6, 18}, {1, 4}, {0, 0}, {6, 9} }, // 18 { {6, 9}, {6, 19}, {1, 4}, {0, 0}, {6, 9} }, // 19 - { {6, 9}, {7, 20}, {1, 4}, {0, 0}, {6, 9} }, // 20 - { {6, 10}, {7, 21}, {1, 4}, {0, 0}, {7, 10} }, // 21 - { {7, 10}, {7, 22}, {1, 5}, {0, 0}, {7, 10} }, // 22 - { {7, 11}, {8, 23}, {1, 5}, {0, 0}, {7, 11} }, // 23 - { {7, 11}, {8, 24}, {1, 5}, {0, 0}, {8, 11} }, // 24 + { {6, 9}, {7, 20}, {1, 4}, {0, 0}, {6, 10} }, // 20 + { {6, 10}, {7, 21}, {1, 4}, {0, 0}, {7, 10} }, // 21 + { {7, 10}, {7, 22}, {1, 5}, {0, 0}, {7, 11} }, // 22 + { {7, 11}, {8, 23}, {1, 5}, {0, 0}, {7, 11} }, // 23 + { {7, 11}, {8, 24}, {1, 5}, {0, 0}, {8, 12} }, // 24 { {8, 12}, {8, 25}, {1, 5}, {0, 0}, {8, 12} }, // 25 - { {8, 12}, {9, 26}, {1, 5}, {0, 0}, {8, 12} }, // 26 + { {8, 12}, {9, 26}, {1, 5}, {0, 0}, {8, 13} }, // 26 { {8, 13}, {9, 27}, {1, 6}, {0, 0}, {9, 13} }, // 27 - { {9, 13}, {9, 28}, {1, 6}, {0, 0}, {9, 13} }, // 28 + { {9, 13}, {9, 28}, {1, 6}, {0, 0}, {9, 14} }, // 28 { {9, 14}, {10, 29}, {1, 6}, {0, 0}, {9, 14} }, // 29 - { {9, 14}, {10, 30}, {1, 6}, {0, 0}, {10, 14} }, // 30 - { {10, 15}, {10, 31}, {1, 6}, {0, 0}, {10, 15} }, // 31 + { {9, 14}, {10, 30}, {1, 6}, {0, 0}, {10, 15} }, // 30 }; TShortPoolCfg StorageCpuTable[MaxPreparedCpuCount + 1][5] { @@ -116,32 +114,31 @@ namespace { { {1, 3}, {0, 3}, {1, 1}, {0, 0}, {1, 1} }, // 3 { {1, 4}, {1, 4}, {1, 1}, {0, 0}, {1, 2} }, // 4 { {2, 5}, {1, 5}, {1, 1}, {0, 0}, {1, 2} }, // 5 - { {3, 6}, {1, 6}, {1, 1}, {0, 0}, {1, 2} }, // 6 + { {3, 6}, {1, 6}, {1, 1}, {0, 0}, {1, 3} }, // 6 { {4, 7}, {1, 7}, {1, 2}, {0, 0}, {1, 3} }, // 7 - { {5, 8}, {1, 8}, {1, 2}, {0, 0}, {1, 3} }, // 8 + { {5, 8}, {1, 8}, {1, 2}, {0, 0}, {1, 4} }, // 8 { {5, 9}, {1, 9}, {1, 2}, {0, 0}, {2, 4} }, // 9 - { {6, 10}, {1, 10}, {1, 2}, {0, 0}, {2, 4} }, // 10 - { {6, 11}, {1, 11}, {2, 3}, {0, 0}, {2, 4} }, // 11 - { {7, 12}, {1, 12}, {2, 3}, {0, 0}, {2, 5} }, // 12 - { {8, 13}, {1, 13}, {2, 3}, {0, 0}, {2, 5} }, // 13 - { {8, 14}, {1, 14}, {2, 3}, {0, 0}, {3, 6} }, // 14 - { {9, 15}, {1, 15}, {2, 4}, {0, 0}, {3, 6} }, // 15 - { {10, 16}, {1, 16}, {2, 4}, {0, 0}, {3, 6} }, // 16 - { {11, 17}, {1, 17}, {2, 4}, {0, 0}, {3, 7} }, // 17 - { {11, 18}, {1, 18}, {3, 5}, {0, 0}, {3, 7} }, // 18 - { {11, 19}, {1, 19}, {3, 5}, {0, 0}, {4, 8} }, // 19 - { {12, 20}, {1, 20}, {3, 5}, {0, 0}, {4, 8} }, // 20 - { {13, 21}, {1, 21}, {3, 5}, {0, 0}, {4, 8} }, // 21 - { {14, 22}, {1, 22}, {3, 6}, {0, 0}, {4, 9} }, // 22 - { {15, 23}, {1, 23}, {3, 6}, {0, 0}, {4, 9} }, // 23 - { {15, 24}, {1, 24}, {3, 6}, {0, 0}, {5, 10} }, // 24 - { {16, 25}, {1, 25}, {3, 6}, {0, 0}, {5, 10} }, // 25 - { {16, 26}, {1, 26}, {4, 7}, {0, 0}, {5, 10} }, // 26 - { {17, 27}, {1, 27}, {4, 7}, {0, 0}, {5, 11} }, // 27 - { {18, 28}, {1, 28}, {4, 7}, {0, 0}, {5, 11} }, // 28 - { {18, 29}, {1, 29}, {4, 7}, {0, 0}, {6, 12} }, // 29 - { {19, 30}, {1, 30}, {4, 8}, {0, 0}, {6, 12} }, // 30 - { {20, 31}, {1, 31}, {4, 8}, {0, 0}, {6, 12} }, // 31 + { {6, 10}, {1, 10}, {1, 2}, {0, 0}, {2, 5} }, // 10 + { {6, 11}, {1, 11}, {2, 3}, {0, 0}, {2, 5} }, // 11 + { {7, 12}, {1, 12}, {2, 3}, {0, 0}, {2, 6} }, // 12 + { {8, 13}, {1, 13}, {2, 3}, {0, 0}, {2, 6} }, // 13 + { {8, 14}, {1, 14}, {2, 3}, {0, 0}, {3, 7} }, // 14 + { {9, 15}, {1, 15}, {2, 4}, {0, 0}, {3, 7} }, // 15 + { {10, 16}, {1, 16}, {2, 4}, {0, 0}, {3, 8} }, // 16 + { {11, 17}, {1, 17}, {2, 4}, {0, 0}, {3, 8} }, // 17 + { {11, 18}, {1, 18}, {3, 5}, {0, 0}, {3, 9} }, // 18 + { {11, 19}, {1, 19}, {3, 5}, {0, 0}, {4, 9} }, // 19 + { {12, 20}, {1, 20}, {3, 5}, {0, 0}, {4, 10} }, // 20 + { {13, 21}, {1, 21}, {3, 5}, {0, 0}, {4, 10} }, // 21 + { {14, 22}, {1, 22}, {3, 6}, {0, 0}, {4, 11} }, // 22 + { {15, 23}, {1, 23}, {3, 6}, {0, 0}, {4, 11} }, // 23 + { {15, 24}, {1, 24}, {3, 6}, {0, 0}, {5, 12} }, // 24 + { {16, 25}, {1, 25}, {3, 6}, {0, 0}, {5, 12} }, // 25 + { {16, 26}, {1, 26}, {4, 7}, {0, 0}, {5, 13} }, // 26 + { {17, 27}, {1, 27}, {4, 7}, {0, 0}, {5, 13} }, // 27 + { {18, 28}, {1, 28}, {4, 7}, {0, 0}, {5, 14} }, // 28 + { {18, 29}, {1, 29}, {4, 7}, {0, 0}, {6, 14} }, // 29 + { {19, 30}, {1, 30}, {4, 8}, {0, 0}, {6, 15} }, // 30 }; i16 GetIOThreadCount(i16 cpuCount) { diff --git a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp index 3d6d55867061..e8594242878e 100644 --- a/ydb/core/driver_lib/run/kikimr_services_initializers.cpp +++ b/ydb/core/driver_lib/run/kikimr_services_initializers.cpp @@ -870,6 +870,7 @@ void TBasicServicesInitializer::InitializeServices(NActors::TActorSystemSetup* s GET_FIELD_FROM_CONFIG(MaxSpansInBatch) GET_FIELD_FROM_CONFIG(MaxBytesInBatch) GET_FIELD_FROM_CONFIG(SpanExportTimeoutSeconds) + GET_FIELD_FROM_CONFIG(MaxExportRequestsInflight) #undef GET_FIELD_FROM_CONFIG diff --git a/ydb/core/fq/libs/compute/ydb/actors_factory.cpp b/ydb/core/fq/libs/compute/ydb/actors_factory.cpp index 9db333a97da9..fc6d2606c0f4 100644 --- a/ydb/core/fq/libs/compute/ydb/actors_factory.cpp +++ b/ydb/core/fq/libs/compute/ydb/actors_factory.cpp @@ -59,8 +59,9 @@ struct TActorFactory : public IActorFactory { std::unique_ptr CreateResultWriter(const NActors::TActorId& parent, const NActors::TActorId& connector, const NActors::TActorId& pinger, - const NKikimr::NOperationId::TOperationId& operationId) const override { - return CreateResultWriterActor(Params, parent, connector, pinger, operationId, Counters); + const NKikimr::NOperationId::TOperationId& operationId, + bool operationEntryExpected) const override { + return CreateResultWriterActor(Params, parent, connector, pinger, operationId, operationEntryExpected, Counters); } std::unique_ptr CreateResourcesCleaner(const NActors::TActorId& parent, diff --git a/ydb/core/fq/libs/compute/ydb/actors_factory.h b/ydb/core/fq/libs/compute/ydb/actors_factory.h index ae85da060f7a..4c91bfc1b6bf 100644 --- a/ydb/core/fq/libs/compute/ydb/actors_factory.h +++ b/ydb/core/fq/libs/compute/ydb/actors_factory.h @@ -28,7 +28,8 @@ struct IActorFactory : public TThrRefBase { virtual std::unique_ptr CreateResultWriter(const NActors::TActorId& parent, const NActors::TActorId& connector, const NActors::TActorId& pinger, - const NKikimr::NOperationId::TOperationId& operationId) const = 0; + const NKikimr::NOperationId::TOperationId& operationId, + bool operationEntryExpected) const = 0; virtual std::unique_ptr CreateResourcesCleaner(const NActors::TActorId& parent, const NActors::TActorId& connector, const NYdb::TOperation::TOperationId& operationId) const = 0; diff --git a/ydb/core/fq/libs/compute/ydb/result_writer_actor.cpp b/ydb/core/fq/libs/compute/ydb/result_writer_actor.cpp index ef6da5653b4a..b6f0ff8efc05 100644 --- a/ydb/core/fq/libs/compute/ydb/result_writer_actor.cpp +++ b/ydb/core/fq/libs/compute/ydb/result_writer_actor.cpp @@ -202,13 +202,14 @@ class TResultWriterActor : public TBaseComputeActor { } }; - TResultWriterActor(const TRunActorParams& params, const TActorId& parent, const TActorId& connector, const TActorId& pinger, const NKikimr::NOperationId::TOperationId& operationId, const ::NYql::NCommon::TServiceCounters& queryCounters) + TResultWriterActor(const TRunActorParams& params, const TActorId& parent, const TActorId& connector, const TActorId& pinger, const NKikimr::NOperationId::TOperationId& operationId, bool operationEntryExpected, const ::NYql::NCommon::TServiceCounters& queryCounters) : TBaseComputeActor(queryCounters, "ResultWriter") , Params(params) , Parent(parent) , Connector(connector) , Pinger(pinger) , OperationId(operationId) + , OperationEntryExpected(operationEntryExpected) , Counters(GetStepCountersSubgroup()) {} @@ -246,6 +247,13 @@ class TResultWriterActor : public TBaseComputeActor { void Handle(const TEvYdbCompute::TEvGetOperationResponse::TPtr& ev) { const auto& response = *ev.Get()->Get(); + if (!OperationEntryExpected && response.Status == NYdb::EStatus::NOT_FOUND) { + LOG_I("Operation has been already removed"); + Send(Parent, new TEvYdbCompute::TEvResultWriterResponse({}, NYdb::EStatus::SUCCESS)); + CompleteAndPassAway(); + return; + } + if (response.Status != NYdb::EStatus::SUCCESS) { LOG_E("Can't get operation: " << ev->Get()->Issues.ToOneLineString()); Send(Parent, new TEvYdbCompute::TEvResultWriterResponse(ev->Get()->Issues, ev->Get()->Status)); @@ -314,6 +322,7 @@ class TResultWriterActor : public TBaseComputeActor { TActorId Connector; TActorId Pinger; NKikimr::NOperationId::TOperationId OperationId; + const bool OperationEntryExpected; TCounters Counters; TInstant StartTime; TString FetchToken; @@ -325,8 +334,9 @@ std::unique_ptr CreateResultWriterActor(const TRunActorParams& const TActorId& connector, const TActorId& pinger, const NKikimr::NOperationId::TOperationId& operationId, + bool operationEntryExpected, const ::NYql::NCommon::TServiceCounters& queryCounters) { - return std::make_unique(params, parent, connector, pinger, operationId, queryCounters); + return std::make_unique(params, parent, connector, pinger, operationId, operationEntryExpected, queryCounters); } } diff --git a/ydb/core/fq/libs/compute/ydb/result_writer_actor.h b/ydb/core/fq/libs/compute/ydb/result_writer_actor.h index ee24d14772b1..ca6c1454d42b 100644 --- a/ydb/core/fq/libs/compute/ydb/result_writer_actor.h +++ b/ydb/core/fq/libs/compute/ydb/result_writer_actor.h @@ -13,6 +13,7 @@ std::unique_ptr CreateResultWriterActor(const TRunActorParams& const NActors::TActorId& connector, const NActors::TActorId& pinger, const NKikimr::NOperationId::TOperationId& operationId, + bool operationEntryExpected, const ::NYql::NCommon::TServiceCounters& queryCounters); } diff --git a/ydb/core/fq/libs/compute/ydb/ydb_run_actor.cpp b/ydb/core/fq/libs/compute/ydb/ydb_run_actor.cpp index e382035e183a..a18d2043c5be 100644 --- a/ydb/core/fq/libs/compute/ydb/ydb_run_actor.cpp +++ b/ydb/core/fq/libs/compute/ydb/ydb_run_actor.cpp @@ -98,10 +98,14 @@ class TYdbRunActor : public NActors::TActorBootstrapped { } void Handle(const TEvYdbCompute::TEvStatusTrackerResponse::TPtr& ev) { + if (CancelOperationIsRunning("StatusTrackerResponse (aborting). ")) { + return; + } + auto& response = *ev->Get(); if (response.Status == NYdb::EStatus::NOT_FOUND) { // FAILING / ABORTING_BY_USER / ABORTING_BY_SYSTEM LOG_I("StatusTrackerResponse (not found). Status: " << response.Status << " Issues: " << response.Issues.ToOneLineString()); - Register(ActorFactory->CreateFinalizer(Params, SelfId(), Pinger, ExecStatus, Params.Status).release()); + CreateFinalizer(Params.Status); return; } @@ -114,13 +118,17 @@ class TYdbRunActor : public NActors::TActorBootstrapped { Params.Status = response.ComputeStatus; LOG_I("StatusTrackerResponse (success) " << response.Status << " ExecStatus: " << static_cast(response.ExecStatus) << " Issues: " << response.Issues.ToOneLineString()); if (response.ExecStatus == NYdb::NQuery::EExecStatus::Completed) { - Register(ActorFactory->CreateResultWriter(SelfId(), Connector, Pinger, Params.OperationId).release()); + Register(ActorFactory->CreateResultWriter(SelfId(), Connector, Pinger, Params.OperationId, true).release()); } else { - Register(ActorFactory->CreateResourcesCleaner(SelfId(), Connector, Params.OperationId).release()); + CreateResourcesCleaner(); } } void Handle(const TEvYdbCompute::TEvResultWriterResponse::TPtr& ev) { + if (CancelOperationIsRunning("ResultWriterResponse (aborting). ")) { + return; + } + auto& response = *ev->Get(); if (response.Status != NYdb::EStatus::SUCCESS) { LOG_I("ResultWriterResponse (failed). Status: " << response.Status << " Issues: " << response.Issues.ToOneLineString()); @@ -128,7 +136,7 @@ class TYdbRunActor : public NActors::TActorBootstrapped { return; } LOG_I("ResultWriterResponse (success) " << response.Status << " Issues: " << response.Issues.ToOneLineString()); - Register(ActorFactory->CreateResourcesCleaner(SelfId(), Connector, Params.OperationId).release()); + CreateResourcesCleaner(); } void Handle(const TEvYdbCompute::TEvResourcesCleanerResponse::TPtr& ev) { @@ -139,20 +147,21 @@ class TYdbRunActor : public NActors::TActorBootstrapped { return; } LOG_I("ResourcesCleanerResponse (success) " << response.Status << " Issues: " << response.Issues.ToOneLineString()); - Register(ActorFactory->CreateFinalizer(Params, SelfId(), Pinger, ExecStatus, IsAborted ? FederatedQuery::QueryMeta::ABORTING_BY_USER : Params.Status).release()); + CreateFinalizer(IsAborted ? FederatedQuery::QueryMeta::ABORTING_BY_USER : Params.Status); } void Handle(const TEvYdbCompute::TEvFinalizerResponse::TPtr ev) { // Pinger is no longer available at this place. // The query can be restarted only after the expiration of lease in case of error auto& response = *ev->Get(); - LOG_I("FinalizerResponse ( " << (response.Status == NYdb::EStatus::SUCCESS ? "success" : "failed") << ") " << response.Status << " Issues: " << response.Issues.ToOneLineString()); + LOG_I("FinalizerResponse ( " << (response.Status == NYdb::EStatus::SUCCESS ? "success" : "failed") << " ) " << response.Status << " Issues: " << response.Issues.ToOneLineString()); FinishAndPassAway(); } void Handle(TEvents::TEvQueryActionResult::TPtr& ev) { LOG_I("QueryActionResult: " << FederatedQuery::QueryAction_Name(ev->Get()->Action)); - if (Params.OperationId.GetKind() != Ydb::TOperationId::UNUSED && !IsAborted) { + // Start cancel operation only when StatusTracker or ResultWriter is running + if (Params.OperationId.GetKind() != Ydb::TOperationId::UNUSED && !IsAborted && !FinalizationStarted) { IsAborted = true; Register(ActorFactory->CreateStopper(SelfId(), Connector, Params.OperationId).release()); } @@ -166,7 +175,7 @@ class TYdbRunActor : public NActors::TActorBootstrapped { return; } LOG_I("StopperResponse (success) " << response.Status << " Issues: " << response.Issues.ToOneLineString()); - Register(ActorFactory->CreateResourcesCleaner(SelfId(), Connector, Params.OperationId).release()); + CreateResourcesCleaner(); } void Run() { // recover points @@ -183,9 +192,9 @@ class TYdbRunActor : public NActors::TActorBootstrapped { break; case FederatedQuery::QueryMeta::COMPLETING: if (Params.OperationId.GetKind() != Ydb::TOperationId::UNUSED) { - Register(ActorFactory->CreateResultWriter(SelfId(), Connector, Pinger, Params.OperationId).release()); + Register(ActorFactory->CreateResultWriter(SelfId(), Connector, Pinger, Params.OperationId, false).release()); } else { - Register(ActorFactory->CreateFinalizer(Params, SelfId(), Pinger, ExecStatus, Params.Status).release()); + CreateFinalizer(Params.Status); } break; case FederatedQuery::QueryMeta::FAILING: @@ -194,7 +203,7 @@ class TYdbRunActor : public NActors::TActorBootstrapped { if (Params.OperationId.GetKind() != Ydb::TOperationId::UNUSED) { Register(ActorFactory->CreateStatusTracker(SelfId(), Connector, Pinger, Params.OperationId).release()); } else { - Register(ActorFactory->CreateFinalizer(Params, SelfId(), Pinger, ExecStatus, Params.Status).release()); + CreateFinalizer(Params.Status); } break; default: @@ -220,8 +229,28 @@ class TYdbRunActor : public NActors::TActorBootstrapped { PassAway(); } + void CreateResourcesCleaner() { + FinalizationStarted = true; + Register(ActorFactory->CreateResourcesCleaner(SelfId(), Connector, Params.OperationId).release()); + } + + void CreateFinalizer(FederatedQuery::QueryMeta::ComputeStatus status) { + FinalizationStarted = true; + Register(ActorFactory->CreateFinalizer(Params, SelfId(), Pinger, ExecStatus, status).release()); + } + + bool CancelOperationIsRunning(const TString& stage) const { + if (!IsAborted) { + return false; + } + + LOG_I(stage << "Stop task execution, cancel operation now is running"); + return true; + } + private: bool IsAborted = false; + bool FinalizationStarted = false; TActorId FetcherId; NYdb::NQuery::EExecStatus ExecStatus = NYdb::NQuery::EExecStatus::Unspecified; TRunActorParams Params; diff --git a/ydb/core/grpc_services/base/base.h b/ydb/core/grpc_services/base/base.h index 9a9f7cc3c68f..f3bbb9463444 100644 --- a/ydb/core/grpc_services/base/base.h +++ b/ydb/core/grpc_services/base/base.h @@ -382,8 +382,6 @@ class IRequestProxyCtx return NJaegerTracing::TRequestDiscriminator::EMPTY; }; - virtual const TString& GetInternalRequestType() const = 0; - // validation virtual bool Validate(TString& error) = 0; @@ -504,10 +502,6 @@ class TRefreshTokenImpl void StartTracing(NWilson::TSpan&& /*span*/) override {} void LegacyFinishSpan() override {} - const TString& GetInternalRequestType() const final { - static const TString empty = ""; - return empty; - } void UpdateAuthState(NYdbGrpc::TAuthState::EAuthState state) override { State_.State = state; @@ -896,10 +890,6 @@ class TGRpcRequestBiStreamWrapper Span_.End(); } - const TString& GetInternalRequestType() const final { - return TRequest::descriptor()->full_name(); - } - // IRequestCtxBase // void AddAuditLogPart(const TStringBuf&, const TString&) override { @@ -1312,10 +1302,6 @@ class TGRpcRequestWrapperImpl void LegacyFinishSpan() override {} - const TString& GetInternalRequestType() const final { - return TRequest::descriptor()->full_name(); - } - void ReplyGrpcError(grpc::StatusCode code, const TString& msg, const TString& details = "") { Ctx_->ReplyError(code, msg, details); } diff --git a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp index 0c7cdb69961c..a5122d01188a 100644 --- a/ydb/core/kqp/executer_actor/kqp_data_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_data_executer.cpp @@ -312,6 +312,8 @@ class TKqpDataExecuter : public TKqpExecuterBase { return TActorBootstrapped::SelfId(); } + TString BuildMemoryLimitExceptionMessage() const { + if (Request.TxAlloc) { + return TStringBuilder() << "Memory limit exception at " << CurrentStateFuncName() + << ", current limit is " << Request.TxAlloc->Alloc.GetLimit() << " bytes."; + } + return TStringBuilder() << "Memory limit exception at " << CurrentStateFuncName(); + } + void ReportEventElapsedTime() { if (Stats) { ui64 elapsedMicros = TlsActivationContext->GetCurrentEventTicksAsSeconds() * 1'000'000; diff --git a/ydb/core/kqp/executer_actor/kqp_literal_executer.cpp b/ydb/core/kqp/executer_actor/kqp_literal_executer.cpp index 3d1383bf0810..6c2df7106680 100644 --- a/ydb/core/kqp/executer_actor/kqp_literal_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_literal_executer.cpp @@ -146,24 +146,6 @@ class TKqpLiteralExecuter { return; } - ui64 mkqlMemoryLimit = Request.MkqlMemoryLimit > 0 - ? Request.MkqlMemoryLimit - : 1_GB; - - auto& alloc = Request.TxAlloc->Alloc; - auto rmConfig = GetKqpResourceManager()->GetConfig(); - ui64 mkqlInitialLimit = std::min(mkqlMemoryLimit, rmConfig.GetMkqlLightProgramMemoryLimit()); - ui64 mkqlMaxLimit = std::max(mkqlMemoryLimit, rmConfig.GetMkqlLightProgramMemoryLimit()); - alloc.SetLimit(mkqlInitialLimit); - - // TODO: KIKIMR-15350 - alloc.Ref().SetIncreaseMemoryLimitCallback([this, &alloc, mkqlMaxLimit](ui64 currentLimit, ui64 required) { - if (required < mkqlMaxLimit) { - LOG_D("Increase memory limit from " << currentLimit << " to " << required); - alloc.SetLimit(required); - } - }); - // task runner settings ComputeCtx = std::make_unique(); RunnerContext = CreateTaskRunnerContext(ComputeCtx.get(), &Request.TxAlloc->TypeEnv); diff --git a/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp b/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp index d6f6f1e9813b..62d799717d20 100644 --- a/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp +++ b/ydb/core/kqp/executer_actor/kqp_scan_executer.cpp @@ -90,6 +90,8 @@ class TKqpScanExecuter : public TKqpExecuterBase #include #include +#include #include #include @@ -111,8 +112,28 @@ class TKqpLogicalOptTransformer : public TOptimizeTransformerBase { } TMaybeNode RewriteAggregate(TExprBase node, TExprContext& ctx) { - TExprBase output = DqRewriteAggregate(node, ctx, TypesCtx, false, KqpCtx.Config->HasOptEnableOlapPushdown() || KqpCtx.Config->HasOptUseFinalizeByKey(), KqpCtx.Config->HasOptUseFinalizeByKey()); - DumpAppliedRule("RewriteAggregate", node.Ptr(), output.Ptr(), ctx); + TMaybeNode output; + auto aggregate = node.Cast(); + auto hopSetting = GetSetting(aggregate.Settings().Ref(), "hopping"); + if (hopSetting) { + auto input = aggregate.Input().Maybe(); + if (!input) { + return node; + } + output = NHopping::RewriteAsHoppingWindow( + node, + ctx, + input.Cast(), + false, // analyticsHopping + TDuration::MilliSeconds(TDqSettings::TDefault::WatermarksLateArrivalDelayMs), + true, // defaultWatermarksMode + true); // syncActor + } else { + output = DqRewriteAggregate(node, ctx, TypesCtx, false, KqpCtx.Config->HasOptEnableOlapPushdown() || KqpCtx.Config->HasOptUseFinalizeByKey(), KqpCtx.Config->HasOptUseFinalizeByKey()); + } + if (output) { + DumpAppliedRule("RewriteAggregate", node.Ptr(), output.Cast().Ptr(), ctx); + } return output; } diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.cpp b/ydb/core/kqp/session_actor/kqp_session_actor.cpp index 13e221e164de..04c47ab0baba 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.cpp +++ b/ydb/core/kqp/session_actor/kqp_session_actor.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -619,6 +620,23 @@ class TKqpSessionActor : public TActorBootstrapped { QueryState->TxId = UlidGen.Next(); QueryState->TxCtx = MakeIntrusive(false, AppData()->FunctionRegistry, AppData()->TimeProvider, AppData()->RandomProvider, Config->EnableKqpImmediateEffects); + + auto& alloc = QueryState->TxCtx->TxAlloc; + ui64 mkqlInitialLimit = Settings.MkqlInitialMemoryLimit; + + const auto& queryLimitsProto = Settings.TableService.GetQueryLimits(); + const auto& phaseLimitsProto = queryLimitsProto.GetPhaseLimits(); + ui64 mkqlMaxLimit = phaseLimitsProto.GetComputeNodeMemoryLimitBytes(); + mkqlMaxLimit = mkqlMaxLimit ? mkqlMaxLimit : ui64(Settings.MkqlMaxMemoryLimit); + + alloc->Alloc.SetLimit(mkqlInitialLimit); + alloc->Alloc.Ref().SetIncreaseMemoryLimitCallback([this, &alloc, mkqlMaxLimit](ui64 currentLimit, ui64 required) { + if (required < mkqlMaxLimit) { + LOG_D("Increase memory limit from " << currentLimit << " to " << required); + alloc->Alloc.SetLimit(required); + } + }); + QueryState->QueryData = std::make_shared(QueryState->TxCtx->TxAlloc); QueryState->TxCtx->SetIsolationLevel(settings); QueryState->TxCtx->OnBeginQuery(); @@ -748,6 +766,8 @@ class TKqpSessionActor : public TActorBootstrapped { } } catch(const yexception& ex) { ythrow TRequestFail(Ydb::StatusIds::BAD_REQUEST) << ex.what(); + } catch(const TMemoryLimitExceededException&) { + ythrow TRequestFail(Ydb::StatusIds::BAD_REQUEST) << BuildMemoryLimitExceptionMessage(); } return true; } @@ -2058,6 +2078,9 @@ class TKqpSessionActor : public TActorBootstrapped { ReplyQueryError(ex.Status, ex.what(), ex.Issues); } catch (const yexception& ex) { InternalError(ex.what()); + } catch (const TMemoryLimitExceededException&) { + ReplyQueryError(Ydb::StatusIds::INTERNAL_ERROR, + BuildMemoryLimitExceptionMessage()); } } @@ -2097,6 +2120,9 @@ class TKqpSessionActor : public TActorBootstrapped { ReplyQueryError(ex.Status, ex.what(), ex.Issues); } catch (const yexception& ex) { InternalError(ex.what()); + } catch (const TMemoryLimitExceededException&) { + ReplyQueryError(Ydb::StatusIds::UNDETERMINED, + BuildMemoryLimitExceptionMessage()); } } @@ -2132,14 +2158,24 @@ class TKqpSessionActor : public TActorBootstrapped { } } catch (const yexception& ex) { InternalError(ex.what()); + } catch (const TMemoryLimitExceededException&) { + ReplyQueryError(Ydb::StatusIds::INTERNAL_ERROR, + BuildMemoryLimitExceptionMessage()); } } STATEFN(FinalCleanupState) { - switch (ev->GetTypeRewrite()) { - hFunc(TEvents::TEvGone, HandleFinalCleanup); - hFunc(TEvents::TEvUndelivered, HandleNoop); - hFunc(TEvKqpSnapshot::TEvCreateSnapshotResponse, Handle); + try { + switch (ev->GetTypeRewrite()) { + hFunc(TEvents::TEvGone, HandleFinalCleanup); + hFunc(TEvents::TEvUndelivered, HandleNoop); + hFunc(TEvKqpSnapshot::TEvCreateSnapshotResponse, Handle); + } + } catch (const yexception& ex) { + InternalError(ex.what()); + } catch (const TMemoryLimitExceededException&) { + ReplyQueryError(Ydb::StatusIds::INTERNAL_ERROR, + BuildMemoryLimitExceptionMessage()); } } @@ -2172,6 +2208,15 @@ class TKqpSessionActor : public TActorBootstrapped { } } + TString BuildMemoryLimitExceptionMessage() const { + if (QueryState && QueryState->TxCtx) { + return TStringBuilder() << "Memory limit exception at " << CurrentStateFuncName() + << ", current limit is " << QueryState->TxCtx->TxAlloc->Alloc.GetLimit() << " bytes."; + } else { + return TStringBuilder() << "Memory limit exception at " << CurrentStateFuncName(); + } + } + void ProcessTopicOps(TEvTxProxySchemeCache::TEvNavigateKeySetResult::TPtr& ev) { YQL_ENSURE(ev->Get()->Request); if (ev->Get()->Request->Cookie < QueryId) { diff --git a/ydb/core/kqp/session_actor/kqp_session_actor.h b/ydb/core/kqp/session_actor/kqp_session_actor.h index 763e43de343d..3dc392551620 100644 --- a/ydb/core/kqp/session_actor/kqp_session_actor.h +++ b/ydb/core/kqp/session_actor/kqp_session_actor.h @@ -7,6 +7,7 @@ #include #include +#include #include namespace NKikimr::NKqp { @@ -19,9 +20,12 @@ struct TKqpWorkerSettings { NKikimrConfig::TTableServiceConfig TableService; NKikimrConfig::TQueryServiceConfig QueryService; + TControlWrapper MkqlInitialMemoryLimit; + TControlWrapper MkqlMaxMemoryLimit; + TKqpDbCountersPtr DbCounters; - TKqpWorkerSettings(const TString& cluster, const TString& database, + explicit TKqpWorkerSettings(const TString& cluster, const TString& database, const NKikimrConfig::TTableServiceConfig& tableServiceConfig, const NKikimrConfig::TQueryServiceConfig& queryServiceConfig, TKqpDbCountersPtr dbCounters) @@ -29,7 +33,15 @@ struct TKqpWorkerSettings { , Database(database) , TableService(tableServiceConfig) , QueryService(queryServiceConfig) - , DbCounters(dbCounters) {} + , MkqlInitialMemoryLimit(2097152, 1, Max()) + , MkqlMaxMemoryLimit(1073741824, 1, Max()) + , DbCounters(dbCounters) + { + AppData()->Icb->RegisterSharedControl( + MkqlInitialMemoryLimit, "KqpSession.MkqlInitialMemoryLimit"); + AppData()->Icb->RegisterSharedControl( + MkqlMaxMemoryLimit, "KqpSession.MkqlMaxMemoryLimit"); + } }; IActor* CreateKqpSessionActor(const TActorId& owner, const TString& sessionId, diff --git a/ydb/core/kqp/ut/join/kqp_join_ut.cpp b/ydb/core/kqp/ut/join/kqp_join_ut.cpp index fedaa77073ea..604e38413d56 100644 --- a/ydb/core/kqp/ut/join/kqp_join_ut.cpp +++ b/ydb/core/kqp/ut/join/kqp_join_ut.cpp @@ -1303,6 +1303,65 @@ Y_UNIT_TEST_SUITE(KqpJoin) { UNIT_ASSERT(explain.GetAst().Contains("GraceJoinCore")); } + Y_UNIT_TEST(FullOuterJoinNotNullJoinKey) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + { // init tables + AssertSuccessResult(session.ExecuteSchemeQuery(R"( + --!syntax_v1 + + CREATE TABLE left + ( + Key Int64 NOT NULL, + Value Int64, + PRIMARY KEY (Key) + ); + + CREATE TABLE right + ( + Key Int64 NOT NULL, + Value Int64, + PRIMARY KEY (Key) + ); + )").GetValueSync()); + + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + REPLACE INTO left (Key, Value) VALUES + (1, 10), + (2, 20), + (3, 30); + + REPLACE INTO right (Key, Value) VALUES + (1, 10), + (2, 200), + (3, 300), + (4, 40); + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::SUCCESS); + } + + { + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT l.Key, l.Value, r.Key, r.Value FROM left as l FULL JOIN right as r + ON (l.Value = r.Value AND l.Key = r.Key); + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [[1];[10];[1];[10]]; + [[2];[20];#;#]; + [[3];[30];#;#]; + [#;#;[3];[300]]; + [#;#;[4];[40]]; + [#;#;[2];[200]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + } } } // namespace NKqp diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp index 4a71469dd748..1a1eba5955b8 100644 --- a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp +++ b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp @@ -386,7 +386,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) { TBase::SendDataViaActorSystem(TablePath, batch); } - void FillPKOnly(const ui32 pkKff = 0, const ui32 numRows = 800000) const { + void FillPKOnly(const double pkKff = 0, const ui32 numRows = 800000) const { std::vector builders; builders.emplace_back(std::make_shared>>("pk_int", numRows * pkKff)); NArrow::NConstruction::TRecordBatchConstructor batchBuilder(builders); @@ -3729,8 +3729,8 @@ Y_UNIT_TEST_SUITE(KqpOlap) { Y_UNIT_TEST(StatsSysViewEnumStringBytes) { ui64 rawBytesPK1; ui64 bytesPK1; - auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); { + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); auto settings = TKikimrSettings() .SetWithSampleTables(false); TKikimrRunner kikimr(settings); @@ -3741,6 +3741,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) { helper.GetVolumes(rawBytesPK1, bytesPK1, false); } + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); ui64 rawBytesUnpack1PK = 0; ui64 bytesUnpack1PK = 0; ui64 rawBytesPackAndUnpack2PK; @@ -3895,19 +3896,31 @@ Y_UNIT_TEST_SUITE(KqpOlap) { NOlap::NDataSharing::TDestinationSession session(std::make_shared(), pathIdsRemap, sessionId, transferContext); Runner.GetTestServer().GetRuntime()->Send(MakePipePeNodeCacheID(false), NActors::TActorId(), new TEvPipeCache::TEvForward( new NOlap::NDataSharing::NEvents::TEvProposeFromInitiator(session), destination, false)); - while (!CSTransferStatus->GetProposed()) { - Sleep(TDuration::Seconds(1)); - Cerr << "WAIT_PROPOSING..." << Endl; + { + const TInstant start = TInstant::Now(); + while (!CSTransferStatus->GetProposed() && TInstant::Now() - start < TDuration::Seconds(10)) { + Sleep(TDuration::Seconds(1)); + Cerr << "WAIT_PROPOSING..." << Endl; + } + AFL_VERIFY(CSTransferStatus->GetProposed()); } Runner.GetTestServer().GetRuntime()->Send(MakePipePeNodeCacheID(false), NActors::TActorId(), new TEvPipeCache::TEvForward( new NOlap::NDataSharing::NEvents::TEvConfirmFromInitiator(sessionId), destination, false)); - while (!CSTransferStatus->GetConfirmed()) { - Sleep(TDuration::Seconds(1)); - Cerr << "WAIT_CONFIRMED..." << Endl; + { + const TInstant start = TInstant::Now(); + while (!CSTransferStatus->GetConfirmed() && TInstant::Now() - start < TDuration::Seconds(10)) { + Sleep(TDuration::Seconds(1)); + Cerr << "WAIT_CONFIRMED..." << Endl; + } + AFL_VERIFY(CSTransferStatus->GetConfirmed()); } - while (!CSTransferStatus->GetFinished()) { - Sleep(TDuration::Seconds(1)); - Cerr << "WAIT_FINISHED..." << Endl; + { + const TInstant start = TInstant::Now(); + while (!CSTransferStatus->GetFinished() && TInstant::Now() - start < TDuration::Seconds(10)) { + Sleep(TDuration::Seconds(1)); + Cerr << "WAIT_FINISHED..." << Endl; + } + AFL_VERIFY(CSTransferStatus->GetFinished()); } CSTransferStatus->Reset(); } @@ -3915,6 +3928,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) { Y_UNIT_TEST(BlobsSharingSplit1_1) { auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + csController->SetCompactionControl(NYDBTest::EOptimizerCompactionWeightControl::Disable); auto settings = TKikimrSettings().SetWithSampleTables(false); TKikimrRunner kikimr(settings); @@ -3929,10 +3943,44 @@ Y_UNIT_TEST_SUITE(KqpOlap) { AFL_VERIFY(pathIds.size() == 1)("count", pathIds.size())("ids", JoinSeq(",", pathIds)); Sleep(TDuration::Seconds(1)); TSharingActor(kikimr).Execute(shardIds[0], {shardIds[1]}, false, NOlap::TSnapshot(TInstant::Now().MilliSeconds(), 1232123), {pathIds[0]}); + AFL_VERIFY(!csController->IsTrivialLinks()); + } + + Y_UNIT_TEST(BlobsSharingSplit1_1_clean) { + auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + csController->SetCompactionControl(NYDBTest::EOptimizerCompactionWeightControl::Disable); + csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetReadTimeoutClean(TDuration::Seconds(1)); + + auto settings = TKikimrSettings().SetWithSampleTables(false); + TKikimrRunner kikimr(settings); + Tests::NCommon::TLoggerInit(kikimr).Initialize(); + TTypedLocalHelper helper("", kikimr, "olapTable", "olapStore12"); + helper.CreateTestOlapTable(2, 2); + helper.FillPKOnly(0, 80000); + + auto shardIds = csController->GetShardActualIds(); + AFL_VERIFY(shardIds.size() == 2)("count", shardIds.size())("ids", JoinSeq(",", shardIds)); + auto pathIds = csController->GetPathIds(shardIds[0]); + AFL_VERIFY(pathIds.size() == 1)("count", pathIds.size())("ids", JoinSeq(",", pathIds)); + Sleep(TDuration::Seconds(1)); + TSharingActor(kikimr).Execute(shardIds[0], {shardIds[1]}, false, NOlap::TSnapshot(TInstant::Now().MilliSeconds(), 1232123), {pathIds[0]}); + AFL_VERIFY(!csController->IsTrivialLinks()); + helper.FillPKOnly(0.8, 80000); + + csController->SetCompactionControl(NYDBTest::EOptimizerCompactionWeightControl::Force); + const auto start = TInstant::Now(); + while (!csController->IsTrivialLinks() && TInstant::Now() - start < TDuration::Seconds(15000)) { + Cerr << "WAIT_TRIVIAL_LINKS..." << Endl; + Sleep(TDuration::Seconds(1)); + } + AFL_VERIFY(csController->IsTrivialLinks()); + csController->CheckInvariants(); } Y_UNIT_TEST(BlobsSharingSplit3_1) { auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + csController->SetCompactionControl(NYDBTest::EOptimizerCompactionWeightControl::Disable); auto settings = TKikimrSettings() .SetWithSampleTables(false); @@ -3948,10 +3996,12 @@ Y_UNIT_TEST_SUITE(KqpOlap) { AFL_VERIFY(pathIds.size() == 1)("count", pathIds.size())("ids", JoinSeq(",", pathIds)); Sleep(TDuration::Seconds(1)); TSharingActor(kikimr).Execute(shardIds[0], {shardIds[1], shardIds[2], shardIds[3]}, false, NOlap::TSnapshot(TInstant::Now().MilliSeconds(), 1232123), {pathIds[0]}); + AFL_VERIFY(!csController->IsTrivialLinks()); } Y_UNIT_TEST(BlobsSharingSplit1_3_1) { auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + csController->SetCompactionControl(NYDBTest::EOptimizerCompactionWeightControl::Disable); auto settings = TKikimrSettings() .SetWithSampleTables(false); @@ -3970,10 +4020,14 @@ Y_UNIT_TEST_SUITE(KqpOlap) { TSharingActor(kikimr).Execute(shardIds[2], {shardIds[0]}, false, NOlap::TSnapshot(TInstant::Now().MilliSeconds(), 1232123), {pathIds[0]}); TSharingActor(kikimr).Execute(shardIds[3], {shardIds[0]}, false, NOlap::TSnapshot(TInstant::Now().MilliSeconds(), 1232123), {pathIds[0]}); TSharingActor(kikimr).Execute(shardIds[0], {shardIds[1], shardIds[2], shardIds[3]}, true, NOlap::TSnapshot(TInstant::Now().MilliSeconds(), 1232123), {pathIds[0]}); + AFL_VERIFY(!csController->IsTrivialLinks()); } - Y_UNIT_TEST(BlobsSharingSplit1_3_2_1) { + Y_UNIT_TEST(BlobsSharingSplit1_3_2_1_clean) { auto csController = NYDBTest::TControllers::RegisterCSControllerGuard(); + csController->SetCompactionControl(NYDBTest::EOptimizerCompactionWeightControl::Disable); + csController->SetPeriodicWakeupActivationPeriod(TDuration::Seconds(1)); + csController->SetReadTimeoutClean(TDuration::Seconds(1)); auto settings = TKikimrSettings() .SetWithSampleTables(false); @@ -3991,9 +4045,20 @@ Y_UNIT_TEST_SUITE(KqpOlap) { TSharingActor(kikimr).Execute(shardIds[1], {shardIds[0]}, false, NOlap::TSnapshot(TInstant::Now().MilliSeconds(), 1232123), {pathIds[0]}); TSharingActor(kikimr).Execute(shardIds[2], {shardIds[0]}, false, NOlap::TSnapshot(TInstant::Now().MilliSeconds(), 1232123), {pathIds[0]}); TSharingActor(kikimr).Execute(shardIds[3], {shardIds[0]}, false, NOlap::TSnapshot(TInstant::Now().MilliSeconds(), 1232123), {pathIds[0]}); - helper.FillPKOnly(1, 800000); + AFL_VERIFY(!csController->IsTrivialLinks()); + helper.FillPKOnly(0.9, 800000); TSharingActor(kikimr).Execute(shardIds[3], {shardIds[2]}, false, NOlap::TSnapshot(TInstant::Now().MilliSeconds(), 1232123), {pathIds[0]}); + AFL_VERIFY(!csController->IsTrivialLinks()); TSharingActor(kikimr).Execute(shardIds[0], {shardIds[1], shardIds[2]}, true, NOlap::TSnapshot(TInstant::Now().MilliSeconds(), 1232123), {pathIds[0]}); + AFL_VERIFY(!csController->IsTrivialLinks()); + csController->SetCompactionControl(NYDBTest::EOptimizerCompactionWeightControl::Force); + const auto start = TInstant::Now(); + while (!csController->IsTrivialLinks() && TInstant::Now() - start < TDuration::Seconds(15)) { + Cerr << "WAIT_TRIVIAL_LINKS..." << Endl; + Sleep(TDuration::Seconds(1)); + } + AFL_VERIFY(csController->IsTrivialLinks()); + csController->CheckInvariants(); } Y_UNIT_TEST(SelectLimit1ManyShards) { @@ -5309,7 +5374,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) { Y_UNIT_TEST(Json_GetValue) { TAggregationTestCase testCase; testCase.SetQuery(R"( - SELECT id, JSON_VALUE(jsonval, "$.col1-proxy"), JSON_VALUE(jsondoc, "$.col1") FROM `/Root/tableWithNulls` + SELECT id, JSON_VALUE(jsonval, "$.col1"), JSON_VALUE(jsondoc, "$.col1") FROM `/Root/tableWithNulls` WHERE JSON_VALUE(jsonval, "$.col1") = "val1" AND id = 1; )") #if SSA_RUNTIME_VERSION >= 3U diff --git a/ydb/core/kqp/ut/opt/kqp_agg_ut.cpp b/ydb/core/kqp/ut/opt/kqp_agg_ut.cpp index 516192e1e19d..6aa539e02e5f 100644 --- a/ydb/core/kqp/ut/opt/kqp_agg_ut.cpp +++ b/ydb/core/kqp/ut/opt/kqp_agg_ut.cpp @@ -90,6 +90,29 @@ Y_UNIT_TEST_SUITE(KqpAgg) { ])", FormatResultSetYson(result.GetResultSet(0))); } + Y_UNIT_TEST(AggWithHop) { + TKikimrRunner kikimr; + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + auto result = session.ExecuteDataQuery(R"( + --!syntax_v1 + + SELECT + Text, + CAST(COUNT(*) as Int32) as Count, + SUM(Data) + FROM EightShard + GROUP BY HOP(CAST(Key AS Timestamp?), "PT1M", "PT1M", "PT1M"), Text + ORDER BY Text; + )", TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString()); + CompareYson(R"([ + [["Value1"];[8];[15]]; + [["Value2"];[8];[16]]; + [["Value3"];[8];[17]] + ])", FormatResultSetYson(result.GetResultSet(0))); + } + Y_UNIT_TEST(GroupByLimit) { TKikimrRunner kikimr; auto db = kikimr.GetTableClient(); diff --git a/ydb/core/kqp/ut/query/kqp_limits_ut.cpp b/ydb/core/kqp/ut/query/kqp_limits_ut.cpp index 2c7f85a4854f..2b0218e43e7b 100644 --- a/ydb/core/kqp/ut/query/kqp_limits_ut.cpp +++ b/ydb/core/kqp/ut/query/kqp_limits_ut.cpp @@ -17,6 +17,87 @@ namespace { } Y_UNIT_TEST_SUITE(KqpLimits) { + Y_UNIT_TEST(KqpMkqlMemoryLimitException) { + TKikimrRunner kikimr; + CreateLargeTable(kikimr, 10, 10, 1'000'000, 1); + + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_SLOW_LOG, NActors::NLog::PRI_ERROR); + + TControlWrapper mkqlInitialMemoryLimit; + TControlWrapper mkqlMaxMemoryLimit; + + mkqlInitialMemoryLimit = kikimr.GetTestServer().GetRuntime()->GetAppData().Icb->RegisterSharedControl( + mkqlInitialMemoryLimit, "KqpSession.MkqlInitialMemoryLimit"); + mkqlMaxMemoryLimit = kikimr.GetTestServer().GetRuntime()->GetAppData().Icb->RegisterSharedControl( + mkqlMaxMemoryLimit, "KqpSession.MkqlMaxMemoryLimit"); + + mkqlInitialMemoryLimit = 1_KB; + mkqlMaxMemoryLimit = 1_KB; + + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + auto result = session.ExecuteDataQuery(Q1_(R"( + SELECT * FROM `/Root/LargeTable`; + )"), TTxControl::BeginTx().CommitTx()).ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::PRECONDITION_FAILED); + } + + Y_UNIT_TEST(LargeParametersAndMkqlFailure) { + auto app = NKikimrConfig::TAppConfig(); + app.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlLightProgramMemoryLimit(1'000'000'000); + + TKikimrRunner kikimr(app); + CreateLargeTable(kikimr, 0, 0, 0); + + kikimr.GetTestServer().GetRuntime()->SetLogPriority(NKikimrServices::KQP_SLOW_LOG, NActors::NLog::PRI_ERROR); + + auto db = kikimr.GetTableClient(); + auto session = db.CreateSession().GetValueSync().GetSession(); + + TControlWrapper mkqlInitialMemoryLimit; + TControlWrapper mkqlMaxMemoryLimit; + + mkqlInitialMemoryLimit = kikimr.GetTestServer().GetRuntime()->GetAppData().Icb->RegisterSharedControl( + mkqlInitialMemoryLimit, "KqpSession.MkqlInitialMemoryLimit"); + mkqlMaxMemoryLimit = kikimr.GetTestServer().GetRuntime()->GetAppData().Icb->RegisterSharedControl( + mkqlMaxMemoryLimit, "KqpSession.MkqlMaxMemoryLimit"); + + + mkqlInitialMemoryLimit = 1_KB; + mkqlMaxMemoryLimit = 1_KB; + + auto paramsBuilder = db.GetParamsBuilder(); + auto& rowsParam = paramsBuilder.AddParam("$rows"); + + rowsParam.BeginList(); + for (ui32 i = 0; i < 100; ++i) { + rowsParam.AddListItem() + .BeginStruct() + .AddMember("Key") + .OptionalUint64(i) + .AddMember("KeyText") + .OptionalString(TString(5000, '0' + i % 10)) + .AddMember("Data") + .OptionalInt64(i) + .AddMember("DataText") + .OptionalString(TString(16, '0' + (i + 1) % 10)) + .EndStruct(); + } + rowsParam.EndList(); + rowsParam.Build(); + + auto result = session.ExecuteDataQuery(Q1_(R"( + DECLARE $rows AS List>; + + UPSERT INTO `/Root/LargeTable` + SELECT * FROM AS_TABLE($rows); + )"), TTxControl::BeginTx().CommitTx(), paramsBuilder.Build()).ExtractValueSync(); + result.GetIssues().PrintTo(Cerr); + UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::BAD_REQUEST); + } + Y_UNIT_TEST(DatashardProgramSize) { auto app = NKikimrConfig::TAppConfig(); app.MutableTableServiceConfig()->MutableResourceManager()->SetMkqlLightProgramMemoryLimit(1'000'000'000); diff --git a/ydb/core/mon/mon.cpp b/ydb/core/mon/mon.cpp index 4d31af077a91..8acaa51baf8b 100644 --- a/ydb/core/mon/mon.cpp +++ b/ydb/core/mon/mon.cpp @@ -6,15 +6,19 @@ namespace NActors { using namespace NMonitoring; +using namespace NKikimr; namespace { const std::vector& GetEntries(const TString& ticket) { if (ticket.StartsWith("Bearer")) { - static std::vector entries = { - {NKikimr::TEvTicketParser::TEvAuthorizeTicket::ToPermissions({"ydb.developerApi.get", "ydb.developerApi.update"}), {{"gizmo_id", "gizmo"}}} - }; - return entries; + if (AppData()->AuthConfig.GetUseAccessService() + && (AppData()->DomainsConfig.GetSecurityConfig().ViewerAllowedSIDsSize() > 0 || AppData()->DomainsConfig.GetSecurityConfig().MonitoringAllowedSIDsSize() > 0)) { + static std::vector entries = { + {NKikimr::TEvTicketParser::TEvAuthorizeTicket::ToPermissions({"ydb.developerApi.get", "ydb.developerApi.update"}), {{"gizmo_id", "gizmo"}}} + }; + return entries; + } } static std::vector emptyEntries = {}; return emptyEntries; diff --git a/ydb/core/persqueue/account_read_quoter.cpp b/ydb/core/persqueue/account_read_quoter.cpp index 65ab6d5a8a8b..78b26095b8e8 100644 --- a/ydb/core/persqueue/account_read_quoter.cpp +++ b/ydb/core/persqueue/account_read_quoter.cpp @@ -19,79 +19,50 @@ const TDuration UPDATE_COUNTERS_INTERVAL = TDuration::Seconds(5); const TDuration DO_NOT_QUOTE_AFTER_ERROR_PERIOD = TDuration::Seconds(5); const TString TAccountReadQuoter::READ_QUOTA_ROOT_PATH = "read-quota"; +const TString TAccountWriteQuoter::WRITE_QUOTA_ROOT_PATH = "write-quota"; -constexpr NKikimrServices::TActivity::EType TAccountReadQuoter::ActorActivityType() { - return NKikimrServices::TActivity::PERSQUEUE_ACCOUNT_READ_QUOTER; -} - -TAccountReadQuoter::TAccountReadQuoter( +TBasicAccountQuoter::TBasicAccountQuoter( TActorId tabletActor, TActorId recepient, ui64 tabletId, const NPersQueue::TTopicConverterPtr& topicConverter, const TPartitionId& partition, - const TString& user, - const TTabletCountersBase& counters + const TQuoterParams& params, + ui64 quotaCreditBytes, + const TTabletCountersBase& counters, + const TDuration& doNotQuoteAfterErrorPeriod ) - : TabletActor(tabletActor) - , Recepient(recepient) - , TabletId(tabletId) + : KesusPath(params.KesusPath) + , ResourcePath(params.ResourcePath) , TopicConverter(topicConverter) , Partition(partition) - , User(user) - , ConsumerPath(NPersQueue::ConvertOldConsumerName(user)) - , ReadCreditBytes(AppData()->PQConfig.GetQuotingConfig().GetReadCreditBytes()) + , TabletActor(tabletActor) + , Recepient(recepient) + , TabletId(tabletId) + , CreditBytes(quotaCreditBytes) + , DoNotQuoteAfterErrorPeriod(doNotQuoteAfterErrorPeriod) { Counters.Populate(counters); - - auto userParts = SplitString(ConsumerPath, "/"); // account/folder/topic // account is first element - - const TString account = userParts[0]; - userParts[0] = READ_QUOTA_ROOT_PATH; // read-quota/folder/topic - - const auto& quotingConfig = AppData()->PQConfig.GetQuotingConfig(); - KesusPath = Join("/", quotingConfig.GetQuotersDirectoryPath(), account); - QuotaResourcePath = JoinSeq("/", userParts); - LOG_INFO_S(TActivationContext::AsActorContext(), NKikimrServices::PQ_READ_SPEED_LIMITER, - LimiterDescription() <<" kesus=" << KesusPath << " resource_path=" << QuotaResourcePath); } -void TAccountReadQuoter::InitCounters(const TActorContext& ctx) { - if (CountersInited) { - return; - } - auto counters = AppData(ctx)->Counters; - if (counters) { - QuotaWaitCounter.Reset(new TPercentileCounter( - GetServiceCounters(counters, "pqproxy|consumerReadQuotaWait"), - NPersQueue::GetLabels(TopicConverter), - { - {"Client", User}, - {"ConsumerPath", ConsumerPath}, - {"sensor", "ConsumerReadQuotaWait"} - }, - "Interval", - TVector>{{0, "0ms"}, {1, "1ms"}, {5, "5ms"}, {10, "10ms"}, - {20, "20ms"}, {50, "50ms"}, {100, "100ms"}, {500, "500ms"}, - {1000, "1000ms"}, {2500, "2500ms"}, {5000, "5000ms"}, {10000, "10000ms"}, {9999999, "999999ms"}}, - true - )); - } - CountersInited = true; -} - - -void TAccountReadQuoter::Bootstrap(const TActorContext& ctx) { +void TBasicAccountQuoter::Bootstrap(const TActorContext& ctx) { Become(&TThis::StateWork); ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters); } -void TAccountReadQuoter::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) { - LOG_INFO_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription() << " killed"); +void TBasicAccountQuoter::InitCounters(const TActorContext& ctx) { + if (!CountersInited) { + InitCountersImpl(ctx); + CountersInited = true; + } +} + +void TBasicAccountQuoter::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) { + LOG_INFO_S(ctx, NKikimrServices::PQ_RATE_LIMITER, LimiterDescription() << " killed"); for (const auto& event : Queue) { - auto cookie = event.Event->Get()->Cookie; + auto cookie = event.Request->Get()->Cookie; ReplyPersQueueError( - TabletActor, ctx, TabletId, TopicConverter->GetClientsideName(), Partition, Counters, NKikimrServices::PQ_READ_SPEED_LIMITER, + TabletActor, ctx, TabletId, TopicConverter->GetClientsideName(), Partition, Counters, NKikimrServices::PQ_RATE_LIMITER, cookie, NPersQueue::NErrorCode::INITIALIZING, TStringBuilder() << "Tablet is restarting, topic " << TopicConverter->GetClientsideName() << " (ReadInfo) cookie " << cookie ); @@ -99,41 +70,41 @@ void TAccountReadQuoter::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorConte Die(ctx); } -void TAccountReadQuoter::HandleUpdateCounters(TEvPQ::TEvUpdateCounters::TPtr&, const TActorContext& ctx) { +void TBasicAccountQuoter::HandleUpdateCounters(TEvPQ::TEvUpdateCounters::TPtr&, const TActorContext& ctx) { ctx.Schedule(UPDATE_COUNTERS_INTERVAL, new TEvPQ::TEvUpdateCounters); - ctx.Send(Recepient, new NAccountReadQuoterEvents::TEvCounters(Counters, User)); + ctx.Send(Recepient, MakeCountersUpdateEvent()); } -void TAccountReadQuoter::HandleReadQuotaRequest(NAccountReadQuoterEvents::TEvRequest::TPtr& ev, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, - LimiterDescription() << " quota required for cookie=" << ev->Get()->ReadRequest->Get()->Cookie +void TBasicAccountQuoter::HandleQuotaRequest(NAccountQuoterEvents::TEvRequest::TPtr& ev, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_RATE_LIMITER, + LimiterDescription() << " quota required for cookie=" << ev->Get()->Cookie ); InitCounters(ctx); - bool hasActualErrors = ctx.Now() - LastReportedErrorTime <= DO_NOT_QUOTE_AFTER_ERROR_PERIOD; - if ((QuotaRequestInFlight || !InProcessReadRequestCookies.empty()) && !hasActualErrors) { - Queue.emplace_back(std::move(ev->Get()->ReadRequest), ctx.Now()); + bool hasActualErrors = ctx.Now() - LastReportedErrorTime < DoNotQuoteAfterErrorPeriod; + if (ResourcePath && (QuotaRequestInFlight || !InProcessQuotaRequestCookies.empty()) && !hasActualErrors) { + Queue.emplace_back(ev, ctx.Now()); } else { - ApproveRead(ev->Get()->ReadRequest, ctx.Now(), ctx); + ApproveQuota(ev, ctx.Now(), ctx); } } -void TAccountReadQuoter::HandleReadQuotaConsumed(NAccountReadQuoterEvents::TEvConsumed::TPtr& ev, const TActorContext& ctx) { - ConsumedBytesInCredit += ev->Get()->ReadBytes; - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription() - << "consumed read quota " << ev->Get()->ReadBytes - << " bytes by cookie=" << ev->Get()->ReadRequestCookie - << ", consumed in credit " << ConsumedBytesInCredit << "/" << ReadCreditBytes +void TBasicAccountQuoter::HandleQuotaConsumed(NAccountQuoterEvents::TEvConsumed::TPtr& ev, const TActorContext& ctx) { + ConsumedBytesInCredit += ev->Get()->BytesConsumed; + LOG_DEBUG_S(ctx, NKikimrServices::PQ_RATE_LIMITER, LimiterDescription() + << "consumed read quota " << ev->Get()->BytesConsumed + << " bytes by cookie=" << ev->Get()->RequestCookie + << ", consumed in credit " << ConsumedBytesInCredit << "/" << CreditBytes ); - auto it = InProcessReadRequestCookies.find(ev->Get()->ReadRequestCookie); - Y_ABORT_UNLESS(it != InProcessReadRequestCookies.end()); - InProcessReadRequestCookies.erase(it); + auto it = InProcessQuotaRequestCookies.find(ev->Get()->RequestCookie); + Y_ABORT_UNLESS(it != InProcessQuotaRequestCookies.end()); + InProcessQuotaRequestCookies.erase(it); if (!QuotaRequestInFlight) { - if (ConsumedBytesInCredit >= ReadCreditBytes) { - Send(MakeQuoterServiceID(), + if (ConsumedBytesInCredit >= CreditBytes && ResourcePath) { + TThis::Send(MakeQuoterServiceID(), new TEvQuota::TEvRequest( TEvQuota::EResourceOperator::And, - { TEvQuota::TResourceLeaf(KesusPath, QuotaResourcePath, ConsumedBytesInCredit) }, + { TEvQuota::TResourceLeaf(KesusPath, ResourcePath, ConsumedBytesInCredit) }, TDuration::Max()), 0, ++CurrentQuotaRequestCookie @@ -141,52 +112,173 @@ void TAccountReadQuoter::HandleReadQuotaConsumed(NAccountReadQuoterEvents::TEvCo QuotaRequestInFlight = true; ConsumedBytesInCredit = 0; } else if (!Queue.empty()){ - ApproveRead(std::move(Queue.front().Event), Queue.front().StartWait, ctx); + ApproveQuota(Queue.front().Request, Queue.front().StartWait, ctx); Queue.pop_front(); } } } -void TAccountReadQuoter::HandleClearance(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx) { +void TBasicAccountQuoter::HandleClearance(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx) { QuotaRequestInFlight = false; const ui64 cookie = ev->Cookie; - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, - LimiterDescription() << "Got read quota:" << ev->Get()->Result << ". Cookie: " << cookie + LOG_DEBUG_S(ctx, NKikimrServices::PQ_RATE_LIMITER, + LimiterDescription() << "Got quota from Kesus:" << ev->Get()->Result << ". Cookie: " << cookie ); Y_ABORT_UNLESS(CurrentQuotaRequestCookie == cookie); if (!Queue.empty()) { - ApproveRead(std::move(Queue.front().Event), Queue.front().StartWait, ctx); + ApproveQuota(Queue.front().Request, Queue.front().StartWait, ctx); Queue.pop_front(); } if (Y_UNLIKELY(ev->Get()->Result != TEvQuota::TEvClearance::EResult::Success)) { Y_ABORT_UNLESS(ev->Get()->Result != TEvQuota::TEvClearance::EResult::Deadline); // We set deadline == inf in quota request. if (ctx.Now() - LastReportedErrorTime > TDuration::Minutes(1)) { - LOG_ERROR_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, LimiterDescription() << "Got read quota error: " << ev->Get()->Result); + LOG_ERROR_S(ctx, NKikimrServices::PQ_RATE_LIMITER, LimiterDescription() << "Got quota request error: " << ev->Get()->Result); LastReportedErrorTime = ctx.Now(); } return; } } -void TAccountReadQuoter::TAccountReadQuoter::ApproveRead(TEvPQ::TEvRead::TPtr ev, TInstant startWait, const TActorContext& ctx) { - LOG_DEBUG_S(ctx, NKikimrServices::PQ_READ_SPEED_LIMITER, +void TBasicAccountQuoter::ApproveQuota(NAccountQuoterEvents::TEvRequest::TPtr& ev, TInstant startWait, const TActorContext& ctx) { + LOG_DEBUG_S(ctx, NKikimrServices::PQ_RATE_LIMITER, LimiterDescription() << " approve read for cookie=" << ev->Get()->Cookie ); - InProcessReadRequestCookies.insert(ev->Get()->Cookie); + InProcessQuotaRequestCookies.insert(ev->Get()->Cookie); auto waitTime = ctx.Now() - startWait; - Send(Recepient, new NAccountReadQuoterEvents::TEvResponse(ev.Release(), waitTime)); + TThis::Send(Recepient, new NAccountQuoterEvents::TEvResponse(ev->Release(), waitTime)); if (QuotaWaitCounter) { QuotaWaitCounter->IncFor(waitTime.MilliSeconds()); } } +TQuoterParams TAccountReadQuoter::GetQuoterParams(const TString& user) { + ConsumerPath = NPersQueue::ConvertOldConsumerName(user); + TQuoterParams ret; + auto userParts = SplitString(ConsumerPath, "/"); // account/folder/topic // account is first element + + const TString account = userParts[0]; + userParts[0] = READ_QUOTA_ROOT_PATH; // read-quota/folder/topic + + const auto& quotingConfig = AppData()->PQConfig.GetQuotingConfig(); + ret.KesusPath = Join("/", quotingConfig.GetQuotersDirectoryPath(), account); + ret.ResourcePath = JoinSeq("/", userParts); + return ret; +} + +TAccountReadQuoter::TAccountReadQuoter( + TActorId tabletActor, + TActorId recepient, + ui64 tabletId, + const NPersQueue::TTopicConverterPtr& topicConverter, + const TPartitionId& partition, + const TString& user, + const TTabletCountersBase& counters +) + : TBasicAccountQuoter(tabletActor, recepient, tabletId, topicConverter, partition, GetQuoterParams(user), + AppData()->PQConfig.GetQuotingConfig().GetReadCreditBytes(), counters, DO_NOT_QUOTE_AFTER_ERROR_PERIOD) + , User(user) +{ + LOG_INFO_S(TActivationContext::AsActorContext(), NKikimrServices::PQ_RATE_LIMITER, + LimiterDescription() <<" kesus=" << KesusPath << " resource_path=" << ResourcePath); +} + + +void TAccountReadQuoter::InitCountersImpl(const TActorContext& ctx) { + auto counters = AppData(ctx)->Counters; + if (counters) { + QuotaWaitCounter.Reset(new TPercentileCounter( + GetServiceCounters(counters, "pqproxy|consumerReadQuotaWait"), + NPersQueue::GetLabels(TopicConverter), + { + {"Client", User}, + {"ConsumerPath", ConsumerPath}, + {"sensor", "ConsumerReadQuotaWait"} + }, + "Interval", + TVector>{{0, "0ms"}, {1, "1ms"}, {5, "5ms"}, {10, "10ms"}, + {20, "20ms"}, {50, "50ms"}, {100, "100ms"}, {500, "500ms"}, + {1000, "1000ms"}, {2500, "2500ms"}, {5000, "5000ms"}, {10000, "10000ms"}, {9999999, "999999ms"}}, + true + )); + } +} + +THolder TAccountReadQuoter::MakeCountersUpdateEvent() { + return MakeHolder(Counters, true, User); +} + TString TAccountReadQuoter::LimiterDescription() const { return TStringBuilder() << "topic=" << TopicConverter->GetClientsideName() << ":" << Partition << " user=" << User << ": "; } + +TAccountWriteQuoter::TAccountWriteQuoter( + TActorId tabletActor, + TActorId recepient, + ui64 tabletId, + const NPersQueue::TTopicConverterPtr& topicConverter, + const TPartitionId& partition, + const TTabletCountersBase& counters, + const TActorContext& ctx +) + : TBasicAccountQuoter(tabletActor, recepient, tabletId, topicConverter, partition, + CreateQuoterParams(AppData()->PQConfig, topicConverter, tabletId, ctx), + 0, counters, + TDuration::Zero()) +{ +} + +TQuoterParams TAccountWriteQuoter::CreateQuoterParams( + const NKikimrPQ::TPQConfig& pqConfig, NPersQueue::TTopicConverterPtr topicConverter, + ui64 tabletId, const TActorContext& ctx +) { + TQuoterParams params; + const auto& quotingConfig = pqConfig.GetQuotingConfig(); + Y_ABORT_UNLESS(quotingConfig.GetTopicWriteQuotaEntityToLimit() != NKikimrPQ::TPQConfig::TQuotingConfig::UNSPECIFIED); + auto topicPath = topicConverter->GetFederationPath(); + + auto topicParts = SplitPath(topicPath); // account/folder/topic // account is first element + if (topicParts.size() < 2) { + LOG_WARN_S(ctx, NKikimrServices::PERSQUEUE, + "tablet " << tabletId << " topic '" << topicPath << "' Bad topic name. Disable quoting for topic"); + return params; + } + topicParts[0] = WRITE_QUOTA_ROOT_PATH; // write-quota/folder/topic + params.KesusPath = TStringBuilder() << quotingConfig.GetQuotersDirectoryPath() << "/" << topicConverter->GetAccount(); + params.ResourcePath = JoinPath(topicParts); + + LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, + "topicWriteQuutaResourcePath '" << params.ResourcePath + << "' topicWriteQuoterPath '" << params.KesusPath + << "' account '" << topicConverter->GetAccount() + << "'" + ); + return params; +} + +void TAccountWriteQuoter::InitCountersImpl(const TActorContext&) { +} + +THolder TAccountWriteQuoter::MakeCountersUpdateEvent() { + return MakeHolder(Counters, false, TString{}); +} + +TString TAccountWriteQuoter::LimiterDescription() const { + return TStringBuilder() << "topic=" << TopicConverter->GetClientsideName() << ":" << Partition << " writeQuoter" << ": "; +} + + +constexpr NKikimrServices::TActivity::EType TAccountReadQuoter::ActorActivityType() { + return NKikimrServices::TActivity::PERSQUEUE_ACCOUNT_READ_QUOTER; +} + +constexpr NKikimrServices::TActivity::EType TAccountWriteQuoter::ActorActivityType() { + return NKikimrServices::TActivity::PERSQUEUE_ACCOUNT_WRITE_QUOTER; +} + }// NPQ }// NKikimr diff --git a/ydb/core/persqueue/account_read_quoter.h b/ydb/core/persqueue/account_read_quoter.h index d6c9e309d87a..f1e57b43588e 100644 --- a/ydb/core/persqueue/account_read_quoter.h +++ b/ydb/core/persqueue/account_read_quoter.h @@ -11,69 +11,103 @@ namespace NPQ { class TPercentileCounter; -namespace NAccountReadQuoterEvents { - struct TEvRequest : public TEventLocal { - TEvRequest(TEvPQ::TEvRead::TPtr readRequest) - : ReadRequest(std::move(readRequest)) - {} +struct TQuoterParams { + TString KesusPath; + TString ResourcePath; +}; - TEvPQ::TEvRead::TPtr ReadRequest; +namespace NAccountQuoterEvents { + struct TEvRequest : public TEventLocal { + TEvRequest(ui64 cookie, TAutoPtr request) + : Cookie(cookie) + , Request(std::move(request)) + {} + ui64 Cookie; + TAutoPtr Request; }; - struct TEvResponse : public TEventLocal { - TEvResponse(TEvPQ::TEvRead::TPtr readRequest, TDuration waitTime) - : ReadRequest(std::move(readRequest)) + struct TEvResponse : public TEventLocal { + TEvResponse(TAutoPtr&& request, const TDuration& waitTime) + : Request(std::move(request)) , WaitTime(waitTime) {} - - TEvPQ::TEvRead::TPtr ReadRequest; + TAutoPtr Request; TDuration WaitTime; }; - struct TEvConsumed : public TEventLocal { - TEvConsumed(ui64 readBytes, ui64 readRequestCookie) - : ReadBytes(readBytes) - , ReadRequestCookie(readRequestCookie) + struct TEvConsumed : public TEventLocal { + TEvConsumed(ui64 bytesConsumed, ui64 requestCookie) + : BytesConsumed(bytesConsumed) + , RequestCookie(requestCookie) {} - ui64 ReadBytes; - ui64 ReadRequestCookie; + ui64 BytesConsumed; + ui64 RequestCookie; }; - struct TEvCounters : public TEventLocal { - TEvCounters(const NKikimr::TTabletCountersBase& counters, const TString& user) - : User(user) + struct TEvCounters : public TEventLocal { + TEvCounters(const NKikimr::TTabletCountersBase& counters, bool isReadCounters, const TString& subject) + : Subject(subject) + , IsReadCounters(isReadCounters) { Counters.Populate(counters); } NKikimr::TTabletCountersBase Counters; - const TString User; + const TString Subject; + bool IsReadCounters; }; } -class TAccountReadQuoter : public TActorBootstrapped { +class TBasicAccountQuoter : public TActorBootstrapped { private: - static const TString READ_QUOTA_ROOT_PATH; - struct TQueueEvent { - TQueueEvent(TEvPQ::TEvRead::TPtr&& event, TInstant startWait) - : Event(event) + TQueueEvent(NAccountQuoterEvents::TEvRequest::TPtr request, TInstant startWait) + : Request(std::move(request)) , StartWait(startWait) {} - TEvPQ::TEvRead::TPtr Event; + NAccountQuoterEvents::TEvRequest::TPtr Request; TInstant StartWait; }; -private: +public: + TBasicAccountQuoter( + TActorId tabletActor, + TActorId recepient, + ui64 tabletId, + const NPersQueue::TTopicConverterPtr& topicConverter, + const TPartitionId& partition, + const TQuoterParams& params, + ui64 quotaCreditBytes, + const TTabletCountersBase& counters, + const TDuration& doNotQuoteAfterErrorPeriod + ); + + void Bootstrap(const TActorContext& ctx); + + void InitCounters(const TActorContext& ctx); + void Handle(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx); + void HandleQuotaRequest(NAccountQuoterEvents::TEvRequest::TPtr& ev, const TActorContext& ctx); + void HandleQuotaConsumed(NAccountQuoterEvents::TEvConsumed::TPtr& ev, const TActorContext& ctx); + void HandleClearance(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx); + + void ApproveQuota(NAccountQuoterEvents::TEvRequest::TPtr& ev, TInstant startWait, const TActorContext& ctx); + + void HandleUpdateCounters(TEvPQ::TEvUpdateCounters::TPtr& ev, const TActorContext& ctx); + +protected: + virtual TString LimiterDescription() const = 0; + virtual void InitCountersImpl(const TActorContext& ctx) = 0; + virtual THolder MakeCountersUpdateEvent() = 0; + STFUNC(StateWork) { - TRACE_EVENT(NKikimrServices::PQ_READ_SPEED_LIMITER); + TRACE_EVENT(NKikimrServices::PQ_RATE_LIMITER); switch (ev->GetTypeRewrite()) { HFuncTraced(TEvPQ::TEvUpdateCounters, HandleUpdateCounters); - HFuncTraced(NAccountReadQuoterEvents::TEvRequest, HandleReadQuotaRequest); - HFuncTraced(NAccountReadQuoterEvents::TEvConsumed, HandleReadQuotaConsumed); + HFuncTraced(NAccountQuoterEvents::TEvRequest, HandleQuotaRequest); + HFuncTraced(NAccountQuoterEvents::TEvConsumed, HandleQuotaConsumed); HFuncTraced(TEvQuota::TEvClearance, HandleClearance); HFuncTraced(TEvents::TEvPoisonPill, Handle); default: @@ -81,6 +115,38 @@ class TAccountReadQuoter : public TActorBootstrapped { }; } + TString KesusPath; + TString ResourcePath; + const NPersQueue::TTopicConverterPtr TopicConverter; + THolder QuotaWaitCounter; + TTabletCountersBase Counters; + const TPartitionId Partition; + +private: + const TActorId TabletActor; + const TActorId Recepient; + const ui64 TabletId; + + const ui64 CreditBytes = 0; + + ui64 ConsumedBytesInCredit = 0; + + TDeque Queue; + + bool QuotaRequestInFlight = false; + ui64 CurrentQuotaRequestCookie = 0; + THashSet InProcessQuotaRequestCookies; + + + bool CountersInited = false; + TInstant LastReportedErrorTime; + TDuration DoNotQuoteAfterErrorPeriod; +}; + +class TAccountReadQuoter : public TBasicAccountQuoter { +private: + static const TString READ_QUOTA_ROOT_PATH; + public: static constexpr NKikimrServices::TActivity::EType ActorActivityType(); @@ -94,45 +160,38 @@ class TAccountReadQuoter : public TActorBootstrapped { const TTabletCountersBase& counters ); - void Bootstrap(const TActorContext& ctx); - void InitCounters(const TActorContext& ctx); - void Handle(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx); - void HandleUpdateCounters(TEvPQ::TEvUpdateCounters::TPtr& ev, const TActorContext& ctx); - void HandleReadQuotaRequest(NAccountReadQuoterEvents::TEvRequest::TPtr& ev, const TActorContext& ctx); - void HandleReadQuotaConsumed(NAccountReadQuoterEvents::TEvConsumed::TPtr& ev, const TActorContext& ctx); - void HandleClearance(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx); - - void ApproveRead(TEvPQ::TEvRead::TPtr ev, TInstant startWait, const TActorContext& ctx); -private: - TString LimiterDescription() const; +protected: + void InitCountersImpl(const TActorContext& ctx) override; + TString LimiterDescription() const override; + THolder MakeCountersUpdateEvent() override; private: - const TActorId TabletActor; - const TActorId Recepient; - const ui64 TabletId; - const NPersQueue::TTopicConverterPtr TopicConverter; - const TPartitionId Partition; + TQuoterParams GetQuoterParams(const TString& user); const TString User; - const TString ConsumerPath; - const ui64 ReadCreditBytes; + TString ConsumerPath; +}; - ui64 ConsumedBytesInCredit = 0; - TString KesusPath; - TString QuotaResourcePath; +class TAccountWriteQuoter : public TBasicAccountQuoter { +private: +static const TString WRITE_QUOTA_ROOT_PATH; - TDeque Queue; +static TQuoterParams CreateQuoterParams(const NKikimrPQ::TPQConfig& pqConfig, + NPersQueue::TTopicConverterPtr topicConverter, + ui64 tabletId, const TActorContext& ctx); - bool QuotaRequestInFlight = false; - ui64 CurrentQuotaRequestCookie = 0; - THashSet InProcessReadRequestCookies; +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType(); + TAccountWriteQuoter(TActorId tabletActor, TActorId recepient, ui64 tabletId, + const NPersQueue::TTopicConverterPtr& topicConverter, const TPartitionId& partition, + const TTabletCountersBase& counters, const TActorContext& ctx); - TTabletCountersBase Counters; - THolder QuotaWaitCounter; - bool CountersInited = false; - TInstant LastReportedErrorTime; +protected: + void InitCountersImpl(const TActorContext& ctx) override; + TString LimiterDescription() const override; + THolder MakeCountersUpdateEvent() override; }; }// NPQ diff --git a/ydb/core/persqueue/events/internal.h b/ydb/core/persqueue/events/internal.h index 77c18b41339b..6b0b7efe8d9e 100644 --- a/ydb/core/persqueue/events/internal.h +++ b/ydb/core/persqueue/events/internal.h @@ -6,10 +6,12 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -36,7 +38,7 @@ namespace NPQ { TMap> Reads; TMaybe Client; TCacheServiceData() = delete; - + TCacheServiceData(ui32 generation) : Generation(generation) {} @@ -73,11 +75,6 @@ namespace NPQ { ui64 CumulativeSize; }; - struct TSeqNoRange { - ui64 Min; - ui64 Max; - }; - struct TErrorInfo { NPersQueue::NErrorCode::EErrorCode ErrorCode; TString ErrorStr; @@ -141,10 +138,10 @@ struct TEvPQ { EvSplitMessageGroup, EvUpdateCounters, EvMirrorerCounters, - EvAccountReadQuotaRequest, - EvAccountReadQuotaResponse, - EvAccountReadQuotaConsumed, - EvAccountReadQuotaCounters, + EvAccountQuotaRequest, + EvAccountQuotaResponse, + EvAccountQuotaConsumed, + EvAccountQuotaCounters, EvRetryWrite, EvInitCredentials, EvCredentialsCreated, @@ -163,7 +160,8 @@ struct TEvPQ { EvSubDomainStatus, EvStatsWakeup, EvRequestQuota, - EvApproveQuota, + EvApproveReadQuota, + EvApproveWriteQuota, EvConsumed, EvQuotaUpdated, EvAccountQuotaCountersUpdated, @@ -903,35 +901,52 @@ struct TEvPQ { }; struct TEvRequestQuota : public TEventLocal { - TEvRequestQuota(TEvPQ::TEvRead::TPtr readRequest) - : - ReadRequest(std::move(readRequest)) + TEvRequestQuota(ui64 cookie, TAutoPtr&& request) + : Cookie(cookie) + , Request(std::move(request)) {} - TEvPQ::TEvRead::TPtr ReadRequest; + ui64 Cookie; + TAutoPtr Request; }; - struct TEvApproveQuota : public TEventLocal { - TEvApproveQuota(TEvPQ::TEvRead::TPtr readRequest, TDuration waitTime) - : - ReadRequest(std::move(readRequest)), - WaitTime(std::move(waitTime)) + struct TEvApproveReadQuota : public TEventLocal { + TEvApproveReadQuota(TEvPQ::TEvRead::TPtr readRequest, TDuration& waitTime) + : ReadRequest(readRequest) + , WaitTime(std::move(waitTime)) {} TEvPQ::TEvRead::TPtr ReadRequest; TDuration WaitTime; }; + struct TEvApproveWriteQuota : public TEventLocal { + TEvApproveWriteQuota(ui64 requestCookie, const TDuration& accountWaitTime, const TDuration& partitionWaitTime) + : Cookie(requestCookie) + , AccountQuotaWaitTime(accountWaitTime) + , PartitionQuotaWaitTime(partitionWaitTime) + {} + ui64 Cookie; + TDuration AccountQuotaWaitTime; + TDuration PartitionQuotaWaitTime; + }; + struct TEvConsumed : public TEventLocal { - TEvConsumed(ui64 readBytes, ui64 readRequestCookie, const TString& consumer) - : ReadBytes(readBytes), - ReadRequestCookie(readRequestCookie), + TEvConsumed(ui64 consumedBytes, ui64 requestCookie, const TString& consumer) + : ConsumedBytes(consumedBytes), + RequestCookie(requestCookie), Consumer(consumer) {} - ui64 ReadBytes; - ui64 ReadRequestCookie; + TEvConsumed(ui64 consumedBytes) + : ConsumedBytes(consumedBytes) + , IsOverhead(true) + {} + + ui64 ConsumedBytes; + ui64 RequestCookie; TString Consumer; + bool IsOverhead = false; }; struct TEvConsumerRemoved : public TEventLocal { @@ -979,7 +994,7 @@ struct TEvPQ { NPQ::TDirectReadKey ReadKey; std::shared_ptr Response; }; - + struct TEvPublishDirectRead : public TEventLocal { TEvPublishDirectRead(const NPQ::TDirectReadKey& readKey, ui32 tabletGeneration) : ReadKey(readKey) @@ -988,7 +1003,7 @@ struct TEvPQ { NPQ::TDirectReadKey ReadKey; ui32 TabletGeneration; }; - + struct TEvForgetDirectRead : public TEventLocal { TEvForgetDirectRead(const NPQ::TDirectReadKey& readKey, ui32 tabletGeneration) : TabletGeneration(tabletGeneration) @@ -997,7 +1012,7 @@ struct TEvPQ { ui32 TabletGeneration; NPQ::TDirectReadKey ReadKey; }; - + struct TEvGetFullDirectReadData : public TEventLocal { TEvGetFullDirectReadData() = default; TEvGetFullDirectReadData(const NPQ::TReadSessionKey& key, ui32 generation) @@ -1035,21 +1050,22 @@ struct TEvPQ { }; struct TEvGetWriteInfoResponse : public TEventLocal { + TEvGetWriteInfoResponse() = default; TEvGetWriteInfoResponse(ui32 cookie, - THashMap&& seqNo, + NPQ::TSourceIdMap&& srcIdInfo, std::deque&& bodyKeys, - TVector&& head) : + TVector&& blobsFromHead) : Cookie(cookie), - SeqNo(std::move(seqNo)), + SrcIdInfo(std::move(srcIdInfo)), BodyKeys(std::move(bodyKeys)), - Head(std::move(head)) + BlobsFromHead(std::move(blobsFromHead)) { } ui32 Cookie; // InternalPartitionId - THashMap SeqNo; // SourceId -> (MinSeqNo, MaxSeqNo) + NPQ::TSourceIdMap SrcIdInfo; std::deque BodyKeys; - TVector Head; + TVector BlobsFromHead; }; struct TEvGetWriteInfoError : public TEventLocal { diff --git a/ydb/core/persqueue/partition.cpp b/ydb/core/persqueue/partition.cpp index 2f2bb4a5be68..a0dd38ee35c8 100644 --- a/ydb/core/persqueue/partition.cpp +++ b/ydb/core/persqueue/partition.cpp @@ -95,7 +95,7 @@ bool TPartition::IsActive() const { bool TPartition::CanWrite() const { if (PartitionConfig == nullptr) { - // Old format without AllPartitions configuration field. + // Old format without AllPartitions configuration field. // It is not split/merge partition. return true; } @@ -107,10 +107,19 @@ bool TPartition::CanWrite() const { // Pending configuration tx inactivate this partition. return false; } + + if (ClosedInternalPartition) { + return false; + } + return IsActive(); } bool TPartition::CanEnqueue() const { + if (ClosedInternalPartition) { + return false; + } + return IsActive(); } @@ -166,7 +175,7 @@ TPartition::TPartition(ui64 tabletId, const TPartitionId& partition, const TActo const NPersQueue::TTopicConverterPtr& topicConverter, TString dcId, bool isServerless, const NKikimrPQ::TPQTabletConfig& tabletConfig, const TTabletCountersBase& counters, bool subDomainOutOfSpace, ui32 numChannels, bool newPartition, - TVector distrTxs) + TVector distrTxs, const TActorId& writeQuoterActorId) : Initializer(this) , TabletID(tabletId) , TabletGeneration(tabletGeneration) @@ -204,6 +213,7 @@ TPartition::TPartition(ui64 tabletId, const TPartitionId& partition, const TActo , DiskIsFull(false) , SubDomainOutOfSpace(subDomainOutOfSpace) , HasDataReqNum(0) + , WriteQuotaTrackerActor(writeQuoterActorId) , AvgWriteBytes{{TDuration::Seconds(1), 1000}, {TDuration::Minutes(1), 1000}, {TDuration::Hours(1), 2000}, {TDuration::Days(1), 2000}} , AvgReadBytes(TDuration::Minutes(1), 1000) , AvgQuotaBytes{{TDuration::Seconds(1), 1000}, {TDuration::Minutes(1), 1000}, {TDuration::Hours(1), 2000}, {TDuration::Days(1), 2000}} @@ -228,7 +238,6 @@ void TPartition::EmplaceResponse(TMessage&& message, const TActorContext& ctx) { const auto now = ctx.Now(); Responses.emplace_back( message.Body, - WriteQuota->GetQuotedTime(now) - message.QuotedTime, (now - TInstant::Zero()) - message.QueueTime, now ); @@ -281,7 +290,7 @@ ui64 TPartition::ImportantClientsMinOffset() const { minOffset = Min(minOffset, curOffset); } - return minOffset; + return minOffset; } void TPartition::HandleWakeup(const TActorContext& ctx) { @@ -336,6 +345,7 @@ void TPartition::HandleWakeup(const TActorContext& ctx) { WriteCycleStartTime = now; WriteStartTime = now; TopicQuotaWaitTimeForCurrentBlob = TDuration::Zero(); + PartitionQuotaWaitTimeForCurrentBlob = TDuration::Zero(); WritesTotal.Inc(); Become(&TThis::StateWrite); AddMetaKey(request.Get()); @@ -490,6 +500,7 @@ void TPartition::Handle(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) } Send(ReadQuotaTrackerActor, new TEvents::TEvPoisonPill()); + Send(WriteQuotaTrackerActor, new TEvents::TEvPoisonPill()); Die(ctx); } @@ -642,7 +653,7 @@ void TPartition::Handle(TEvPQ::TEvPartitionStatus::TPtr& ev, const TActorContext result.SetSourceIdCount(SourceIdStorage.GetInMemorySourceIds().size()); result.SetSourceIdRetentionPeriodSec((ctx.Now() - SourceIdStorage.MinAvailableTimestamp(ctx.Now())).Seconds()); - result.SetWriteBytesQuota(WriteQuota->GetTotalSpeed()); + result.SetWriteBytesQuota(TotalPartitionWriteSpeed); bool inactivePartition = !IsActive(); @@ -945,6 +956,26 @@ void TPartition::Handle(TEvPQ::TEvTxRollback::TPtr& ev, const TActorContext& ctx ContinueProcessTxsAndUserActs(ctx); } +void TPartition::Handle(TEvPQ::TEvGetWriteInfoRequest::TPtr& ev, const TActorContext& ctx) { + if (ClosedInternalPartition || CurrentStateFunc() != &TThis::StateIdle || !Requests.empty()) { + auto* response = new TEvPQ::TEvGetWriteInfoError(Partition.InternalPartitionId, + "Write info requested while writes are not complete"); + ctx.Send(ev->Sender, response); + ClosedInternalPartition = true; + return; + } + ClosedInternalPartition = true; + auto response = new TEvPQ::TEvGetWriteInfoResponse(); + response->Cookie = Partition.InternalPartitionId; + response->BodyKeys = std::move(DataKeysBody); + response->SrcIdInfo = std::move(SourceIdStorage.ExtractInMemorySourceIds()); + ui32 rcount = 0, rsize = 0; + ui64 insideHeadOffset = 0; + + response->BlobsFromHead = std::move(GetReadRequestFromHead(0, 0, std::numeric_limits::max(), std::numeric_limits::max(), 0, &rcount, &rsize, &insideHeadOffset, 0)); + ctx.Send(ev->Sender, response); +} + void TPartition::Handle(TEvPQ::TEvGetMaxSeqNoRequest::TPtr& ev, const TActorContext& ctx) { auto response = MakeHolder(ev->Get()->Cookie); NKikimrClient::TResponse& resp = *response->Response; @@ -981,7 +1012,7 @@ void TPartition::Handle(TEvPQ::TEvBlobResponse::TPtr& ev, const TActorContext& c auto it = ReadInfo.find(cookie); Y_ABORT_UNLESS(it != ReadInfo.end()); - + TReadInfo info = std::move(it->second); ReadInfo.erase(it); @@ -992,7 +1023,7 @@ void TPartition::Handle(TEvPQ::TEvBlobResponse::TPtr& ev, const TActorContext& c info.Destination, GetSizeLag(info.Offset), Tablet, Config.GetMeteringMode() )); const auto& resp = dynamic_cast(answer.Event.Get())->Response; - + if (HasError(*ev->Get())) { if (info.IsSubscription) { TabletCounters.Cumulative()[COUNTER_PQ_READ_SUBSCRIPTION_ERROR].Increment(1); @@ -1232,10 +1263,9 @@ bool TPartition::UpdateCounters(const TActorContext& ctx, bool force) { PartitionCountersLabeled->GetCounters()[METRIC_GAPS_COUNT].Set(gapsCount); } - ui64 speed = WriteQuota->GetTotalSpeed(); - if (speed != PartitionCountersLabeled->GetCounters()[METRIC_WRITE_QUOTA_BYTES].Get()) { + if (TotalPartitionWriteSpeed != PartitionCountersLabeled->GetCounters()[METRIC_WRITE_QUOTA_BYTES].Get()) { haveChanges = true; - PartitionCountersLabeled->GetCounters()[METRIC_WRITE_QUOTA_BYTES].Set(speed); + PartitionCountersLabeled->GetCounters()[METRIC_WRITE_QUOTA_BYTES].Set(TotalPartitionWriteSpeed); } ui32 id = METRIC_TOTAL_WRITE_SPEED_1; @@ -1263,8 +1293,8 @@ bool TPartition::UpdateCounters(const TActorContext& ctx, bool force) { } Y_ABORT_UNLESS(id == METRIC_MAX_QUOTA_SPEED_4 + 1); - if (WriteQuota->GetTotalSpeed()) { - ui64 quotaUsage = ui64(AvgQuotaBytes[1].GetValue()) * 1000000 / WriteQuota->GetTotalSpeed() / 60; + if (TotalPartitionWriteSpeed) { + ui64 quotaUsage = ui64(AvgQuotaBytes[1].GetValue()) * 1000000 / TotalPartitionWriteSpeed / 60; if (quotaUsage != PartitionCountersLabeled->GetCounters()[METRIC_WRITE_QUOTA_USAGE].Get()) { haveChanges = true; PartitionCountersLabeled->GetCounters()[METRIC_WRITE_QUOTA_USAGE].Set(quotaUsage); @@ -1327,7 +1357,7 @@ void TPartition::Handle(NReadQuoterEvents::TEvQuotaUpdated::TPtr& ev, const TAct userInfo->LabeledCounters->GetCounters()[METRIC_READ_QUOTA_PER_CONSUMER_BYTES].Set(quota); } } - PartitionCountersLabeled->GetCounters()[METRIC_READ_QUOTA_PARTITION_TOTAL_BYTES].Set(ev->Get()->UpdatedTotalPartitionQuota); + PartitionCountersLabeled->GetCounters()[METRIC_READ_QUOTA_PARTITION_TOTAL_BYTES].Set(ev->Get()->UpdatedTotalPartitionReadQuota); } void TPartition::Handle(TEvKeyValue::TEvResponse::TPtr& ev, const TActorContext& ctx) { @@ -1871,9 +1901,8 @@ void TPartition::EndChangePartitionConfig(const NKikimrPQ::TPQTabletConfig& conf UsersInfoStorage->UpdateConfig(Config); - WriteQuota->UpdateConfigIfChanged(Config.GetPartitionConfig().GetBurstSize(), Config.GetPartitionConfig().GetWriteSpeedInBytesPerSecond()); - Send(ReadQuotaTrackerActor, new TEvPQ::TEvChangePartitionConfig(TopicConverter, Config)); + Send(WriteQuotaTrackerActor, new TEvPQ::TEvChangePartitionConfig(TopicConverter, Config)); if (Config.GetPartitionConfig().HasMirrorFrom()) { if (Mirrorer) { @@ -2513,50 +2542,43 @@ ui32 TPartition::NextChannel(bool isHead, ui32 blobSize) { return res; } -void TPartition::Handle(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx) { - const ui64 cookie = ev->Cookie; +void TPartition::Handle(TEvPQ::TEvApproveWriteQuota::TPtr& ev, const TActorContext& ctx) { + const ui64 cookie = ev->Get()->Cookie; LOG_DEBUG_S( ctx, NKikimrServices::PERSQUEUE, "Got quota." << " Topic: \"" << TopicName() << "\"." << - " Partition: " << Partition << ": " << ev->Get()->Result << "." << - " Cookie: " << cookie + " Partition: " << Partition << ": Cookie: " << cookie ); - // Check - if (Y_UNLIKELY(ev->Get()->Result != TEvQuota::TEvClearance::EResult::Success)) { - // We set deadline == inf in quota request. - Y_ABORT_UNLESS(ev->Get()->Result != TEvQuota::TEvClearance::EResult::Deadline); - LOG_ERROR_S( - ctx, NKikimrServices::PERSQUEUE, - "Got quota error." << - " Topic: \"" << TopicName() << "\"." << - " Partition " << Partition << ": " << ev->Get()->Result - ); - ctx.Send(Tablet, new TEvents::TEvPoisonPill()); - return; - } - + // Search for proper request Y_ABORT_UNLESS(TopicQuotaRequestCookie == cookie); TopicQuotaRequestCookie = 0; + TopicQuotaConsumedCookie = cookie; Y_ASSERT(!WaitingForPreviousBlobQuota()); + Y_ABORT_UNLESS(PendingWriteRequest); + WritePendingBlob(); // Metrics - TopicQuotaWaitTimeForCurrentBlob = StartTopicQuotaWaitTimeForCurrentBlob ? TActivationContext::Now() - StartTopicQuotaWaitTimeForCurrentBlob : TDuration::Zero(); + TopicQuotaWaitTimeForCurrentBlob = ev->Get()->AccountQuotaWaitTime; + PartitionQuotaWaitTimeForCurrentBlob = ev->Get()->PartitionQuotaWaitTime; if (TopicWriteQuotaWaitCounter) { TopicWriteQuotaWaitCounter->IncFor(TopicQuotaWaitTimeForCurrentBlob.MilliSeconds()); } - // Reset quota wait time - StartTopicQuotaWaitTimeForCurrentBlob = TInstant::Zero(); - + if (CurrentStateFunc() == &TThis::StateIdle) HandleWrites(ctx); } -size_t TPartition::GetQuotaRequestSize(const TEvKeyValue::TEvRequest& request) { - if (Config.GetMeteringMode() != NKikimrPQ::TPQTabletConfig::METERING_MODE_RESERVED_CAPACITY) { - return 0; +void TPartition::Handle(NReadQuoterEvents::TEvQuotaCountersUpdated::TPtr& ev, const TActorContext& /*ctx*/) { + if (ev->Get()->ForWriteQuota) { + TotalPartitionWriteSpeed = ev->Get()->TotalPartitionWriteSpeed; + } else { + PartitionCountersLabeled->GetCounters()[METRIC_READ_INFLIGHT_LIMIT_THROTTLED].Set(ev->Get()->AvgInflightLimitThrottledMicroseconds); } +} + +size_t TPartition::GetQuotaRequestSize(const TEvKeyValue::TEvRequest& request) { if (AppData()->PQConfig.GetQuotingConfig().GetTopicWriteQuotaEntityToLimit() == NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE) { return WriteNewSize; @@ -2577,7 +2599,7 @@ bool IsQuotingEnabled(const NKikimrPQ::TPQConfig& pqConfig, bool isLocalDC) { const auto& quotingConfig = pqConfig.GetQuotingConfig(); - return isLocalDC && !pqConfig.GetTopicsAreFirstClassCitizen() && quotingConfig.GetEnableQuoting(); + return isLocalDC && quotingConfig.GetEnableQuoting() && !pqConfig.GetTopicsAreFirstClassCitizen(); } bool TPartition::IsQuotingEnabled() const diff --git a/ydb/core/persqueue/partition.h b/ydb/core/persqueue/partition.h index 75caceba2759..15871de134e0 100644 --- a/ydb/core/persqueue/partition.h +++ b/ydb/core/persqueue/partition.h @@ -113,7 +113,7 @@ class TPartition : public TActorBootstrapped { void CancelAllWritesOnWrite(const TActorContext& ctx, TEvKeyValue::TEvRequest* request, const TString& errorStr, const TWriteMsg& p, TPartitionSourceManager::TModificationBatch& sourceIdBatch, NPersQueue::NErrorCode::EErrorCode errorCode = NPersQueue::NErrorCode::BAD_REQUEST); void ClearOldHead(const ui64 offset, const ui16 partNo, TEvKeyValue::TEvRequest* request); void CreateMirrorerActor(); - void DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const TActorContext& ctx); + void DoRead(TEvPQ::TEvRead::TPtr&& ev, TDuration waitQuotaTime, const TActorContext& ctx); void FailBadClient(const TActorContext& ctx); void FillReadFromTimestamps(const NKikimrPQ::TPQTabletConfig& config, const TActorContext& ctx); void FilterDeadlinedWrites(const TActorContext& ctx); @@ -121,7 +121,7 @@ class TPartition : public TActorBootstrapped { void Handle(NReadQuoterEvents::TEvAccountQuotaCountersUpdated::TPtr& ev, const TActorContext& ctx); void Handle(NReadQuoterEvents::TEvQuotaCountersUpdated::TPtr& ev, const TActorContext& ctx); void Handle(NReadQuoterEvents::TEvQuotaUpdated::TPtr& ev, const TActorContext& ctx); - void Handle(TEvPQ::TEvApproveQuota::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPQ::TEvApproveReadQuota::TPtr& ev, const TActorContext& ctx); void Handle(TEvKeyValue::TEvResponse::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQ::TEvBlobResponse::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQ::TEvChangeOwner::TPtr& ev, const TActorContext& ctx); @@ -144,11 +144,12 @@ class TPartition : public TActorBootstrapped { void Handle(TEvPQ::TEvUpdateWriteTimestamp::TPtr& ev, const TActorContext& ctx); void Handle(TEvPersQueue::TEvHasDataInfo::TPtr& ev, const TActorContext& ctx); void Handle(TEvPersQueue::TEvProposeTransaction::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPQ::TEvGetWriteInfoRequest::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQ::TEvTxCalcPredicate::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQ::TEvTxCommit::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQ::TEvTxRollback::TPtr& ev, const TActorContext& ctx); void Handle(TEvPersQueue::TEvReportPartitionError::TPtr& ev, const TActorContext& ctx); - void Handle(TEvQuota::TEvClearance::TPtr& ev, const TActorContext& ctx); + void Handle(TEvPQ::TEvApproveWriteQuota::TPtr& ev, const TActorContext& ctx); void Handle(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx); void Handle(TEvPQ::TEvSubDomainStatus::TPtr& ev, const TActorContext& ctx); void HandleMonitoring(TEvPQ::TEvMonRequest::TPtr& ev, const TActorContext& ctx); @@ -202,7 +203,8 @@ class TPartition : public TActorBootstrapped { void CheckHeadConsistency() const; void HandleWrites(const TActorContext& ctx); void RequestQuotaForWriteBlobRequest(size_t dataSize, ui64 cookie); - void WriteBlobWithQuota(const TActorContext& ctx, THolder&& request); + void RequestBlobQuota(); + void WritePendingBlob(); void UpdateUserInfoEndOffset(const TInstant& now); void UpdateWriteBufferIsFullState(const TInstant& now); @@ -327,7 +329,7 @@ class TPartition : public TActorBootstrapped { template void EmplaceRequest(T&& body, const TActorContext& ctx) { const auto now = ctx.Now(); - Requests.emplace_back(body, WriteQuota->GetQuotedTime(now), now - TInstant::Zero()); + Requests.emplace_back(body, now - TInstant::Zero()); } void EmplaceResponse(TMessage&& message, const TActorContext& ctx); @@ -349,7 +351,7 @@ class TPartition : public TActorBootstrapped { void Handle(TEvPQ::TEvCheckPartitionStatusRequest::TPtr& ev, const TActorContext& ctx); TString LogPrefix() const; - + public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::PERSQUEUE_PARTITION_ACTOR; @@ -359,7 +361,7 @@ class TPartition : public TActorBootstrapped { const NPersQueue::TTopicConverterPtr& topicConverter, TString dcId, bool isServerless, const NKikimrPQ::TPQTabletConfig& config, const TTabletCountersBase& counters, bool SubDomainOutOfSpace, ui32 numChannels, bool newPartition = false, - TVector distrTxs = {}); + TVector distrTxs = {}, const TActorId& writeQuoterActorId = {}); void Bootstrap(const TActorContext& ctx); @@ -430,6 +432,7 @@ class TPartition : public TActorBootstrapped { HFuncTraced(NReadQuoterEvents::TEvQuotaUpdated, Handle); HFuncTraced(NReadQuoterEvents::TEvAccountQuotaCountersUpdated, Handle); HFuncTraced(NReadQuoterEvents::TEvQuotaCountersUpdated, Handle); + HFuncTraced(TEvPQ::TEvGetWriteInfoRequest, Handle); default: if (!Initializer.Handle(ev)) { ALOG_ERROR(NKikimrServices::PERSQUEUE, "Unexpected " << EventStr("StateInit", ev)); @@ -451,7 +454,7 @@ class TPartition : public TActorBootstrapped { HFuncTraced(TEvPQ::TEvBlobResponse, Handle); HFuncTraced(TEvPQ::TEvWrite, HandleOnIdle); HFuncTraced(TEvPQ::TEvRead, Handle); - HFuncTraced(TEvPQ::TEvApproveQuota, Handle); + HFuncTraced(TEvPQ::TEvApproveReadQuota, Handle); HFuncTraced(TEvPQ::TEvReadTimeout, Handle); HFuncTraced(TEvents::TEvPoisonPill, Handle); HFuncTraced(TEvPQ::TEvMonRequest, HandleMonitoring); @@ -472,13 +475,14 @@ class TPartition : public TActorBootstrapped { HFuncTraced(TEvPQ::TEvUpdateAvailableSize, HandleOnIdle); HFuncTraced(TEvPQ::TEvReserveBytes, Handle); HFuncTraced(TEvPQ::TEvPipeDisconnected, Handle); - HFuncTraced(TEvQuota::TEvClearance, Handle); + HFuncTraced(TEvPQ::TEvApproveWriteQuota, Handle); HFuncTraced(TEvPQ::TEvQuotaDeadlineCheck, Handle); HFuncTraced(TEvPQ::TEvRegisterMessageGroup, HandleOnIdle); HFuncTraced(TEvPQ::TEvDeregisterMessageGroup, HandleOnIdle); HFuncTraced(TEvPQ::TEvSplitMessageGroup, HandleOnIdle); HFuncTraced(TEvPersQueue::TEvProposeTransaction, Handle); HFuncTraced(TEvPQ::TEvTxCalcPredicate, Handle); + HFuncTraced(TEvPQ::TEvGetWriteInfoRequest, Handle); HFuncTraced(TEvPQ::TEvProposePartitionConfig, Handle); HFuncTraced(TEvPQ::TEvTxCommit, Handle); HFuncTraced(TEvPQ::TEvTxRollback, Handle); @@ -507,7 +511,7 @@ class TPartition : public TActorBootstrapped { HFuncTraced(TEvPQ::TEvBlobResponse, Handle); HFuncTraced(TEvPQ::TEvWrite, HandleOnWrite); HFuncTraced(TEvPQ::TEvRead, Handle); - HFuncTraced(TEvPQ::TEvApproveQuota, Handle); + HFuncTraced(TEvPQ::TEvApproveReadQuota, Handle); HFuncTraced(TEvPQ::TEvReadTimeout, Handle); HFuncTraced(TEvents::TEvPoisonPill, Handle); HFuncTraced(TEvPQ::TEvMonRequest, HandleMonitoring); @@ -529,12 +533,13 @@ class TPartition : public TActorBootstrapped { HFuncTraced(TEvPQ::TEvPipeDisconnected, Handle); HFuncTraced(TEvPQ::TEvUpdateAvailableSize, HandleOnWrite); HFuncTraced(TEvPQ::TEvQuotaDeadlineCheck, Handle); - HFuncTraced(TEvQuota::TEvClearance, Handle); + HFuncTraced(TEvPQ::TEvApproveWriteQuota, Handle); HFuncTraced(TEvPQ::TEvRegisterMessageGroup, HandleOnWrite); HFuncTraced(TEvPQ::TEvDeregisterMessageGroup, HandleOnWrite); HFuncTraced(TEvPQ::TEvSplitMessageGroup, HandleOnWrite); HFuncTraced(TEvPersQueue::TEvProposeTransaction, Handle); HFuncTraced(TEvPQ::TEvTxCalcPredicate, Handle); + HFuncTraced(TEvPQ::TEvGetWriteInfoRequest, Handle); HFuncTraced(TEvPQ::TEvProposePartitionConfig, Handle); HFuncTraced(TEvPQ::TEvTxCommit, Handle); HFuncTraced(TEvPQ::TEvTxRollback, Handle); @@ -648,7 +653,7 @@ class TPartition : public TActorBootstrapped { using TUserActionAndTransactionEvent = std::variant, // user actions TSimpleSharedPtr, // immediate transaction - TSimpleSharedPtr>; // distributed transaction or update config + TSimpleSharedPtr>; // distributed transaction or update config std::deque UserActionAndTransactionEvents; size_t ImmediateTxCount = 0; THashMap UserActCount; @@ -709,8 +714,8 @@ class TPartition : public TActorBootstrapped { TSet HasDataDeadlines; ui64 HasDataReqNum; - TMaybe WriteQuota; TActorId ReadQuotaTrackerActor; + TActorId WriteQuotaTrackerActor; THolder PartitionWriteQuotaWaitCounter; TInstant QuotaDeadline = TInstant::Zero(); @@ -746,17 +751,19 @@ class TPartition : public TActorBootstrapped { // Writing blob with topic quota variables ui64 TopicQuotaRequestCookie = 0; + ui64 NextTopicWriteQuotaRequestCookie = 1; + ui64 TopicQuotaConsumedCookie = 0; // Wait topic quota metrics + ui64 TotalPartitionWriteSpeed = 0; THolder TopicWriteQuotaWaitCounter; - TInstant StartTopicQuotaWaitTimeForCurrentBlob; TInstant WriteStartTime; TDuration TopicQuotaWaitTimeForCurrentBlob; + TDuration PartitionQuotaWaitTimeForCurrentBlob; + + //Pending request + THolder PendingWriteRequest; - // Topic quota parameters - TString TopicWriteQuoterPath; - TString TopicWriteQuotaResourcePath; - ui64 NextTopicWriteQuotaRequestCookie = 1; TDeque Errors; @@ -768,6 +775,8 @@ class TPartition : public TActorBootstrapped { TRowVersion LastEmittedHeartbeat; const NKikimrPQ::TPQTabletConfig::TPartition* GetPartitionConfig(const NKikimrPQ::TPQTabletConfig& config); + + bool ClosedInternalPartition = false; }; } // namespace NKikimr::NPQ diff --git a/ydb/core/persqueue/partition_init.cpp b/ydb/core/persqueue/partition_init.cpp index 4513e9987de2..76e431156940 100644 --- a/ydb/core/persqueue/partition_init.cpp +++ b/ydb/core/persqueue/partition_init.cpp @@ -7,13 +7,6 @@ static const ui32 LEVEL0 = 32; static const TString WRITE_QUOTA_ROOT_PATH = "write-quota"; -void CalcTopicWriteQuotaParams(const NKikimrPQ::TPQConfig& pqConfig, - bool isLocalDC, - NPersQueue::TTopicConverterPtr topicConverter, - ui64 tabletId, - const TActorContext& ctx, - TString& topicWriteQuoterPath, - TString& topicWriteQuotaResourcePath); bool DiskIsFull(TEvKeyValue::TEvResponse::TPtr& ev); void RequestInfoRange(const TActorContext& ctx, const TActorId& dst, const TPartitionId& partition, const TString& key); void RequestDataRange(const TActorContext& ctx, const TActorId& dst, const TPartitionId& partition, const TString& key); @@ -668,19 +661,31 @@ void TPartition::Initialize(const TActorContext& ctx) { CreationTime = ctx.Now(); WriteCycleStartTime = ctx.Now(); - WriteQuota.ConstructInPlace(Config.GetPartitionConfig().GetBurstSize(), - Config.GetPartitionConfig().GetWriteSpeedInBytesPerSecond(), - ctx.Now()); + ReadQuotaTrackerActor = Register(new TReadQuoter( ctx, - SelfId(), TopicConverter, Config, Partition, Tablet, + SelfId(), TabletID, Counters )); + if (WriteQuotaTrackerActor == TActorId{} && AppData()->PQConfig.GetQuotingConfig().GetEnableQuoting()) { + WriteQuotaTrackerActor = Register(new TWriteQuoter( + TopicConverter, + Config, + Partition, + Tablet, + SelfId(), + TabletID, + IsLocalDC, + Counters, + ctx + )); + } + TotalPartitionWriteSpeed = Config.GetPartitionConfig().GetWriteSpeedInBytesPerSecond(); WriteTimestamp = ctx.Now(); LastUsedStorageMeterTimestamp = ctx.Now(); WriteTimestampEstimate = ManageWriteTimestampEstimate ? ctx.Now() : TInstant::Zero(); @@ -690,14 +695,6 @@ void TPartition::Initialize(const TActorContext& ctx) { DbPath = Config.GetYdbDatabasePath(); FolderId = Config.GetYcFolderId(); - CalcTopicWriteQuotaParams(AppData()->PQConfig, - IsLocalDC, - TopicConverter, - TabletID, - ctx, - TopicWriteQuoterPath, - TopicWriteQuotaResourcePath); - UsersInfoStorage.ConstructInPlace(DCId, TopicConverter, Partition.InternalPartitionId, @@ -794,7 +791,7 @@ void TPartition::SetupTopicCounters(const TActorContext& ctx) { 5000, 10'000, 30'000, 99'999'999}); SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WriteBigLatency"}, true, "sensor", false); WritesTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WritesTotal"}, true, "sensor", false); - if (IsQuotingEnabled() && !TopicWriteQuotaResourcePath.empty()) { + if (IsQuotingEnabled()) { TopicWriteQuotaWaitCounter = THolder( new NKikimr::NPQ::TPercentileCounter( GetServiceCounters(counters, "pqproxy|topicWriteQuotaWait"), labels, @@ -885,7 +882,7 @@ void TPartition::SetupStreamCounters(const TActorContext& ctx) { 5000, 10'000, 30'000, 99'999'999}); SLIBigLatency = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WriteBigLatency"}, true, "name", false); WritesTotal = NKikimr::NPQ::TMultiCounter(subGroup, aggr, {}, {"WritesTotal"}, true, "name", false); - if (IsQuotingEnabled() && !TopicWriteQuotaResourcePath.empty()) { + if (IsQuotingEnabled()) { subgroups.push_back({"name", "api.grpc.topic.stream_write.topic_throttled_milliseconds"}); TopicWriteQuotaWaitCounter = THolder( new NKikimr::NPQ::TPercentileCounter( @@ -941,43 +938,6 @@ bool ValidateResponse(const TInitializerStep& step, TEvKeyValue::TEvResponse::TP return true; } -void CalcTopicWriteQuotaParams(const NKikimrPQ::TPQConfig& pqConfig, - bool isLocalDC, - NPersQueue::TTopicConverterPtr topicConverter, - ui64 tabletId, - const TActorContext& ctx, - TString& topicWriteQuoterPath, - TString& topicWriteQuotaResourcePath) -{ - if (IsQuotingEnabled(pqConfig, isLocalDC)) { // Mirrored topics are not quoted in local dc. - const auto& quotingConfig = pqConfig.GetQuotingConfig(); - - Y_ABORT_UNLESS(quotingConfig.GetTopicWriteQuotaEntityToLimit() != NKikimrPQ::TPQConfig::TQuotingConfig::UNSPECIFIED); - - // ToDo[migration] - double check - auto topicPath = topicConverter->GetFederationPath(); - - // ToDo[migration] - separate quoter paths? - auto topicParts = SplitPath(topicPath); // account/folder/topic // account is first element - if (topicParts.size() < 2) { - LOG_WARN_S(ctx, NKikimrServices::PERSQUEUE, - "tablet " << tabletId << " topic '" << topicPath << "' Bad topic name. Disable quoting for topic"); - return; - } - topicParts[0] = WRITE_QUOTA_ROOT_PATH; // write-quota/folder/topic - - topicWriteQuotaResourcePath = JoinPath(topicParts); - topicWriteQuoterPath = TStringBuilder() << quotingConfig.GetQuotersDirectoryPath() << "/" << topicConverter->GetAccount(); - - LOG_DEBUG_S(ctx, NKikimrServices::PERSQUEUE, - "topicWriteQuutaResourcePath '" << topicWriteQuotaResourcePath - << "' topicWriteQuoterPath '" << topicWriteQuoterPath - << "' account '" << topicConverter->GetAccount() - << "'" - ); - } -} - bool DiskIsFull(TEvKeyValue::TEvResponse::TPtr& ev) { auto& response = ev->Get()->Record; diff --git a/ydb/core/persqueue/partition_read.cpp b/ydb/core/persqueue/partition_read.cpp index 090946901428..453b1bd019c7 100644 --- a/ydb/core/persqueue/partition_read.cpp +++ b/ydb/core/persqueue/partition_read.cpp @@ -139,9 +139,6 @@ void TPartition::ProcessHasDataRequests(const TActorContext& ctx) { void TPartition::UpdateAvailableSize(const TActorContext& ctx) { FilterDeadlinedWrites(ctx); - - auto now = ctx.Now(); - WriteQuota->Update(now); ScheduleUpdateAvailableSize(ctx); } @@ -180,10 +177,6 @@ void TPartition::Handle(NReadQuoterEvents::TEvAccountQuotaCountersUpdated::TPtr& TabletCounters.Populate(*ev->Get()->AccountQuotaCounters.Get()); } -void TPartition::Handle(NReadQuoterEvents::TEvQuotaCountersUpdated::TPtr& ev, const TActorContext& /*ctx*/) { - PartitionCountersLabeled->GetCounters()[METRIC_READ_INFLIGHT_LIMIT_THROTTLED].Set(ev->Get()->AvgInflightLimitThrottledMicroseconds); -} - void TPartition::InitUserInfoForImportantClients(const TActorContext& ctx) { TSet important; for (const auto& importantUser : Config.GetPartitionConfig().GetImportantClientId()) { @@ -618,9 +611,9 @@ TVector TPartition::GetReadRequestFromHead( } if (lastOffset > 0 && offset >= lastOffset) break; - + if (skip) continue; - + if (blobs[i].IsLastPart()) { bool messageSkippingBehaviour = AppData()->PQConfig.GetTopicsAreFirstClassCitizen() && readTimestampMs > blobs[i].WriteTimestamp.MilliSeconds(); @@ -694,7 +687,7 @@ void TPartition::Handle(TEvPQ::TEvRead::TPtr& ev, const TActorContext& ctx) { Y_ABORT_UNLESS(read->Offset <= EndOffset); auto& userInfo = UsersInfoStorage->GetOrCreate(user, ctx); - + if (!read->SessionId.empty() && !userInfo.NoConsumer) { if (userInfo.Session != read->SessionId) { TabletCounters.Cumulative()[COUNTER_PQ_READ_ERROR_NO_SESSION].Increment(1); @@ -705,20 +698,21 @@ void TPartition::Handle(TEvPQ::TEvRead::TPtr& ev, const TActorContext& ctx) { } } userInfo.ReadsInQuotaQueue++; - Send(ReadQuotaTrackerActor, new TEvPQ::TEvRequestQuota(ev.Release())); + Send(ReadQuotaTrackerActor, + new TEvPQ::TEvRequestQuota(ev->Get()->Cookie, std::move(IEventHandle::Upcast(std::move(ev)))) + ); } -void TPartition::Handle(TEvPQ::TEvApproveQuota::TPtr& ev, const TActorContext& ctx) { - DoRead(ev->Get()->ReadRequest.Release(), ev->Get()->WaitTime, ctx); +void TPartition::Handle(TEvPQ::TEvApproveReadQuota::TPtr& ev, const TActorContext& ctx) { + DoRead(std::move(ev->Get()->ReadRequest), ev->Get()->WaitTime, ctx); } -void TPartition::DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const TActorContext& ctx) { - auto read = ev->Get(); +void TPartition::DoRead(TEvPQ::TEvRead::TPtr&& readEvent, TDuration waitQuotaTime, const TActorContext& ctx) { + auto* read = readEvent->Get(); const TString& user = read->ClientId; auto userInfo = UsersInfoStorage->GetIfExists(user); if(!userInfo) { - ReplyError(ctx, read->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, - TStringBuilder() << "cannot finish read request. Consumer " << read->ClientId << " is gone from partition"); + ReplyError(ctx, read->Cookie, NPersQueue::NErrorCode::BAD_REQUEST, TStringBuilder() << "cannot finish read request. Consumer " << read->ClientId << " is gone from partition"); Send(ReadQuotaTrackerActor, new TEvPQ::TEvConsumerRemoved(user)); OnReadRequestFinished(read->Cookie, 0, user, ctx); return; @@ -771,6 +765,7 @@ void TPartition::DoRead(TEvPQ::TEvRead::TPtr ev, TDuration waitQuotaTime, const TStringBuilder() << "Offset more than EndOffset. Offset=" << offset << ", EndOffset=" << EndOffset); return; } + Y_ABORT_UNLESS(offset < EndOffset); ProcessRead(ctx, std::move(info), cookie, false); } @@ -885,7 +880,7 @@ void TPartition::Handle(TEvPQ::TEvProxyResponse::TPtr& ev, const TActorContext& LOG_INFO_S( ctx, NKikimrServices::PERSQUEUE, - "Reading Timestamp failed for offset " << ReadingForOffset << " ( "<< userInfo->Offset << " ) " + "Reading Timestamp failed for offset " << ReadingForOffset << " ( "<< userInfo->Offset << " ) " << ev->Get()->Response->DebugString() ); if (ev->Get()->Response->GetStatus() == NMsgBusProxy::MSTATUS_OK && diff --git a/ydb/core/persqueue/partition_types.h b/ydb/core/persqueue/partition_types.h index 214c70cbb1c5..55824f53f36e 100644 --- a/ydb/core/persqueue/partition_types.h +++ b/ydb/core/persqueue/partition_types.h @@ -69,14 +69,12 @@ struct TMessage { TSplitMessageGroupMsg > Body; - TDuration QuotedTime; // baseline for request and duration for response TDuration QueueTime; // baseline for request and duration for response TInstant WriteTimeBaseline; template - explicit TMessage(T&& body, TDuration quotedTime, TDuration queueTime, TInstant writeTimeBaseline = TInstant::Zero()) + explicit TMessage(T&& body, TDuration queueTime, TInstant writeTimeBaseline = TInstant::Zero()) : Body(std::forward(body)) - , QuotedTime(quotedTime) , QueueTime(queueTime) , WriteTimeBaseline(writeTimeBaseline) { diff --git a/ydb/core/persqueue/partition_write.cpp b/ydb/core/persqueue/partition_write.cpp index ebcd3df4917f..247f84710a60 100644 --- a/ydb/core/persqueue/partition_write.cpp +++ b/ydb/core/persqueue/partition_write.cpp @@ -211,6 +211,11 @@ void TPartition::ProcessReserveRequests(const TActorContext& ctx) { const bool& lastRequest = ReserveRequests.front()->LastRequest; auto it = Owners.find(owner); + if (ClosedInternalPartition) { + ReplyError(ctx, cookie, NPersQueue::NErrorCode::BAD_REQUEST, "ReserveRequest to closed supportive partition"); + ReserveRequests.pop_front(); + continue; + } if (it == Owners.end() || it->second.OwnerCookie != ownerCookie) { ReplyError(ctx, cookie, NPersQueue::NErrorCode::BAD_REQUEST, "ReserveRequest from dead ownership session"); ReserveRequests.pop_front(); @@ -284,7 +289,6 @@ void TPartition::AnswerCurrentWrites(const TActorContext& ctx) { while (!Responses.empty()) { const auto& response = Responses.front(); - const TDuration quotedTime = response.QuotedTime; const TDuration queueTime = response.QueueTime; const TDuration writeTime = ctx.Now() - response.WriteTimeBaseline; @@ -341,7 +345,7 @@ void TPartition::AnswerCurrentWrites(const TActorContext& ctx) { ReplyWrite( ctx, writeResponse.Cookie, s, seqNo, partNo, totalParts, already ? maxOffset : offset, CurrentTimestamp, already, maxSeqNo, - quotedTime, TopicQuotaWaitTimeForCurrentBlob, queueTime, writeTime + PartitionQuotaWaitTimeForCurrentBlob, TopicQuotaWaitTimeForCurrentBlob, queueTime, writeTime ); LOG_DEBUG_S( ctx, @@ -353,7 +357,7 @@ void TPartition::AnswerCurrentWrites(const TActorContext& ctx) { ", Offset: " << offset << " is " << (already ? "already written" : "stored on disk") ); if (PartitionWriteQuotaWaitCounter) { - PartitionWriteQuotaWaitCounter->IncFor(quotedTime.MilliSeconds()); + PartitionWriteQuotaWaitCounter->IncFor(PartitionQuotaWaitTimeForCurrentBlob.MilliSeconds()); } if (!already && partNo + 1 == totalParts && !writeResponse.Msg.HeartbeatVersion) @@ -409,6 +413,7 @@ void TPartition::AnswerCurrentWrites(const TActorContext& ctx) { Responses.pop_front(); } TopicQuotaWaitTimeForCurrentBlob = TDuration::Zero(); + PartitionQuotaWaitTimeForCurrentBlob = TDuration::Zero(); } void TPartition::SyncMemoryStateWithKVState(const TActorContext& ctx) { @@ -481,13 +486,11 @@ void TPartition::SyncMemoryStateWithKVState(const TActorContext& ctx) { void TPartition::Handle(TEvPQ::TEvHandleWriteResponse::TPtr&, const TActorContext& ctx) { PQ_LOG_T("TPartition::HandleOnWrite TEvHandleWriteResponse."); - HandleWriteResponse(ctx); } void TPartition::HandleWriteResponse(const TActorContext& ctx) { PQ_LOG_T("TPartition::HandleWriteResponse."); - Y_ABORT_UNLESS(CurrentStateFunc() == &TThis::StateWrite); ui64 prevEndOffset = EndOffset; @@ -521,10 +524,20 @@ void TPartition::HandleWriteResponse(const TActorContext& ctx) { //All ok auto now = ctx.Now(); const auto& quotingConfig = AppData()->PQConfig.GetQuotingConfig(); - if (quotingConfig.GetTopicWriteQuotaEntityToLimit() == NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE) { - WriteQuota->Exaust(WriteNewSize, now); - } else { - WriteQuota->Exaust(WriteCycleSize, now); + if (WriteQuotaTrackerActor) { + ui64 size = 0; + + if (quotingConfig.GetTopicWriteQuotaEntityToLimit() == NKikimrPQ::TPQConfig::TQuotingConfig::USER_PAYLOAD_SIZE) { + size = WriteNewSize; + } else { + size = WriteCycleSize; + } + if (TopicQuotaConsumedCookie == 0) { + Send(WriteQuotaTrackerActor, new TEvPQ::TEvConsumed(size)); + } else { + Send(WriteQuotaTrackerActor, new TEvPQ::TEvConsumed(size, TopicQuotaConsumedCookie, {})); + } + TopicQuotaConsumedCookie = 0; } for (auto& avg : AvgWriteBytes) { avg.Update(WriteNewSize, now); @@ -667,6 +680,9 @@ void TPartition::HandleOnWrite(TEvPQ::TEvWrite::TPtr& ev, const TActorContext& c if (offset && needToChangeOffset) ++*offset; } + if (WaitingForPreviousBlobQuota() || WaitingForSubDomainQuota(ctx)) { + SetDeadlinesForWrites(ctx); + } WriteInflightSize += size; // TODO: remove decReservedSize == 0 @@ -932,6 +948,7 @@ TPartition::ProcessResult TPartition::ProcessRequest(TWriteMsg& p, ProcessParame ); sourceId.Update(THeartbeat{*hbVersion, p.Msg.Data}); + return ProcessResult::Continue; } @@ -1202,7 +1219,6 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const break; } } - if (const auto heartbeat = sourceIdBatch.CanEmitHeartbeat()) { if (heartbeat->Version > LastEmittedHeartbeat) { LOG_INFO_S( @@ -1238,6 +1254,7 @@ bool TPartition::AppendHeadWithNewWrites(TEvKeyValue::TEvRequest* request, const } } + UpdateWriteBufferIsFullState(ctx.Now()); if (!NewHead.Batches.empty() && !NewHead.Batches.back().Packed) { @@ -1397,25 +1414,15 @@ void TPartition::Handle(TEvPQ::TEvQuotaDeadlineCheck::TPtr&, const TActorContext FilterDeadlinedWrites(ctx); } -bool TPartition::ProcessWrites(TEvKeyValue::TEvRequest* request, TInstant now, const TActorContext& ctx) { +bool TPartition::ProcessWrites(TEvKeyValue::TEvRequest* request, TInstant, const TActorContext& ctx) { PQ_LOG_T("TPartition::ProcessWrites."); - FilterDeadlinedWrites(ctx); - if (!WriteQuota->CanExaust(now)) { // Waiting for partition quota. - SetDeadlinesForWrites(ctx); - return false; - } - if (WaitingForPreviousBlobQuota() || WaitingForSubDomainQuota(ctx)) { // Waiting for topic quota. SetDeadlinesForWrites(ctx); - - if (StartTopicQuotaWaitTimeForCurrentBlob == TInstant::Zero() && !Requests.empty()) { - StartTopicQuotaWaitTimeForCurrentBlob = now; - } return false; } - + Y_ABORT_UNLESS(!PendingWriteRequest); QuotaDeadline = TInstant::Zero(); if (Requests.empty()) @@ -1453,13 +1460,12 @@ bool TPartition::ProcessWrites(TEvKeyValue::TEvRequest* request, TInstant now, c LOG_DEBUG_S( ctx, NKikimrServices::PERSQUEUE, - "writing blob: topic '" << TopicName() << "' partition " << Partition + "Add new write blob: topic '" << TopicName() << "' partition " << Partition << " compactOffset " << key.GetOffset() << "," << key.GetCount() << " HeadOffset " << Head.Offset << " endOffset " << EndOffset << " curOffset " << NewHead.GetNextOffset() << " " << key.ToString() << " size " << res.second << " WTime " << ctx.Now().MilliSeconds() ); - AddNewWriteBlob(res, request, headCleared, ctx); return true; } @@ -1467,7 +1473,6 @@ bool TPartition::ProcessWrites(TEvKeyValue::TEvRequest* request, TInstant now, c void TPartition::FilterDeadlinedWrites(const TActorContext& ctx) { if (QuotaDeadline == TInstant::Zero() || QuotaDeadline > ctx.Now()) return; - PQ_LOG_T("TPartition::FilterDeadlinedWrites."); std::deque newRequests; @@ -1511,9 +1516,11 @@ void TPartition::HandleWrites(const TActorContext& ctx) { return; } } + if (PendingWriteRequest) { + return; + } PQ_LOG_T("TPartition::HandleWrites. Requests.size()=" << Requests.size()); - Become(&TThis::StateWrite); THolder request(new TEvKeyValue::TEvRequest); @@ -1542,7 +1549,6 @@ void TPartition::HandleWrites(const TActorContext& ctx) { Y_ABORT_UNLESS(!res); } Y_ABORT_UNLESS(Requests.empty() - || !WriteQuota->CanExaust(now) || WaitingForPreviousBlobQuota() || WaitingForSubDomainQuota(ctx)); //in this case all writes must be processed or no quota left AnswerCurrentWrites(ctx); //in case if all writes are already done - no answer will be called on kv write, no kv write at all @@ -1551,26 +1557,20 @@ void TPartition::HandleWrites(const TActorContext& ctx) { } WritesTotal.Inc(); - WriteBlobWithQuota(ctx, std::move(request)); + Y_ABORT_UNLESS(!PendingWriteRequest); + PendingWriteRequest = std::move(request); + RequestBlobQuota(); } void TPartition::RequestQuotaForWriteBlobRequest(size_t dataSize, ui64 cookie) { LOG_DEBUG_S( TActivationContext::AsActorContext(), NKikimrServices::PERSQUEUE, - "Send write quota request." << - " Topic: \"" << TopicName() << "\"." << + "Send write quota request." <<" Topic: \"" << TopicName() << "\"." << " Partition: " << Partition << "." << " Amount: " << dataSize << "." << " Cookie: " << cookie ); - - Send(MakeQuoterServiceID(), - new TEvQuota::TEvRequest( - TEvQuota::EResourceOperator::And, - { TEvQuota::TResourceLeaf(TopicWriteQuoterPath, TopicWriteQuotaResourcePath, dataSize) }, - TDuration::Max()), - 0, - cookie); + Send(WriteQuotaTrackerActor, new TEvPQ::TEvRequestQuota(cookie, nullptr)); } bool TPartition::WaitingForPreviousBlobQuota() const { @@ -1590,29 +1590,37 @@ bool TPartition::WaitingForSubDomainQuota(const TActorContext& ctx, const ui64 w return MeteringDataSize(ctx) + withSize > ReserveSize(); } -void TPartition::WriteBlobWithQuota(const TActorContext& /*ctx*/, THolder&& request) { - PQ_LOG_T("TPartition::WriteBlobWithQuota."); +void TPartition::RequestBlobQuota() { + PQ_LOG_T("TPartition::RequestBlobQuota."); // Request quota and write blob. // Mirrored topics are not quoted in local dc. - const bool skip = !IsQuotingEnabled() || TopicWriteQuotaResourcePath.empty(); - if (size_t quotaRequestSize = skip ? 0 : GetQuotaRequestSize(*request)) { + const bool skip = !WriteQuotaTrackerActor; + if (size_t quotaRequestSize = skip ? 0 : GetQuotaRequestSize(*PendingWriteRequest)) { // Request with data. We should check before attempting to write data whether we have enough quota. Y_ABORT_UNLESS(!WaitingForPreviousBlobQuota()); TopicQuotaRequestCookie = NextTopicWriteQuotaRequestCookie++; RequestQuotaForWriteBlobRequest(quotaRequestSize, TopicQuotaRequestCookie); + } else { + WritePendingBlob(); } - AddMetaKey(request.Get()); +} + +void TPartition::WritePendingBlob() { + Y_ABORT_UNLESS(CurrentStateFunc() == &TThis::StateWrite); + Y_ABORT_UNLESS(PendingWriteRequest); + AddMetaKey(PendingWriteRequest.Get()); WriteStartTime = TActivationContext::Now(); // Write blob #if 1 // PQ -> CacheProxy -> KV - Send(BlobCache, request.Release()); + Send(BlobCache, PendingWriteRequest.Release()); + PendingWriteRequest = nullptr; #else - Send(Tablet, request.Release()); + Send(Tablet, PendingWriteRequest.Release()); #endif } diff --git a/ydb/core/persqueue/read_quoter.cpp b/ydb/core/persqueue/read_quoter.cpp index 1bfba7e68c60..f6c756063230 100644 --- a/ydb/core/persqueue/read_quoter.cpp +++ b/ydb/core/persqueue/read_quoter.cpp @@ -5,164 +5,219 @@ namespace NKikimr { namespace NPQ { -void TReadQuoter::Bootstrap(const TActorContext &ctx) { +void TPartitionQuoterBase::Bootstrap(const TActorContext &ctx) { Become(&TThis::StateWork); ScheduleWakeUp(ctx); - - //UpdateConsumersWithCustomQuota(ctx); // depricated. Delete this after 01.10.2023 } -void TReadQuoter::HandleQuotaRequest(TEvPQ::TEvRequestQuota::TPtr& ev, const TActorContext& ctx) { - auto readRequest = ev->Get()->ReadRequest; - GetOrCreateConsumerQuota(readRequest->Get()->ClientId, ctx); - QuotaRequestedTimes.emplace(readRequest->Cookie, ctx.Now()); - if (RequestsInflight >= AppData(ctx)->PQConfig.GetMaxInflightReadRequestsPerPartition() || !WaitingInflightReadRequests.empty()) { - if (WaitingInflightReadRequests.empty()) +void TPartitionQuoterBase::HandleQuotaRequest(TEvPQ::TEvRequestQuota::TPtr& ev, const TActorContext& ctx) { + QuotaRequestedTimes.emplace(ev->Get()->Cookie, ctx.Now()); + TRequestContext context{ev->Release(), ev->Sender}; + HandleQuotaRequestImpl(context); + if (RequestsInflight >= MaxInflightRequests || !WaitingInflightRequests.empty()) { + if (WaitingInflightRequests.empty()) InflightIsFullStartTime = ctx.Now(); - WaitingInflightReadRequests.push_back(readRequest); + WaitingInflightRequests.push_back(std::move(context)); } else { - StartQuoting(readRequest, ctx); + StartQuoting(std::move(context)); } } -void TReadQuoter::StartQuoting(TEvPQ::TEvRead::TPtr ev, const TActorContext& ctx) { +void TPartitionQuoterBase::StartQuoting(TRequestContext&& context) { RequestsInflight++; - TConsumerReadQuota* consumerQuota = GetOrCreateConsumerQuota(ev->Get()->ClientId, ctx); - if (consumerQuota->AccountQuotaTracker) { - Send(consumerQuota->AccountQuotaTracker->Actor, new NAccountReadQuoterEvents::TEvRequest(ev.Release())); + auto& request = context.Request; + auto& accountQuotaTracker = GetAccountQuotaTracker(request); + if (accountQuotaTracker) { + Send(accountQuotaTracker->Actor, new NAccountQuoterEvents::TEvRequest(request->Cookie, request->Request.Release())); + PendingAccountQuotaRequests[request->Cookie] = std::move(context); } else { - CheckConsumerPerPartitionQuota(ev, ctx); + context.PartitionQuotaWaitStart = ActorContext().Now(); + OnAccountQuotaApproved(std::move(context)); } } +void TPartitionQuoterBase::CheckTotalPartitionQuota(TRequestContext&& context) { + if (!PartitionTotalQuotaTracker) + return ApproveQuota(context); + if (!PartitionTotalQuotaTracker->CanExaust(ActorContext().Now()) || !WaitingTotalPartitionQuotaRequests.empty()) { + WaitingTotalPartitionQuotaRequests.push_back(std::move(context)); + return; + } + ApproveQuota(context); +} + +void TPartitionQuoterBase::HandleAccountQuotaApproved(NAccountQuoterEvents::TEvResponse::TPtr& ev, const TActorContext& ctx) { + auto pendingIter = PendingAccountQuotaRequests.find(ev->Get()->Request->Cookie); + Y_ABORT_UNLESS(!pendingIter.IsEnd()); + pendingIter->second.Request->Request = std::move(ev->Get()->Request->Request); + TRequestContext context{std::move(pendingIter->second.Request), pendingIter->second.PartitionActor, ev->Get()->WaitTime, ctx.Now()}; + context.Request->Request = std::move(ev->Get()->Request->Request); + OnAccountQuotaApproved(std::move(context)); + PendingAccountQuotaRequests.erase(pendingIter); +} -void TReadQuoter::HandleAccountQuotaApproved(NAccountReadQuoterEvents::TEvResponse::TPtr& ev, const TActorContext& ctx) { - CheckConsumerPerPartitionQuota(ev->Get()->ReadRequest, ctx); +void TPartitionQuoterBase::ApproveQuota(TRequestContext& context) { + auto waitTimeIter = QuotaRequestedTimes.find(context.Request->Cookie); + if (waitTimeIter != QuotaRequestedTimes.end()) { + auto waitTime = ActorContext().Now() - waitTimeIter->second; + QuotaRequestedTimes.erase(waitTimeIter); + context.TotalQuotaWaitTime = waitTime; + } + Send(context.PartitionActor, MakeQuotaApprovedEvent(context)); } -void TReadQuoter::CheckConsumerPerPartitionQuota(TEvPQ::TEvRead::TPtr ev, const TActorContext& ctx) { - auto consumerQuota = GetOrCreateConsumerQuota(ev->Get()->ClientId, ctx); - if (!consumerQuota->PartitionPerConsumerQuotaTracker.CanExaust(ctx.Now()) || !consumerQuota->ReadRequests.empty()) { - consumerQuota->ReadRequests.push_back(ev); +void TPartitionQuoterBase::ProcessPartitionTotalQuotaQueue() { + if (!PartitionTotalQuotaTracker.Defined()) return; + while (!WaitingTotalPartitionQuotaRequests.empty() && PartitionTotalQuotaTracker->CanExaust(ActorContext().Now())) { + auto& request = WaitingTotalPartitionQuotaRequests.front(); + ApproveQuota(request); + WaitingTotalPartitionQuotaRequests.pop_front(); } - CheckTotalPartitionQuota(ev, ctx); } -void TReadQuoter::CheckTotalPartitionQuota(TEvPQ::TEvRead::TPtr ev, const TActorContext& ctx) { - if (!PartitionTotalQuotaTracker.CanExaust(ctx.Now()) || !WaitingTotalPartitionQuotaReadRequests.empty()) { - WaitingTotalPartitionQuotaReadRequests.push_back(ev); +void TPartitionQuoterBase::HandleConsumed(TEvPQ::TEvConsumed::TPtr& ev, const TActorContext& ctx) { + if (PartitionTotalQuotaTracker.Defined()) + PartitionTotalQuotaTracker->Exaust(ev->Get()->ConsumedBytes, ctx.Now()); + HandleConsumedImpl(ev); + + if (ev->Get()->IsOverhead) return; + if (RequestsInflight > 0) { + RequestsInflight--; + ProcessInflightQueue(); + } else { + LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, + "Attempt to make the inflight counter below zero. Topic " << TopicConverter->GetClientsideName() << + " partition " << Partition << + " readCookie " << ev->Get()->RequestCookie); } - ApproveQuota(ev, ctx); } -void TReadQuoter::ApproveQuota(TEvPQ::TEvRead::TPtr ev, const TActorContext& ctx) { - auto waitTime = TDuration::Zero(); - auto waitTimeIter = QuotaRequestedTimes.find(ev->Get()->Cookie); - if (waitTimeIter != QuotaRequestedTimes.end()) { - waitTime = ctx.Now() - waitTimeIter->second; - QuotaRequestedTimes.erase(waitTimeIter); +void TPartitionQuoterBase::ProcessInflightQueue() { + auto now = ActorContext().Now(); + while (!WaitingInflightRequests.empty() && RequestsInflight < MaxInflightRequests) { + StartQuoting(std::move(WaitingInflightRequests.front())); + WaitingInflightRequests.pop_front(); + if (WaitingInflightRequests.size() == 0) { + InflightLimitSlidingWindow.Update((now - InflightIsFullStartTime).MicroSeconds(), now); + UpdateCounters(ActorContext()); + } } - Send(PartitionActor, new TEvPQ::TEvApproveQuota(ev, waitTime)); } -void TReadQuoter::HandleWakeUp(TEvents::TEvWakeup::TPtr&, const TActorContext& ctx) { - ProcessPerConsumerQuotaQueue(ctx); - ProcessPartititonTotalQuotaQueue(ctx); +void TPartitionQuoterBase::HandleConfigUpdate(TEvPQ::TEvChangePartitionConfig::TPtr& ev, const TActorContext& ctx) { + PQTabletConfig = ev->Get()->Config; + bool totalQuotaUpdated = false; + if (PartitionTotalQuotaTracker.Defined()) { + totalQuotaUpdated = PartitionTotalQuotaTracker->UpdateConfigIfChanged( + GetTotalPartitionSpeedBurst(PQTabletConfig, ctx), GetTotalPartitionSpeed(PQTabletConfig, ctx) + ); + } + UpdateQuotaConfigImpl(totalQuotaUpdated, ctx); +} + + +TQuotaTracker TPartitionQuoterBase::CreatePartitionTotalQuotaTracker(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext& ctx) const { + return {GetTotalPartitionSpeedBurst(pqTabletConfig, ctx), GetTotalPartitionSpeed(pqTabletConfig, ctx), ctx.Now()}; +} + +void TPartitionQuoterBase::HandleWakeUp(TEvents::TEvWakeup::TPtr&, const TActorContext& ctx) { + HandleWakeUpImpl(); + ProcessPartitionTotalQuotaQueue(); UpdateCounters(ctx); ScheduleWakeUp(ctx); } -void TReadQuoter::ProcessInflightQueue(const TActorContext& ctx) { - auto now = ctx.Now(); - while (!WaitingInflightReadRequests.empty() && RequestsInflight < AppData(ctx)->PQConfig.GetMaxInflightReadRequestsPerPartition()) { - auto readEvent(std::move(WaitingInflightReadRequests.front())); - WaitingInflightReadRequests.pop_front(); - StartQuoting(readEvent, ctx); - if (WaitingInflightReadRequests.size() == 0) { - InflightLimitSlidingWindow.Update((now - InflightIsFullStartTime).MicroSeconds(), now); - UpdateCounters(ctx); - } - } +void TPartitionQuoterBase::ScheduleWakeUp(const TActorContext& ctx) { + ctx.Schedule(WAKE_UP_TIMEOUT, new TEvents::TEvWakeup()); } -void TReadQuoter::ProcessPerConsumerQuotaQueue(const TActorContext& ctx) { - for (auto& [consumerStr, consumer] : ConsumerQuotas) { - while (!consumer.ReadRequests.empty() && consumer.PartitionPerConsumerQuotaTracker.CanExaust(ctx.Now())) { - auto readEvent(std::move(consumer.ReadRequests.front())); - consumer.ReadRequests.pop_front(); - CheckTotalPartitionQuota(readEvent, ctx); - } - } +void TReadQuoter::HandleQuotaRequestImpl(TRequestContext& context) { + auto* readRequest = context.Request->Request->CastAsLocal(); + GetOrCreateConsumerQuota(readRequest->ClientId, ActorContext()); } -void TReadQuoter::ProcessPartititonTotalQuotaQueue(const TActorContext& ctx) { - while (!WaitingTotalPartitionQuotaReadRequests.empty() && PartitionTotalQuotaTracker.CanExaust(ctx.Now())) { - auto readEvent(std::move(WaitingTotalPartitionQuotaReadRequests.front())); - WaitingTotalPartitionQuotaReadRequests.pop_front(); - ApproveQuota(readEvent, ctx); - } +void TReadQuoter::OnAccountQuotaApproved(TRequestContext&& context) { + CheckConsumerPerPartitionQuota(std::move(context)); } -void TReadQuoter::HandleConsumerRemoved(TEvPQ::TEvConsumerRemoved::TPtr& ev, const TActorContext&) { - auto it = ConsumerQuotas.find(ev->Get()->Consumer); - if (it != ConsumerQuotas.end()) { - if (it->second.AccountQuotaTracker) { - Send(it->second.AccountQuotaTracker->Actor, new TEvents::TEvPoisonPill()); - } - ConsumerQuotas.erase(it); +THolder& TReadQuoter::GetAccountQuotaTracker(const THolder& request) { + auto clientId = request->Request->CastAsLocal()->ClientId; + return GetOrCreateConsumerQuota(clientId, ActorContext())->AccountQuotaTracker; +} + +IEventBase* TReadQuoter::MakeQuotaApprovedEvent(TRequestContext& context) { + return new TEvPQ::TEvApproveReadQuota(IEventHandle::Downcast(std::move(context.Request->Request)), context.TotalQuotaWaitTime); +}; + +void TReadQuoter::CheckConsumerPerPartitionQuota(TRequestContext&& context) { + auto consumerQuota = GetOrCreateConsumerQuota( + context.Request->Request->CastAsLocal()->ClientId, + ActorContext() + ); + if (!consumerQuota->PartitionPerConsumerQuotaTracker.CanExaust(ActorContext().Now()) || !consumerQuota->ReadRequests.empty()) { + consumerQuota->ReadRequests.push_back(std::move(context)); + return; } + CheckTotalPartitionQuota(std::move(context)); } -void TReadQuoter::HandleConsumed(TEvPQ::TEvConsumed::TPtr& ev, const TActorContext& ctx) { - PartitionTotalQuotaTracker.Exaust(ev->Get()->ReadBytes, ctx.Now()); +void TReadQuoter::HandleConsumedImpl(TEvPQ::TEvConsumed::TPtr& ev) { auto consumerQuota = GetConsumerQuotaIfExists(ev->Get()->Consumer); if (consumerQuota) { if (consumerQuota->AccountQuotaTracker) { Send( consumerQuota->AccountQuotaTracker->Actor, - new NAccountReadQuoterEvents::TEvConsumed(ev->Get()->ReadBytes, ev->Get()->ReadRequestCookie) + new NAccountQuoterEvents::TEvConsumed(ev->Get()->ConsumedBytes, ev->Get()->RequestCookie) ); } - consumerQuota->PartitionPerConsumerQuotaTracker.Exaust(ev->Get()->ReadBytes, ctx.Now()); - } - if (RequestsInflight > 0) { - RequestsInflight--; - ProcessInflightQueue(ctx); - } else { - LOG_ERROR_S(ctx, NKikimrServices::PERSQUEUE, - "Attempt to make the inflight counter below zero. Topic " << TopicConverter->GetClientsideName() << - " partition " << Partition << - " readCookie " << ev->Get()->ReadRequestCookie); + consumerQuota->PartitionPerConsumerQuotaTracker.Exaust(ev->Get()->ConsumedBytes, ActorContext().Now()); } } -void TReadQuoter::HandleConfigUpdate(TEvPQ::TEvChangePartitionConfig::TPtr& ev, const TActorContext& ctx) { - PQTabletConfig = ev->Get()->Config; - UpdateQuota(ctx); - //UpdateConsumersWithCustomQuota(ctx); // depricated. Delete this after 01.10.2023 -} - -void TReadQuoter::HandleUpdateAccountQuotaCounters(NAccountReadQuoterEvents::TEvCounters::TPtr& ev, const TActorContext&) { - auto consumerQuota = GetConsumerQuotaIfExists(ev->Get()->User); +void TReadQuoter::HandleUpdateAccountQuotaCounters(NAccountQuoterEvents::TEvCounters::TPtr& ev, const TActorContext&) { + auto consumerQuota = GetConsumerQuotaIfExists(ev->Get()->Subject); if (!consumerQuota) return; if (consumerQuota->AccountQuotaTracker) { auto diff = ev->Get()->Counters.MakeDiffForAggr(consumerQuota->AccountQuotaTracker->Baseline); ev->Get()->Counters.RememberCurrentStateAsBaseline(consumerQuota->AccountQuotaTracker->Baseline); - Send(PartitionActor, new NReadQuoterEvents::TEvAccountQuotaCountersUpdated(diff)); + Send(GetParent(), new NReadQuoterEvents::TEvAccountQuotaCountersUpdated(diff)); + } +} + +void TReadQuoter::HandleWakeUpImpl() { + ProcessPerConsumerQuotaQueue(ActorContext()); +} + +void TReadQuoter::ProcessPerConsumerQuotaQueue(const TActorContext& ctx) { + for (auto& [consumerStr, consumer] : ConsumerQuotas) { + while (!consumer.ReadRequests.empty() && consumer.PartitionPerConsumerQuotaTracker.CanExaust(ctx.Now())) { + CheckTotalPartitionQuota(std::move(consumer.ReadRequests.front())); + consumer.ReadRequests.pop_front(); + } + } +} + +void TReadQuoter::HandleConsumerRemoved(TEvPQ::TEvConsumerRemoved::TPtr& ev, const TActorContext&) { + auto it = ConsumerQuotas.find(ev->Get()->Consumer); + if (it != ConsumerQuotas.end()) { + if (it->second.AccountQuotaTracker) { + Send(it->second.AccountQuotaTracker->Actor, new TEvents::TEvPoisonPill()); + } + ConsumerQuotas.erase(it); } } void TReadQuoter::UpdateCounters(const TActorContext& ctx) { auto now = ctx.Now(); - if (!WaitingInflightReadRequests.empty()) { + if (!WaitingInflightRequests.empty()) { InflightLimitSlidingWindow.Update((now - InflightIsFullStartTime).MicroSeconds(), now); InflightIsFullStartTime = now; } else { InflightLimitSlidingWindow.Update(now); } - Send(PartitionActor, new NReadQuoterEvents::TEvQuotaCountersUpdated(InflightLimitSlidingWindow.GetValue() / 60)); + Send(GetParent(), NReadQuoterEvents::TEvQuotaCountersUpdated::ReadCounters(InflightLimitSlidingWindow.GetValue() / 60)); } void TReadQuoter::HandlePoisonPill(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) { @@ -175,63 +230,88 @@ void TReadQuoter::HandlePoisonPill(TEvents::TEvPoisonPill::TPtr&, const TActorCo Die(ctx); } -void TReadQuoter::UpdateQuota(const TActorContext &ctx) { +void TReadQuoter::UpdateQuotaConfigImpl(bool totalQuotaUpdated, const TActorContext& ctx) { TVector> updatedQuotas; for (auto& [consumerStr, consumerQuota] : ConsumerQuotas) { - if (consumerQuota.PartitionPerConsumerQuotaTracker.UpdateConfigIfChanged(GetConsumerReadBurst(ctx), GetConsumerReadSpeed(ctx))) { + if (consumerQuota.PartitionPerConsumerQuotaTracker.UpdateConfigIfChanged( + GetConsumerReadBurst(PQTabletConfig, ctx), GetConsumerReadSpeed(PQTabletConfig, ctx) + )) { updatedQuotas.push_back({consumerStr, consumerQuota.PartitionPerConsumerQuotaTracker.GetTotalSpeed()}); } } - auto totalQuotaUpdated = PartitionTotalQuotaTracker.UpdateConfigIfChanged(GetTotalPartitionReadBurst(ctx), GetTotalPartitionReadSpeed(ctx)); - if (updatedQuotas.size() || totalQuotaUpdated) { - Send(PartitionActor, new NReadQuoterEvents::TEvQuotaUpdated(updatedQuotas, PartitionTotalQuotaTracker.GetTotalSpeed())); - } -} -void TReadQuoter::UpdateConsumersWithCustomQuota(const TActorContext &ctx) { - TVector> updatedQuotas; - for (const auto& readQuota : PQTabletConfig.GetPartitionConfig().GetReadQuota()) { - auto consumerQuota = GetOrCreateConsumerQuota(readQuota.GetClientId(), ctx); - if (consumerQuota->PartitionPerConsumerQuotaTracker.UpdateConfigIfChanged(readQuota.GetBurstSize(), readQuota.GetSpeedInBytesPerSecond())) { - updatedQuotas.push_back({readQuota.GetClientId(), consumerQuota->PartitionPerConsumerQuotaTracker.GetTotalSpeed()}); - } + ui64 totalSpeed = 0; + if (PartitionTotalQuotaTracker.Defined()) { + totalSpeed = PartitionTotalQuotaTracker->GetTotalSpeed(); } - if (updatedQuotas.size()) { - Send(PartitionActor, new NReadQuoterEvents::TEvQuotaUpdated(updatedQuotas, PartitionTotalQuotaTracker.GetTotalSpeed())); + if (updatedQuotas.size() || totalQuotaUpdated) { + Send(GetParent(), new NReadQuoterEvents::TEvQuotaUpdated(updatedQuotas, totalSpeed)); } } -ui64 TReadQuoter::GetConsumerReadSpeed(const TActorContext& ctx) const { +ui64 TReadQuoter::GetConsumerReadSpeed(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext& ctx) const { return AppData(ctx)->PQConfig.GetQuotingConfig().GetPartitionReadQuotaIsTwiceWriteQuota() ? - PQTabletConfig.GetPartitionConfig().GetWriteSpeedInBytesPerSecond() * 2 + pqTabletConfig.GetPartitionConfig().GetWriteSpeedInBytesPerSecond() * 2 : DEFAULT_READ_SPEED_AND_BURST; } -ui64 TReadQuoter::GetConsumerReadBurst(const TActorContext& ctx) const { +ui64 TReadQuoter::GetConsumerReadBurst(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext& ctx) const { return AppData(ctx)->PQConfig.GetQuotingConfig().GetPartitionReadQuotaIsTwiceWriteQuota() ? - PQTabletConfig.GetPartitionConfig().GetBurstSize() * 2 + pqTabletConfig.GetPartitionConfig().GetBurstSize() * 2 : DEFAULT_READ_SPEED_AND_BURST; } -ui64 TReadQuoter::GetTotalPartitionReadSpeed(const TActorContext& ctx) const { +ui64 TReadQuoter::GetTotalPartitionSpeed(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext& ctx) const { auto consumersPerPartition = AppData(ctx)->PQConfig.GetQuotingConfig().GetMaxParallelConsumersPerPartition(); - return GetConsumerReadSpeed(ctx) * consumersPerPartition; + return GetConsumerReadSpeed(pqTabletConfig, ctx) * consumersPerPartition; } -ui64 TReadQuoter::GetTotalPartitionReadBurst(const TActorContext& ctx) const { +ui64 TReadQuoter::GetTotalPartitionSpeedBurst(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext& ctx) const { auto consumersPerPartition = AppData(ctx)->PQConfig.GetQuotingConfig().GetMaxParallelConsumersPerPartition(); - return GetConsumerReadBurst(ctx) * consumersPerPartition; + return GetConsumerReadBurst(pqTabletConfig, ctx) * consumersPerPartition; +} + +THolder TReadQuoter::CreateAccountQuotaTracker(const TString& user, const TActorContext& ctx) const { + const auto& quotingConfig = AppData()->PQConfig.GetQuotingConfig(); + TActorId actorId; + if (GetTabletActor() && quotingConfig.GetEnableQuoting()) { + if(quotingConfig.GetEnableReadQuoting()) { + actorId = TActivationContext::Register( + new TAccountReadQuoter( + GetTabletActor(), + ctx.SelfID, + GetTabletId(), + TopicConverter, + GetPartition(), + user, + Counters + ), + GetParent() + ); + } + } + if (actorId) { + return MakeHolder(actorId, Counters); + } else { + return nullptr; + } } TConsumerReadQuota* TReadQuoter::GetOrCreateConsumerQuota(const TString& consumerStr, const TActorContext& ctx) { Y_ABORT_UNLESS(!consumerStr.empty()); auto it = ConsumerQuotas.find(consumerStr); if (it == ConsumerQuotas.end()) { - TConsumerReadQuota consumer(CreateAccountQuotaTracker(consumerStr), GetConsumerReadBurst(ctx), GetConsumerReadSpeed(ctx)); - Send(PartitionActor, new NReadQuoterEvents::TEvQuotaUpdated({{consumerStr, consumer.PartitionPerConsumerQuotaTracker.GetTotalSpeed()}}, PartitionTotalQuotaTracker.GetTotalSpeed())); + TConsumerReadQuota consumer( + CreateAccountQuotaTracker(consumerStr, ctx), + GetConsumerReadBurst(PQTabletConfig, ctx), + GetConsumerReadSpeed(PQTabletConfig, ctx) + ); + Send(GetParent(), new NReadQuoterEvents::TEvQuotaUpdated( + {{consumerStr, consumer.PartitionPerConsumerQuotaTracker.GetTotalSpeed()}}, + GetTotalPartitionSpeed(PQTabletConfig, ctx) + )); auto result = ConsumerQuotas.emplace(consumerStr, std::move(consumer)); - Y_ABORT_UNLESS(result.second); return &result.first->second; } return &it->second; @@ -242,33 +322,5 @@ TConsumerReadQuota* TReadQuoter::GetConsumerQuotaIfExists(const TString& consume return it != ConsumerQuotas.end() ? &it->second : nullptr; } -void TReadQuoter::ScheduleWakeUp(const TActorContext& ctx) { - ctx.Schedule(WAKE_UP_TIMEOUT, new TEvents::TEvWakeup()); -} - -THolder TReadQuoter::CreateAccountQuotaTracker(const TString& user) const { - const auto& quotingConfig = AppData()->PQConfig.GetQuotingConfig(); - if (TabletActor && quotingConfig.GetEnableQuoting() && quotingConfig.GetEnableReadQuoting()) { - TActorId actorId = TActivationContext::Register( - new TAccountReadQuoter( - TabletActor, - SelfId(), - TabletId, - TopicConverter, - Partition, - user, - Counters - ), - PartitionActor - ); - return MakeHolder(actorId, Counters); - } - return nullptr; -} - -TQuotaTracker TReadQuoter::CreatePartitionTotalQuotaTracker(const TActorContext& ctx) const { - return {GetTotalPartitionReadBurst(ctx), GetTotalPartitionReadSpeed(ctx), ctx.Now()}; -} - }// NPQ }// NKikimr diff --git a/ydb/core/persqueue/read_quoter.h b/ydb/core/persqueue/read_quoter.h index 3e37f4840e74..5cc05a47e688 100644 --- a/ydb/core/persqueue/read_quoter.h +++ b/ydb/core/persqueue/read_quoter.h @@ -17,13 +17,13 @@ namespace NPQ { namespace NReadQuoterEvents { struct TEvQuotaUpdated : public TEventLocal { - TEvQuotaUpdated(TVector> updatedConsumerQuotas, ui64 updatedTotalPartitionQuota) + TEvQuotaUpdated(TVector> updatedConsumerQuotas, ui64 updatedTotalPartitionReadQuota) : UpdatedConsumerQuotas(std::move(updatedConsumerQuotas)), - UpdatedTotalPartitionQuota(updatedTotalPartitionQuota) + UpdatedTotalPartitionReadQuota(updatedTotalPartitionReadQuota) {} TVector> UpdatedConsumerQuotas; - ui64 UpdatedTotalPartitionQuota; + ui64 UpdatedTotalPartitionReadQuota; }; struct TEvAccountQuotaCountersUpdated : public TEventLocal { @@ -35,17 +35,52 @@ struct TEvAccountQuotaCountersUpdated : public TEventLocal { - TEvQuotaCountersUpdated(ui32 avgInflightLimitThrottledMicroseconds) - : AvgInflightLimitThrottledMicroseconds(avgInflightLimitThrottledMicroseconds) - {} + TEvQuotaCountersUpdated() = default; + + static TEvQuotaCountersUpdated* ReadCounters(ui32 avgInflightLimitThrottledMicroseconds) { + auto* result = new TEvQuotaCountersUpdated{}; + result->AvgInflightLimitThrottledMicroseconds = avgInflightLimitThrottledMicroseconds; + result->ForWriteQuota = false; + return result; + } + static TEvQuotaCountersUpdated* WriteCounters(ui64 totalPartitionWriteSpeed) { + auto result = new TEvQuotaCountersUpdated{}; + result->TotalPartitionWriteSpeed = totalPartitionWriteSpeed; + result->ForWriteQuota = true; + return result; + } + + bool ForWriteQuota; + ui64 TotalPartitionWriteSpeed; ui32 AvgInflightLimitThrottledMicroseconds; + }; }// NReadQuoterEvents -struct TAccountReadQuoterHolder { - TAccountReadQuoterHolder(const TActorId& actor, const TTabletCountersBase& baseline) +struct TRequestContext { + THolder Request; + TDuration AccountQuotaWaitTime; + TInstant PartitionQuotaWaitStart; + TDuration TotalQuotaWaitTime; + TActorId PartitionActor; + + TRequestContext() = default; + TRequestContext(THolder&& request, const TActorId& partitionActor) + : Request(std::move(request)) + , PartitionActor(partitionActor) + {} + TRequestContext(THolder&& request, const TActorId& partitionActor, const TDuration& accountWaitTime, TInstant now) + : Request(std::move(request)) + , AccountQuotaWaitTime(accountWaitTime) + , PartitionQuotaWaitStart(std::move(now)) + , PartitionActor(partitionActor) + {} +}; + +struct TAccountQuoterHolder { + TAccountQuoterHolder(const TActorId& actor, const TTabletCountersBase& baseline) : Actor(actor) { Baseline.Populate(baseline); @@ -57,118 +92,247 @@ struct TAccountReadQuoterHolder { class TConsumerReadQuota { public: - TConsumerReadQuota(THolder accountQuotaTracker, ui64 readQuotaBurst, ui64 readQuotaSpeed): + TConsumerReadQuota(THolder accountQuotaTracker, ui64 readQuotaBurst, ui64 readQuotaSpeed): PartitionPerConsumerQuotaTracker(readQuotaBurst, readQuotaSpeed, TAppData::TimeProvider->Now()), AccountQuotaTracker(std::move(accountQuotaTracker)) { } public: TQuotaTracker PartitionPerConsumerQuotaTracker; - THolder AccountQuotaTracker; - std::deque ReadRequests; + THolder AccountQuotaTracker; + std::deque ReadRequests; }; -class TReadQuoter : public TActorBootstrapped { +class TPartitionQuoterBase : public TActorBootstrapped { const TDuration WAKE_UP_TIMEOUT = TDuration::Seconds(1); -const ui64 DEFAULT_READ_SPEED_AND_BURST = 1'000'000'000; public: - TReadQuoter( - const TActorContext& ctx, - TActorId partitionActor, + TPartitionQuoterBase( const NPersQueue::TTopicConverterPtr& topicConverter, const NKikimrPQ::TPQTabletConfig& config, const TPartitionId& partition, TActorId tabletActor, + const TActorId& parent, + TMaybe&& partitionTotalQuotaTracker, ui64 tabletId, - const TTabletCountersBase& counters - ): - TabletActor(tabletActor), - PartitionActor(partitionActor), - PQTabletConfig(config), - PartitionTotalQuotaTracker(CreatePartitionTotalQuotaTracker(ctx)), - TopicConverter(topicConverter), - Partition(partition), - TabletId(tabletId), - RequestsInflight(0), - InflightLimitSlidingWindow(1000, TDuration::Minutes(1)) + const TTabletCountersBase& counters, + ui64 maxRequestsInflight + ) + : InflightLimitSlidingWindow(1000, TDuration::Minutes(1)) + , RequestsInflight(0) + , PQTabletConfig(config) + , PartitionTotalQuotaTracker(std::move(partitionTotalQuotaTracker)) + , TopicConverter(topicConverter) + , TabletActor(tabletActor) + , Parent(parent) + , Partition(partition) + , TabletId(tabletId) + , MaxInflightRequests(maxRequestsInflight) { Counters.Populate(counters); } +public: + + void Bootstrap(const TActorContext &ctx); + + void HandleQuotaRequest(TEvPQ::TEvRequestQuota::TPtr& ev,const TActorContext& ctx); + void HandleAccountQuotaApproved(NAccountQuoterEvents::TEvResponse::TPtr& ev, const TActorContext& ctx); + void HandleWakeUp(TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx); + void HandleConsumed(TEvPQ::TEvConsumed::TPtr& ev, const TActorContext& ctx); + void HandleConfigUpdate(TEvPQ::TEvChangePartitionConfig::TPtr& ev, const TActorContext& ctx); + virtual void HandlePoisonPill(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx) = 0; + virtual void HandleUpdateAccountQuotaCounters(NAccountQuoterEvents::TEvCounters::TPtr& ev, const TActorContext& ctx) = 0; + +protected: + virtual void HandleQuotaRequestImpl(TRequestContext& context) = 0; + virtual void HandleConsumedImpl(TEvPQ::TEvConsumed::TPtr& ev) = 0; + + virtual THolder& GetAccountQuotaTracker(const THolder& request) = 0; + virtual void OnAccountQuotaApproved(TRequestContext&& context) = 0; + virtual IEventBase* MakeQuotaApprovedEvent(TRequestContext& context) = 0; + virtual void HandleWakeUpImpl() = 0; + + virtual void ProcessEventImpl(TAutoPtr& ev) = 0; + virtual void UpdateQuotaConfigImpl(bool totalQuotaUpdated, const TActorContext& ctx) = 0; + virtual void UpdateCounters(const TActorContext& ctx) = 0; + virtual ui64 GetTotalPartitionSpeed(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext& ctx) const = 0; + virtual ui64 GetTotalPartitionSpeedBurst(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext& ctx) const = 0; + virtual THolder CreateAccountQuotaTracker(const TString& user, const TActorContext& ctx) const = 0; + +protected: + void CheckTotalPartitionQuota(TRequestContext&& context); + void ApproveQuota(TRequestContext& context); + TQuotaTracker CreatePartitionTotalQuotaTracker(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext& ctx) const; + + inline const TActorId& GetParent() const {return Parent;} + inline const TActorId& GetTabletActor() const {return TabletActor;} + inline ui64 GetTabletId() const {return TabletId;} + inline const TPartitionId& GetPartition() const {return Partition;} + private: + void StartQuoting(TRequestContext&& context); + void UpdateQuota(); + void ProcessInflightQueue(); + void ProcessPartitionTotalQuotaQueue(); + + void ScheduleWakeUp(const TActorContext& ctx); + STFUNC(StateWork) { switch (ev->GetTypeRewrite()) { HFunc(TEvPQ::TEvRequestQuota, HandleQuotaRequest); HFunc(TEvents::TEvWakeup, HandleWakeUp); - HFunc(NAccountReadQuoterEvents::TEvResponse, HandleAccountQuotaApproved); + HFunc(NAccountQuoterEvents::TEvResponse, HandleAccountQuotaApproved); HFunc(TEvPQ::TEvConsumed, HandleConsumed); HFunc(TEvPQ::TEvChangePartitionConfig, HandleConfigUpdate); - HFunc(NAccountReadQuoterEvents::TEvCounters, HandleUpdateAccountQuotaCounters); + HFunc(NAccountQuoterEvents::TEvCounters, HandleUpdateAccountQuotaCounters); HFunc(TEvents::TEvPoisonPill, HandlePoisonPill); - HFunc(TEvPQ::TEvConsumerRemoved, HandleConsumerRemoved); default: + ProcessEventImpl(ev); break; }; } +protected: + std::deque WaitingInflightRequests; + TMicrosecondsSlidingWindow InflightLimitSlidingWindow; + TInstant InflightIsFullStartTime; + ui32 RequestsInflight; + THashMap QuotaRequestedTimes; + NKikimrPQ::TPQTabletConfig PQTabletConfig; + TMaybe PartitionTotalQuotaTracker; + NPersQueue::TTopicConverterPtr TopicConverter; + TTabletCountersBase Counters; + +private: + TActorId TabletActor; + TActorId Parent; + std::deque WaitingTotalPartitionQuotaRequests; + THashMap PendingAccountQuotaRequests; + const TPartitionId Partition; + ui64 TabletId; + ui64 MaxInflightRequests; + + +}; + + +class TReadQuoter : public TPartitionQuoterBase { + +const static ui64 DEFAULT_READ_SPEED_AND_BURST = 1'000'000'000; + +public: + TReadQuoter( + const TActorContext& ctx, + const NPersQueue::TTopicConverterPtr& topicConverter, + const NKikimrPQ::TPQTabletConfig& config, + const TPartitionId& partition, + TActorId tabletActor, + const TActorId& parent, + ui64 tabletId, + const TTabletCountersBase& counters + ) + : TPartitionQuoterBase( + topicConverter, config, partition, tabletActor, parent, + CreatePartitionTotalQuotaTracker(config, ctx), + tabletId, counters, AppData(ctx)->PQConfig.GetMaxInflightReadRequestsPerPartition() + ) + { + } + public: static constexpr NKikimrServices::TActivity::EType ActorActivityType() { return NKikimrServices::TActivity::PERSQUEUE_READ_QUOTER; } - void Bootstrap(const TActorContext &ctx); - - void HandleQuotaRequest(TEvPQ::TEvRequestQuota::TPtr& ev,const TActorContext& ctx); - void HandleAccountQuotaApproved(NAccountReadQuoterEvents::TEvResponse::TPtr& ev, const TActorContext& ctx); - void HandleWakeUp(TEvents::TEvWakeup::TPtr& ev, const TActorContext& ctx); - void HandleConsumed(TEvPQ::TEvConsumed::TPtr& ev, const TActorContext& ctx); - void HandlePoisonPill(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx); - void HandleConfigUpdate(TEvPQ::TEvChangePartitionConfig::TPtr& ev, const TActorContext& ctx); - void HandleUpdateAccountQuotaCounters(NAccountReadQuoterEvents::TEvCounters::TPtr& ev, const TActorContext& ctx); + void HandlePoisonPill(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx) override; + void HandleUpdateAccountQuotaCounters(NAccountQuoterEvents::TEvCounters::TPtr& ev, const TActorContext& ctx) override; void HandleConsumerRemoved(TEvPQ::TEvConsumerRemoved::TPtr& ev, const TActorContext& ctx); + + void UpdateQuotaConfigImpl(bool totalQuotaUpdated, const TActorContext& ctx) override; + IEventBase* MakeQuotaApprovedEvent(TRequestContext& context) override; +protected: + void HandleQuotaRequestImpl(TRequestContext& context) override; + void HandleConsumedImpl(TEvPQ::TEvConsumed::TPtr& ev) override; + THolder& GetAccountQuotaTracker(const THolder& request) override; + void OnAccountQuotaApproved(TRequestContext&& request) override; + ui64 GetTotalPartitionSpeed(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext& ctx) const override; + ui64 GetTotalPartitionSpeedBurst(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext& ctx) const override; + void UpdateCounters(const TActorContext& ctx) override; + void HandleWakeUpImpl() override; + THolder CreateAccountQuotaTracker(const TString& user, const TActorContext& ctx) const override; + + STFUNC(ProcessEventImpl) override + { + switch (ev->GetTypeRewrite()) { + HFunc(TEvPQ::TEvConsumerRemoved, HandleConsumerRemoved); + default: + break; + }; + } private: TConsumerReadQuota* GetOrCreateConsumerQuota(const TString& consumerStr, const TActorContext& ctx); - THolder CreateAccountQuotaTracker(const TString& user) const; - TQuotaTracker CreatePartitionTotalQuotaTracker(const TActorContext& ctx) const; - void StartQuoting(TEvPQ::TEvRead::TPtr ev, const TActorContext& ctx); - void CheckConsumerPerPartitionQuota(TEvPQ::TEvRead::TPtr, const TActorContext& ctx); - void CheckTotalPartitionQuota(TEvPQ::TEvRead::TPtr ev, const TActorContext& ctx); - void ApproveQuota(TEvPQ::TEvRead::TPtr ev, const TActorContext& ctx); - void ScheduleWakeUp(const TActorContext& ctx); - void UpdateConsumersWithCustomQuota(const TActorContext &ctx); - void UpdateQuota(const TActorContext &ctx); - void ProcessInflightQueue(const TActorContext& ctx); + void CheckConsumerPerPartitionQuota(TRequestContext&& context); + void ApproveQuota(TAutoPtr&& ev, const TActorContext& ctx); void ProcessPerConsumerQuotaQueue(const TActorContext& ctx); - void ProcessPartititonTotalQuotaQueue(const TActorContext& ctx); - void UpdateCounters(const TActorContext& ctx); TConsumerReadQuota* GetConsumerQuotaIfExists(const TString& consumerStr); - ui64 GetConsumerReadSpeed(const TActorContext& ctx) const; - ui64 GetConsumerReadBurst(const TActorContext& ctx) const; - ui64 GetTotalPartitionReadSpeed(const TActorContext& ctx) const; - ui64 GetTotalPartitionReadBurst(const TActorContext& ctx) const; + ui64 GetConsumerReadSpeed(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext& ctx) const; + ui64 GetConsumerReadBurst(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext& ctx) const; private: - TActorId TabletActor; - TActorId PartitionActor; THashMap ConsumerQuotas; - THashMap QuotaRequestedTimes; - std::deque WaitingTotalPartitionQuotaReadRequests; - NKikimrPQ::TPQTabletConfig PQTabletConfig; - TQuotaTracker PartitionTotalQuotaTracker; - NPersQueue::TTopicConverterPtr TopicConverter; - const TPartitionId Partition; - ui64 TabletId; - TTabletCountersBase Counters; - ui32 RequestsInflight; - std::deque WaitingInflightReadRequests; - TMicrosecondsSlidingWindow InflightLimitSlidingWindow; - TInstant InflightIsFullStartTime; }; +class TWriteQuoter : public TPartitionQuoterBase { +public: + TWriteQuoter( + const NPersQueue::TTopicConverterPtr& topicConverter, + const NKikimrPQ::TPQTabletConfig& config, + const TPartitionId& partition, + TActorId tabletActor, + const TActorId& parent, + ui64 tabletId, + bool isLocalDc, + const TTabletCountersBase& counters, + const TActorContext& ctx + ); + +public: + static constexpr NKikimrServices::TActivity::EType ActorActivityType() { + return NKikimrServices::TActivity::PERSQUEUE_WRITE_QUOTER; + } + + void HandlePoisonPill(TEvents::TEvPoisonPill::TPtr& ev, const TActorContext& ctx) override; + void HandleUpdateAccountQuotaCounters(NAccountQuoterEvents::TEvCounters::TPtr& ev, const TActorContext& ctx) override; + + void UpdateQuotaConfigImpl(bool totalQuotaUpdated, const TActorContext& ctx) override; + IEventBase* MakeQuotaApprovedEvent(TRequestContext& context) override; + +protected: + void HandleQuotaRequestImpl(TRequestContext& context) override; + void HandleConsumedImpl(TEvPQ::TEvConsumed::TPtr& ev) override; + THolder& GetAccountQuotaTracker(const THolder& request) override; + void OnAccountQuotaApproved(TRequestContext&& request) override; + ui64 GetTotalPartitionSpeed(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext& ctx) const override; + ui64 GetTotalPartitionSpeedBurst(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext& ctx) const override; + void UpdateCounters(const TActorContext& ctx) override; + void HandleWakeUpImpl() override; + THolder CreateAccountQuotaTracker(const TString& user, const TActorContext& ctx) const override; + + STFUNC(ProcessEventImpl) override + { + Y_UNUSED(ev); + } + +private: + bool IsLocalDC; + bool QuotingEnabled; + bool AccountQuotingEnabled; + THolder AccountQuotaTracker; +}; }// NPQ diff --git a/ydb/core/persqueue/sourceid.h b/ydb/core/persqueue/sourceid.h index 81c70bd4603e..3d86a5e39d8a 100644 --- a/ydb/core/persqueue/sourceid.h +++ b/ydb/core/persqueue/sourceid.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -16,49 +17,6 @@ enum class ESourceIdFormat: ui8 { Proto = 1, }; -struct TSourceIdInfo { - enum class EState { - Unknown, - Registered, - PendingRegistration, - }; - - ui64 SeqNo = 0; - ui64 MinSeqNo = 0; - ui64 Offset = 0; - TInstant WriteTimestamp; - TInstant CreateTimestamp; - bool Explicit = false; - TMaybe KeyRange; - TMaybe LastHeartbeat; - EState State = EState::Registered; - - TSourceIdInfo() = default; - TSourceIdInfo(ui64 seqNo, ui64 offset, TInstant createTs); - TSourceIdInfo(ui64 seqNo, ui64 offset, TInstant createTs, THeartbeat&& heartbeat); - TSourceIdInfo(ui64 seqNo, ui64 offset, TInstant createTs, TMaybe&& keyRange, bool isInSplit = false); - - TSourceIdInfo Updated(ui64 seqNo, ui64 offset, TInstant writeTs) const; - TSourceIdInfo Updated(ui64 seqNo, ui64 offset, TInstant writeTs, THeartbeat&& heartbeat) const; - - static EState ConvertState(NKikimrPQ::TMessageGroupInfo::EState value); - static NKikimrPQ::TMessageGroupInfo::EState ConvertState(EState value); - - // Raw format - static TSourceIdInfo Parse(const TString& data, TInstant now); - void Serialize(TBuffer& data) const; - - // Proto format - static TSourceIdInfo Parse(const NKikimrPQ::TMessageGroupInfo& proto); - void Serialize(NKikimrPQ::TMessageGroupInfo& proto) const; - - bool operator==(const TSourceIdInfo& rhs) const; - void Out(IOutputStream& out) const; - - bool IsExpired(TDuration ttl, TInstant now) const; - -}; // TSourceIdInfo - class THeartbeatProcessor { protected: using TSourceIdsByHeartbeat = TMap>; @@ -74,7 +32,6 @@ class THeartbeatProcessor { }; // THeartbeatProcessor -using TSourceIdMap = THashMap; class THeartbeatEmitter; class TSourceIdStorage: private THeartbeatProcessor { @@ -84,6 +41,11 @@ class TSourceIdStorage: private THeartbeatProcessor { const TSourceIdMap& GetInMemorySourceIds() const { return InMemorySourceIds; } + TSourceIdMap ExtractInMemorySourceIds() { + auto ret = std::move(InMemorySourceIds); + InMemorySourceIds = {}; + return ret; + } template void RegisterSourceId(const TString& sourceId, Args&&... args) { diff --git a/ydb/core/persqueue/sourceid_info.h b/ydb/core/persqueue/sourceid_info.h new file mode 100644 index 000000000000..ba409ded7f00 --- /dev/null +++ b/ydb/core/persqueue/sourceid_info.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include + +namespace NKikimr::NPQ { + +struct TSourceIdInfo { + enum class EState { + Unknown, + Registered, + PendingRegistration, + }; + + ui64 SeqNo = 0; + ui64 MinSeqNo = 0; + ui64 Offset = 0; + TInstant WriteTimestamp; + TInstant CreateTimestamp; + bool Explicit = false; + TMaybe KeyRange; + TMaybe LastHeartbeat; + EState State = EState::Registered; + + TSourceIdInfo() = default; + TSourceIdInfo(ui64 seqNo, ui64 offset, TInstant createTs); + TSourceIdInfo(ui64 seqNo, ui64 offset, TInstant createTs, THeartbeat&& heartbeat); + TSourceIdInfo(ui64 seqNo, ui64 offset, TInstant createTs, TMaybe&& keyRange, bool isInSplit = false); + + TSourceIdInfo Updated(ui64 seqNo, ui64 offset, TInstant writeTs) const; + TSourceIdInfo Updated(ui64 seqNo, ui64 offset, TInstant writeTs, THeartbeat&& heartbeat) const; + + static EState ConvertState(NKikimrPQ::TMessageGroupInfo::EState value); + static NKikimrPQ::TMessageGroupInfo::EState ConvertState(EState value); + + // Raw format + static TSourceIdInfo Parse(const TString& data, TInstant now); + void Serialize(TBuffer& data) const; + + // Proto format + static TSourceIdInfo Parse(const NKikimrPQ::TMessageGroupInfo& proto); + void Serialize(NKikimrPQ::TMessageGroupInfo& proto) const; + + bool operator==(const TSourceIdInfo& rhs) const; + void Out(IOutputStream& out) const; + + bool IsExpired(TDuration ttl, TInstant now) const; + +}; // TSourceIdInfo + +using TSourceIdMap = THashMap; + +} //namespace NKikimr::NPQ diff --git a/ydb/core/persqueue/ut/common/pq_ut_common.cpp b/ydb/core/persqueue/ut/common/pq_ut_common.cpp index 8d13437c5571..1b94c85818e5 100644 --- a/ydb/core/persqueue/ut/common/pq_ut_common.cpp +++ b/ydb/core/persqueue/ut/common/pq_ut_common.cpp @@ -537,6 +537,21 @@ void CmdWrite(TTestActorRuntime* runtime, ui64 tabletId, const TActorId& sender, TAutoPtr handle; TEvPersQueue::TEvResponse *result; + + runtime->SetObserverFunc( + [&](TAutoPtr& ev) { + if (auto* msg = ev->CastAsLocal()) { + Cerr << "Captured kesus quota request event\n"; + runtime->Send(new IEventHandle( + ev->Sender, TActorId{}, + new TEvQuota::TEvClearance(TEvQuota::TEvClearance::EResult::Success), 0, ev->Cookie)); + + return TTestActorRuntimeBase::EEventAction::DROP; + } + return TTestActorRuntimeBase::EEventAction::PROCESS; + } + ); + if (msn != -1) msgSeqNo = msn; TString cookie = ownerCookie; for (i32 retriesLeft = 2; retriesLeft > 0; --retriesLeft) { diff --git a/ydb/core/persqueue/ut/counters_ut.cpp b/ydb/core/persqueue/ut/counters_ut.cpp index 6be6bb67bbb6..d634bbd66842 100644 --- a/ydb/core/persqueue/ut/counters_ut.cpp +++ b/ydb/core/persqueue/ut/counters_ut.cpp @@ -105,6 +105,44 @@ Y_UNIT_TEST(Partition) { } } +Y_UNIT_TEST(PartitionWriteQuota) { + TTestContext tc; + + TFinalizer finalizer(tc); + bool activeZone{false}; + tc.Prepare("", [](TTestActorRuntime&) {}, activeZone, false, true); + tc.Runtime->SetScheduledLimit(100); + tc.Runtime->GetAppData(0).PQConfig.MutableQuotingConfig()->SetEnableQuoting(true); + + PQTabletPrepare({.partitions = 1, .writeSpeed = 50_KB}, {}, tc); + TVector> data; + TString s{50_KB, 'c'}; + data.push_back({1, s}); + for (auto i = 0u; i < 7; i++) { + CmdWrite(0, "sourceid0", data, tc, false); + data[0].first++; + } + + { + auto counters = tc.Runtime->GetAppData(0).Counters; + Y_ABORT_UNLESS(counters); + auto dbGroup = GetServiceCounters(counters, "pqproxy"); + + auto quotaWait = dbGroup->FindSubgroup("subsystem", "partitionWriteQuotaWait") + ->FindSubgroup("Account", "total") + ->FindSubgroup("Producer", "total") + ->FindSubgroup("Topic", "total") + ->FindSubgroup("TopicPath", "total") + ->FindSubgroup("OriginDC", "cluster"); + auto histogram = quotaWait->FindSubgroup("sensor", "PartitionWriteQuotaWaitOriginal"); + TStringStream histogramStr; + histogram->OutputHtml(histogramStr); + Cerr << "**** Total histogram: **** \n " << histogramStr.Str() << "**** **** **** ****" << Endl; + UNIT_ASSERT_VALUES_EQUAL(histogram->FindNamedCounter("Interval", "1000ms")->Val(), 3); + UNIT_ASSERT_VALUES_EQUAL(histogram->FindNamedCounter("Interval", "2500ms")->Val(), 1); + } +} + Y_UNIT_TEST(PartitionFirstClass) { TTestContext tc; TFinalizer finalizer(tc); @@ -131,6 +169,7 @@ Y_UNIT_TEST(PartitionFirstClass) { { auto counters = tc.Runtime->GetAppData(0).Counters; auto dbGroup = GetServiceCounters(counters, "datastreams"); + TStringStream countersStr; dbGroup->OutputHtml(countersStr); const TString referenceCounters = NResource::Find(TStringBuf("counters_datastreams.html")); @@ -252,19 +291,22 @@ Y_UNIT_TEST(PartitionFirstClass) { TFinalizer finalizer(tc); activeZone = false; bool dbRegistered{false}; + bool labeledCountersReceived =false ; tc.Prepare(dispatchName, setup, activeZone, true, true, true); - tc.Runtime->SetScheduledLimit(1000); + tc.Runtime->SetScheduledLimit(10000); + tc.Runtime->SetObserverFunc([&](TAutoPtr& event) { if (event->GetTypeRewrite() == NSysView::TEvSysView::EvRegisterDbCounters) { auto database = event.Get()->Get()->Database; UNIT_ASSERT_VALUES_EQUAL(database, "/Root/PQ"); dbRegistered = true; + } else if (event->GetTypeRewrite() == TEvTabletCounters::EvTabletAddLabeledCounters) { + labeledCountersReceived = true; } return TTestActorRuntime::DefaultObserverFunc(event); }); - - PQTabletPrepare({.deleteTime=3600, .meteringMode = NKikimrPQ::TPQTabletConfig::METERING_MODE_REQUEST_UNITS}, {{"client", true}}, tc); + PQTabletPrepare({.deleteTime=3600, .writeSpeed = 100_KB, .meteringMode = NKikimrPQ::TPQTabletConfig::METERING_MODE_REQUEST_UNITS}, {{"client", true}}, tc); TFakeSchemeShardState::TPtr state{new TFakeSchemeShardState()}; ui64 ssId = 325; BootFakeSchemeShard(*tc.Runtime, ssId, state); @@ -285,7 +327,7 @@ Y_UNIT_TEST(PartitionFirstClass) { options.FinalEvents.emplace_back(TEvTabletCounters::EvTabletAddLabeledCounters); tc.Runtime->DispatchEvents(options); } - UNIT_ASSERT(dbRegistered); + //UNIT_ASSERT(labeledCountersReceived); { NSchemeCache::TDescribeResult::TPtr result = new NSchemeCache::TDescribeResult{}; diff --git a/ydb/core/persqueue/ut/partition_ut.cpp b/ydb/core/persqueue/ut/partition_ut.cpp index 4d58647a2f7a..76498ac86660 100644 --- a/ydb/core/persqueue/ut/partition_ut.cpp +++ b/ydb/core/persqueue/ut/partition_ut.cpp @@ -31,7 +31,7 @@ struct TConfigParams { }; struct TCreatePartitionParams { - ui32 Partition = 1; + TPartitionId Partition = TPartitionId{1}; ui64 Begin = 0; ui64 End = 0; TMaybe PlanStep; @@ -127,7 +127,7 @@ class TPartitionFixture : public NUnitTest::TBaseFixture { void SetUp(NUnitTest::TTestContext&) override; void TearDown(NUnitTest::TTestContext&) override; - void CreatePartitionActor(ui32 partition, + void CreatePartitionActor(const TPartitionId& partition, const TConfigParams& config, bool newPartition, TVector txs); @@ -205,7 +205,9 @@ class TPartitionFixture : public NUnitTest::TBaseFixture { void SendSubDomainStatus(bool subDomainOutOfSpace = false); void SendReserveBytes(const ui64 cookie, const ui32 size, const TString& ownerCookie, const ui64 messageNo, bool lastRequest = false); void SendChangeOwner(const ui64 cookie, const TString& owner, const TActorId& pipeClient, const bool force = true); - void SendWrite(const ui64 cookie, const ui64 messageNo, const TString& ownerCookie, const TMaybe offset, const TString& data, bool ignoreQuotaDeadline = false); + void SendWrite(const ui64 cookie, const ui64 messageNo, const TString& ownerCookie, const TMaybe offset, const TString& data, + bool ignoreQuotaDeadline = false, ui64 seqNo = 0); + void SendGetWriteInfo(ui32 internalPartitionId); TMaybe Ctx; TMaybe Finalizer; @@ -231,7 +233,7 @@ void TPartitionFixture::TearDown(NUnitTest::TTestContext&) { } -void TPartitionFixture::CreatePartitionActor(ui32 id, +void TPartitionFixture::CreatePartitionActor(const TPartitionId& id, const TConfigParams& config, bool newPartition, TVector txs) @@ -263,7 +265,7 @@ void TPartitionFixture::CreatePartitionActor(ui32 id, TopicConverter = factory.MakeTopicConverter(Config); auto actor = new NPQ::TPartition(Ctx->TabletId, - TPartitionId(id), + id, Ctx->Edge, 0, Ctx->Edge, @@ -300,7 +302,7 @@ void TPartitionFixture::CreatePartition(const TCreatePartitionParams& params, SendMetaReadResponse(params.PlanStep, params.TxId); WaitInfoRangeRequest(); - SendInfoRangeResponse(params.Partition, params.Config.Consumers); + SendInfoRangeResponse(params.Partition.InternalPartitionId, params.Config.Consumers); WaitDataRangeRequest(); SendDataRangeResponse(params.Begin, params.End); @@ -486,11 +488,13 @@ void TPartitionFixture::SendReserveBytes(const ui64 cookie, const ui32 size, con Ctx->Runtime->SingleSys()->Send(new IEventHandle(ActorId, Ctx->Edge, event.Release())); } -void TPartitionFixture::SendWrite(const ui64 cookie, const ui64 messageNo, const TString& ownerCookie, const TMaybe offset, const TString& data, bool ignoreQuotaDeadline) -{ +void TPartitionFixture::SendWrite + (const ui64 cookie, const ui64 messageNo, const TString& ownerCookie, const TMaybe offset, const TString& data, + bool ignoreQuotaDeadline, ui64 seqNo +) { TEvPQ::TEvWrite::TMsg msg; msg.SourceId = "SourceId"; - msg.SeqNo = messageNo; + msg.SeqNo = seqNo ? seqNo : messageNo; msg.PartNo = 0; msg.TotalParts = 1; msg.TotalSize = data.size(); @@ -518,6 +522,11 @@ void TPartitionFixture::SendChangeOwner(const ui64 cookie, const TString& owner, Ctx->Runtime->SingleSys()->Send(new IEventHandle(ActorId, Ctx->Edge, event.Release())); } +void TPartitionFixture::SendGetWriteInfo(const ui32 internalPartitionId) { + auto event = MakeHolder(internalPartitionId); + Ctx->Runtime->SingleSys()->Send(new IEventHandle(ActorId, Ctx->Edge, event.Release())); +} + void TPartitionFixture::WaitProxyResponse(const TProxyResponseMatcher& matcher) { auto event = Ctx->Runtime->GrabEdgeEvent(); @@ -900,7 +909,7 @@ Y_UNIT_TEST_F(Batching, TPartitionFixture) Y_UNIT_TEST_F(SetOffset, TPartitionFixture) { - const ui32 partition = 0; + const TPartitionId partition{0}; const ui64 begin = 0; const ui64 end = 10; const TString client = "client"; @@ -952,7 +961,7 @@ Y_UNIT_TEST_F(SetOffset, TPartitionFixture) Y_UNIT_TEST_F(TooManyImmediateTxs, TPartitionFixture) { - const ui32 partition = 0; + const TPartitionId partition{0}; const ui64 begin = 0; const ui64 end = 2'000; const TString client = "client"; @@ -963,7 +972,7 @@ Y_UNIT_TEST_F(TooManyImmediateTxs, TPartitionFixture) CreateSession(client, session); for (ui64 txId = 1; txId <= 1'002; ++txId) { - SendProposeTransactionRequest(partition, + SendProposeTransactionRequest(partition.InternalPartitionId, txId - 1, txId, // range client, "topic-path", @@ -995,7 +1004,7 @@ Y_UNIT_TEST_F(TooManyImmediateTxs, TPartitionFixture) // // while the writing is in progress, another command has arrived // - SendProposeTransactionRequest(partition, + SendProposeTransactionRequest(partition.InternalPartitionId, 1'001, 1'002, // range client, "topic-path", @@ -1011,7 +1020,7 @@ Y_UNIT_TEST_F(TooManyImmediateTxs, TPartitionFixture) Y_UNIT_TEST_F(CommitOffsetRanges, TPartitionFixture) { - const ui32 partition = 0; + const TPartitionId partition{0}; const ui64 begin = 0; const ui64 end = 10; const TString client = "client"; @@ -1024,7 +1033,7 @@ Y_UNIT_TEST_F(CommitOffsetRanges, TPartitionFixture) // CreateSession(client, session); - SendProposeTransactionRequest(partition, + SendProposeTransactionRequest(partition.InternalPartitionId, 0, 2, // 0 --> 2 client, "topic-path", @@ -1032,31 +1041,31 @@ Y_UNIT_TEST_F(CommitOffsetRanges, TPartitionFixture) 1); WaitCmdWrite({.Count=2, .UserInfos={{0, {.Session=session, .Offset=2}}}}); - SendProposeTransactionRequest(partition, + SendProposeTransactionRequest(partition.InternalPartitionId, 2, 0, // begin > end client, "topic-path", true, 2); - SendProposeTransactionRequest(partition, + SendProposeTransactionRequest(partition.InternalPartitionId, 4, 6, // begin > client.end client, "topic-path", true, 3); - SendProposeTransactionRequest(partition, + SendProposeTransactionRequest(partition.InternalPartitionId, 1, 4, // begin < client.end client, "topic-path", true, 4); - SendProposeTransactionRequest(partition, + SendProposeTransactionRequest(partition.InternalPartitionId, 2, 4, // begin == client.end client, "topic-path", true, 5); - SendProposeTransactionRequest(partition, + SendProposeTransactionRequest(partition.InternalPartitionId, 4, 13, // end > partition.end client, "topic-path", @@ -1081,7 +1090,7 @@ Y_UNIT_TEST_F(CommitOffsetRanges, TPartitionFixture) Y_UNIT_TEST_F(CorrectRange_Commit, TPartitionFixture) { - const ui32 partition = 3; + const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; const TString client = "client"; @@ -1106,7 +1115,7 @@ Y_UNIT_TEST_F(CorrectRange_Commit, TPartitionFixture) Y_UNIT_TEST_F(CorrectRange_Multiple_Transactions, TPartitionFixture) { - const ui32 partition = 3; + const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; const TString client = "client"; @@ -1142,7 +1151,7 @@ Y_UNIT_TEST_F(CorrectRange_Multiple_Transactions, TPartitionFixture) Y_UNIT_TEST_F(CorrectRange_Multiple_Consumers, TPartitionFixture) { - const ui32 partition = 3; + const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -1173,7 +1182,7 @@ Y_UNIT_TEST_F(CorrectRange_Multiple_Consumers, TPartitionFixture) Y_UNIT_TEST_F(OldPlanStep, TPartitionFixture) { - const ui32 partition = 3; + const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -1188,7 +1197,7 @@ Y_UNIT_TEST_F(OldPlanStep, TPartitionFixture) Y_UNIT_TEST_F(AfterRestart_1, TPartitionFixture) { - const ui32 partition = 3; + const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; const TString consumer = "client"; @@ -1219,7 +1228,7 @@ Y_UNIT_TEST_F(AfterRestart_1, TPartitionFixture) Y_UNIT_TEST_F(AfterRestart_2, TPartitionFixture) { - const ui32 partition = 3; + const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; const TString consumer = "client"; @@ -1245,7 +1254,7 @@ Y_UNIT_TEST_F(AfterRestart_2, TPartitionFixture) Y_UNIT_TEST_F(IncorrectRange, TPartitionFixture) { - const ui32 partition = 3; + const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; const TString client = "client"; @@ -1281,7 +1290,7 @@ Y_UNIT_TEST_F(IncorrectRange, TPartitionFixture) Y_UNIT_TEST_F(CorrectRange_Rollback, TPartitionFixture) { - const ui32 partition = 3; + const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; const TString client = "client"; @@ -1305,7 +1314,7 @@ Y_UNIT_TEST_F(CorrectRange_Rollback, TPartitionFixture) Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) { - const ui32 partition = 3; + const TPartitionId partition{3}; const ui64 begin = 0; const ui64 end = 10; @@ -1362,7 +1371,7 @@ Y_UNIT_TEST_F(ChangeConfig, TPartitionFixture) Y_UNIT_TEST_F(TabletConfig_Is_Newer_That_PartitionConfig, TPartitionFixture) { CreatePartition({ - .Partition=3, + .Partition=TPartitionId{3}, .Begin=0, .End=10, // // конфиг партиции @@ -1389,7 +1398,7 @@ Y_UNIT_TEST_F(ReserveSubDomainOutOfSpace, TPartitionFixture) Ctx->Runtime->GetAppData().FeatureFlags.SetEnableTopicDiskSubDomainQuota(true); CreatePartition({ - .Partition=1, + .Partition=TPartitionId{1}, .Begin=0, .End=10, // // partition configuration @@ -1432,9 +1441,8 @@ Y_UNIT_TEST_F(WriteSubDomainOutOfSpace, TPartitionFixture) { Ctx->Runtime->GetAppData().FeatureFlags.SetEnableTopicDiskSubDomainQuota(true); Ctx->Runtime->GetAppData().PQConfig.MutableQuotingConfig()->SetQuotaWaitDurationMs(300); - CreatePartition({ - .Partition=1, + .Partition=TPartitionId{1}, .Begin=0, .End=10, // // partition configuration @@ -1482,9 +1490,10 @@ Y_UNIT_TEST_F(WriteSubDomainOutOfSpace_DisableExpiration, TPartitionFixture) Ctx->Runtime->GetAppData().FeatureFlags.SetEnableTopicDiskSubDomainQuota(true); // disable write request expiration while thes wait quota Ctx->Runtime->GetAppData().PQConfig.MutableQuotingConfig()->SetQuotaWaitDurationMs(0); + Ctx->Runtime->SetLogPriority( NKikimrServices::PERSQUEUE, NActors::NLog::PRI_DEBUG); CreatePartition({ - .Partition=1, + .Partition=TPartitionId{1}, .Begin=0, .End=10, // // partition configuration @@ -1507,7 +1516,9 @@ Y_UNIT_TEST_F(WriteSubDomainOutOfSpace_DisableExpiration, TPartitionFixture) auto ownerCookie = ownerEvent->Response->GetPartitionResponse().GetCmdGetOwnershipResult().GetOwnerCookie(); TAutoPtr handle; - std::function truth = [&](const TEvPQ::TEvProxyResponse& e) { return cookie == e.Cookie; }; + std::function truth = [&](const TEvPQ::TEvProxyResponse& e) { + return cookie == e.Cookie; + }; TString data = "data for write"; @@ -1516,6 +1527,10 @@ Y_UNIT_TEST_F(WriteSubDomainOutOfSpace_DisableExpiration, TPartitionFixture) messageNo++; SendDiskStatusResponse(); + { + auto event = Ctx->Runtime->GrabEdgeEventIf(handle, truth, TDuration::Seconds(1)); + UNIT_ASSERT(event != nullptr); + } // Second message will not be processed because the limit is exceeded. SendWrite(++cookie, messageNo, ownerCookie, (messageNo + 1) * 100, data); @@ -1540,7 +1555,7 @@ Y_UNIT_TEST_F(WriteSubDomainOutOfSpace_IgnoreQuotaDeadline, TPartitionFixture) Ctx->Runtime->GetAppData().PQConfig.MutableQuotingConfig()->SetQuotaWaitDurationMs(300); CreatePartition({ - .Partition=1, + .Partition=TPartitionId{1}, .Begin=0, .End=10, // // partition configuration @@ -1570,8 +1585,11 @@ Y_UNIT_TEST_F(WriteSubDomainOutOfSpace_IgnoreQuotaDeadline, TPartitionFixture) // First message will be processed because used storage 0 and limit 0. That is, the limit is not exceeded. SendWrite(++cookie, messageNo, ownerCookie, (messageNo + 1) * 100, data, true); messageNo++; - SendDiskStatusResponse(); + { + auto event = Ctx->Runtime->GrabEdgeEventIf(handle, truth, TDuration::Seconds(1)); + UNIT_ASSERT(event != nullptr); + } // Second message will not be processed because the limit is exceeded. SendWrite(++cookie, messageNo, ownerCookie, (messageNo + 1) * 100, data, true); @@ -1590,6 +1608,106 @@ Y_UNIT_TEST_F(WriteSubDomainOutOfSpace_IgnoreQuotaDeadline, TPartitionFixture) UNIT_ASSERT_EQUAL(NMsgBusProxy::MSTATUS_OK, event->Response->GetStatus()); } -} +Y_UNIT_TEST_F(GetPartitionWriteInfoSuccess, TPartitionFixture) { + CreatePartition({ + .Partition=TPartitionId{2, 10, 100'001}, + .Begin=0, .End=10, + // + // partition configuration + // + .Config={.Version=1, .Consumers={}} + }, + // + // tablet configuration + // + {.Version=2, .Consumers={}} + ); -} + ui64 cookie = 1; + + SendChangeOwner(cookie, "owner1", Ctx->Edge, true); + auto ownerEvent = Ctx->Runtime->GrabEdgeEvent(TDuration::Seconds(1)); + UNIT_ASSERT(ownerEvent != nullptr); + auto ownerCookie = ownerEvent->Response->GetPartitionResponse().GetCmdGetOwnershipResult().GetOwnerCookie(); + + TAutoPtr handle; + std::function truth = [&](const TEvPQ::TEvError& e) { return cookie == e.Cookie; }; + + TString data = "data for write"; + + for (auto i = 0; i < 3; i++) { + SendWrite(++cookie, i, ownerCookie, i + 100, data, false, (i+1)*2); + SendDiskStatusResponse(); + auto event = Ctx->Runtime->GrabEdgeEvent(handle, TDuration::Seconds(1)); + UNIT_ASSERT(event != nullptr); + } + SendGetWriteInfo(100'001); + { + auto event = Ctx->Runtime->GrabEdgeEvent(TDuration::Seconds(1)); + UNIT_ASSERT(event != nullptr); + Cerr << "Got write info resposne. Body keys: " << event->BodyKeys.size() << ", head: " << event->BlobsFromHead.size() << ", src id info: " << event->SrcIdInfo.size() << Endl; + UNIT_ASSERT_VALUES_EQUAL(event->BodyKeys.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(event->BlobsFromHead.size(), 3); + UNIT_ASSERT_VALUES_EQUAL(event->SrcIdInfo.size(), 1); + + UNIT_ASSERT_VALUES_EQUAL(event->SrcIdInfo.begin()->second.MinSeqNo, 2); + UNIT_ASSERT_VALUES_EQUAL(event->SrcIdInfo.begin()->second.SeqNo, 6); + UNIT_ASSERT_VALUES_EQUAL(event->SrcIdInfo.begin()->second.Offset, 102); + + UNIT_ASSERT(event->BodyKeys.begin()->Key.ToString().StartsWith("d0000000001_")); + UNIT_ASSERT(event->BlobsFromHead.begin()->GetBlobSize() > 0); + Cerr << "Body key 1: " << event->BodyKeys.begin()->Key.ToString() << ", size: " << event->BodyKeys.begin()->CumulativeSize << Endl; + Cerr << "Body key last " << event->BodyKeys.back().Key.ToString() << ", size: " << event->BodyKeys.back().CumulativeSize << Endl; + Cerr << "Head blob 1 size: " << event->BlobsFromHead.begin()->GetBlobSize() << Endl; + } + +} // GetPartitionWriteInfoSuccess + +Y_UNIT_TEST_F(GetPartitionWriteInfoError, TPartitionFixture) { + CreatePartition({ + .Partition=TPartitionId{2, 10, 100'001}, + .Begin=0, .End=10, + // + // partition configuration + // + .Config={.Version=1, .Consumers={}} + }, + // + // tablet configuration + // + {.Version=2, .Consumers={}} + ); + + ui64 cookie = 1; + + SendChangeOwner(cookie, "owner1", Ctx->Edge, true); + auto ownerEvent = Ctx->Runtime->GrabEdgeEvent(TDuration::Seconds(1)); + UNIT_ASSERT(ownerEvent != nullptr); + auto ownerCookie = ownerEvent->Response->GetPartitionResponse().GetCmdGetOwnershipResult().GetOwnerCookie(); + + TAutoPtr handle; + std::function truth = [&](const TEvPQ::TEvError& e) { return cookie == e.Cookie; }; + + TString data = "data for write"; + + SendWrite(++cookie, 0, ownerCookie, 100, data, false, 1); + { + SendGetWriteInfo(100'001); + auto event = Ctx->Runtime->GrabEdgeEvent(TDuration::Seconds(1)); + UNIT_ASSERT(event != nullptr); + } + SendDiskStatusResponse(); + { + auto event = Ctx->Runtime->GrabEdgeEvent(handle, TDuration::Seconds(1)); + UNIT_ASSERT(event != nullptr); + } + { + SendGetWriteInfo(100'001); + auto event = Ctx->Runtime->GrabEdgeEvent(TDuration::Seconds(1)); + UNIT_ASSERT(event != nullptr); + } +} // GetPartitionWriteInfoErrors + +} // End of suite + +} // namespace diff --git a/ydb/core/persqueue/ut/pq_ut.cpp b/ydb/core/persqueue/ut/pq_ut.cpp index 7b89ee28ae4b..d07ac2799ad7 100644 --- a/ydb/core/persqueue/ut/pq_ut.cpp +++ b/ydb/core/persqueue/ut/pq_ut.cpp @@ -173,7 +173,7 @@ Y_UNIT_TEST(TestPartitionTotalQuota) { CmdRead(0, 0, Max(), Max(), 1, false, tc, {0}, 0, 0, "user1"); CmdRead(0, 0, Max(), Max(), 1, false, tc, {0}, 0, 0, "user2"); auto diff = (tc.Runtime->GetTimeProvider()->Now() - startTime).Seconds(); - UNIT_ASSERT(diff >= 9); //read quota is twice write quota. So, it's 200kb per seconds and 200kb burst. (2mb - 200kb) / 200kb = 9 seconds needed to get quota + UNIT_ASSERT_C(diff >= 9, TStringBuilder() << "Expected >= 9, actual: " << diff); //read quota is twice write quota. So, it's 200kb per seconds and 200kb burst. (2mb - 200kb) / 200kb = 9 seconds needed to get quota }); } @@ -208,11 +208,41 @@ Y_UNIT_TEST(TestPartitionPerConsumerQuota) { auto startTimeReadWithDifferentConsumers = tc.Runtime->GetTimeProvider()->Now(); CmdRead(0, 0, Max(), Max(), 1, false, tc, {0}, 0, 0, "user2"); CmdRead(0, 0, Max(), Max(), 1, false, tc, {0}, 0, 0, "user3"); + auto diffReadWithDifferentConsumers = (tc.Runtime->GetTimeProvider()->Now() - startTimeReadWithDifferentConsumers).Seconds(); UNIT_ASSERT(diffReadWithDifferentConsumers <= 1); //different consumers. No throttling }); } +Y_UNIT_TEST(TestPartitionWriteQuota) { + TTestContext tc; + RunTestWithReboots(tc.TabletIds, [&]() { + return tc.InitialEventsFilter.Prepare(); + }, [&](const TString& dispatchName, std::function setup, bool& activeZone) { + TFinalizer finalizer(tc); + tc.Prepare(dispatchName, setup, activeZone); + activeZone = false; + tc.Runtime->SetScheduledLimit(1000); + + tc.Runtime->GetAppData(0).PQConfig.MutableQuotingConfig()->SetEnableQuoting(true); + + PQTabletPrepare({.partitions = 1, .writeSpeed = 100_KB}, {{"important_user", true}}, tc); + TVector> data; + TString s{2_MB, 'c'}; + data.push_back({1, s}); + auto startTime = tc.Runtime->GetTimeProvider()->Now(); + CmdWrite(0, "sourceid0", data, tc); + data[0].first++; + CmdWrite(0, "sourceid1", data, tc); + data[0].first++; + CmdWrite(0, "sourceid2", data, tc); + + //check throttling on total partition quota + auto diff = (tc.Runtime->GetTimeProvider()->Now() - startTime).Seconds(); + UNIT_ASSERT_C(diff >= 3, TStringBuilder() << "Actual: " << diff); + }); +} + Y_UNIT_TEST(TestGroupsBalancer) { TTestContext tc; TFinalizer finalizer(tc); diff --git a/ydb/core/persqueue/ut/resources/counters_topics.html b/ydb/core/persqueue/ut/resources/counters_topics.html index 2084d1e21989..4e5528739e00 100644 --- a/ydb/core/persqueue/ut/resources/counters_topics.html +++ b/ydb/core/persqueue/ut/resources/counters_topics.html @@ -24,8 +24,8 @@ name=topic.partition.write.bytes_per_minute_max: 540 name=topic.partition.write.idle_milliseconds_max: 0 name=topic.partition.write.lag_milliseconds_max: 600 - name=topic.partition.write.speed_limit_bytes_per_second: 50000000 - name=topic.partition.write.throttled_microseconds_max: 0 + name=topic.partition.write.speed_limit_bytes_per_second: 102400 + name=topic.partition.write.throttled_microseconds_max: 87 name=topic.producers_count: 3 name=topic.reserve.limit_bytes: 0 name=topic.reserve.used_bytes: 0 diff --git a/ydb/core/persqueue/ut/user_action_processor_ut.cpp b/ydb/core/persqueue/ut/user_action_processor_ut.cpp index 3e892bbeaea0..f30c6c48f6ad 100644 --- a/ydb/core/persqueue/ut/user_action_processor_ut.cpp +++ b/ydb/core/persqueue/ut/user_action_processor_ut.cpp @@ -26,7 +26,7 @@ Y_UNIT_TEST_SUITE(TUserActionProcessorTests) { namespace NHelpers { struct TCreatePartitionParams { - ui32 Partition = 1; + TPartitionId Partition = TPartitionId{1}; ui64 Begin = 0; ui64 End = 0; TMaybe PlanStep; @@ -141,7 +141,7 @@ class TUserActionProcessorFixture : public NUnitTest::TBaseFixture { void SetUp(NUnitTest::TTestContext&) override; void TearDown(NUnitTest::TTestContext&) override; - void CreatePartitionActor(ui32 partition, + void CreatePartitionActor(const TPartitionId& partition, const TVector& consumers, bool newPartition, TVector txs); @@ -238,7 +238,7 @@ void TUserActionProcessorFixture::TearDown(NUnitTest::TTestContext&) { } -void TUserActionProcessorFixture::CreatePartitionActor(ui32 id, +void TUserActionProcessorFixture::CreatePartitionActor(const TPartitionId& id, const TVector& consumers, bool newPartition, TVector txs) @@ -306,7 +306,7 @@ void TUserActionProcessorFixture::CreatePartition(const TCreatePartitionParams& SendMetaReadResponse(params.PlanStep, params.TxId); WaitInfoRangeRequest(); - SendInfoRangeResponse(params.Partition, consumers); + SendInfoRangeResponse(params.Partition.InternalPartitionId, consumers); WaitDataRangeRequest(); SendDataRangeResponse(params.Begin, params.End); diff --git a/ydb/core/persqueue/write_quoter.cpp b/ydb/core/persqueue/write_quoter.cpp new file mode 100644 index 000000000000..f24368e95df5 --- /dev/null +++ b/ydb/core/persqueue/write_quoter.cpp @@ -0,0 +1,111 @@ +#include "partition_util.h" +#include "read_quoter.h" + +namespace NKikimr::NPQ { + +TWriteQuoter::TWriteQuoter( + const NPersQueue::TTopicConverterPtr& topicConverter, + const NKikimrPQ::TPQTabletConfig& config, + const TPartitionId& partition, + TActorId tabletActor, + const TActorId& parent, + ui64 tabletId, + bool isLocalDc, + const TTabletCountersBase& counters, + const TActorContext& ctx +) + : TPartitionQuoterBase( + topicConverter, config, partition, tabletActor, parent, + AppData()->PQConfig.GetQuotingConfig().GetEnableQuoting() ? TMaybe{CreatePartitionTotalQuotaTracker(config, ctx)} + : Nothing(), + tabletId, counters, 1 + ) + , IsLocalDC(isLocalDc) + , QuotingEnabled(AppData()->PQConfig.GetQuotingConfig().GetEnableQuoting()) + , AccountQuotingEnabled(IsQuotingEnabled(AppData()->PQConfig, isLocalDc)) +{ + UpdateQuotaConfigImpl(true, ctx); +} + +void TWriteQuoter::OnAccountQuotaApproved(TRequestContext&& context) { + CheckTotalPartitionQuota(std::move(context)); +} + +void TWriteQuoter::HandleQuotaRequestImpl(TRequestContext& context) { + Y_UNUSED(context); + //ToDo !! - check, do nothing? +} + +void TWriteQuoter::HandleConsumedImpl(TEvPQ::TEvConsumed::TPtr& ev) { + if (AccountQuotaTracker) { + Send( + AccountQuotaTracker->Actor, + new NAccountQuoterEvents::TEvConsumed(ev->Get()->ConsumedBytes, ev->Get()->RequestCookie) + ); + } +} + +void TWriteQuoter::HandleWakeUpImpl() { +} + +void TWriteQuoter::UpdateQuotaConfigImpl(bool, const TActorContext& ctx) { + AccountQuotingEnabled = IsQuotingEnabled(AppData()->PQConfig, IsLocalDC); + if (PartitionTotalQuotaTracker.Defined()) { + ctx.Send(GetParent(), NReadQuoterEvents::TEvQuotaCountersUpdated::WriteCounters(PartitionTotalQuotaTracker->GetTotalSpeed())); + } +} + +THolder TWriteQuoter::CreateAccountQuotaTracker(const TString&, const TActorContext& ctx) const { + TActorId actorId; + if (GetTabletActor() && AccountQuotingEnabled) { + actorId = TActivationContext::Register( + new TAccountWriteQuoter( + GetTabletActor(), + SelfId(), + GetTabletId(), + TopicConverter, + GetPartition(), + Counters, + ctx + ), + GetParent() + ); + } + if (actorId) { + return MakeHolder(actorId, Counters); + } else { + return nullptr; + } +} + +void TWriteQuoter::UpdateCounters(const TActorContext&) { +} + +void TWriteQuoter::HandlePoisonPill(TEvents::TEvPoisonPill::TPtr&, const TActorContext& ctx) { + Die(ctx); +} + +void TWriteQuoter:: HandleUpdateAccountQuotaCounters(NAccountQuoterEvents::TEvCounters::TPtr&, const TActorContext&) { + +} + +ui64 TWriteQuoter::GetTotalPartitionSpeed(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext&) const { + return pqTabletConfig.GetPartitionConfig().GetWriteSpeedInBytesPerSecond(); +} + +ui64 TWriteQuoter::GetTotalPartitionSpeedBurst(const NKikimrPQ::TPQTabletConfig& pqTabletConfig, const TActorContext&) const { + return pqTabletConfig.GetPartitionConfig().GetBurstSize(); +} + +IEventBase* TWriteQuoter::MakeQuotaApprovedEvent(TRequestContext& context) { + return new TEvPQ::TEvApproveWriteQuota(context.Request->Cookie, context.AccountQuotaWaitTime, ActorContext().Now() - context.PartitionQuotaWaitStart); +}; + +THolder& TWriteQuoter::GetAccountQuotaTracker(const THolder&) { + if (!AccountQuotaTracker && AccountQuotingEnabled && QuotingEnabled) + AccountQuotaTracker = CreateAccountQuotaTracker(TString{}, ActorContext()); + return AccountQuotaTracker; +} + +} //namespace + diff --git a/ydb/core/persqueue/ya.make b/ydb/core/persqueue/ya.make index 8000bbe68ae2..4fe481f75722 100644 --- a/ydb/core/persqueue/ya.make +++ b/ydb/core/persqueue/ya.make @@ -1,5 +1,11 @@ LIBRARY() +OWNER( + alexnick + g:kikimr + g:logbroker +) + SRCS( actor_persqueue_client_iface.h blob.cpp @@ -35,11 +41,12 @@ SRCS( user_info.cpp utils.cpp write_meta.cpp + write_quoter.cpp microseconds_sliding_window.cpp dread_cache_service/caching_service.cpp ) -GENERATE_ENUM_SERIALIZATION(sourceid.h) +GENERATE_ENUM_SERIALIZATION(sourceid_info.h) PEERDIR( ydb/library/actors/core diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto index c44197db2baf..8ef627415748 100644 --- a/ydb/core/protos/config.proto +++ b/ydb/core/protos/config.proto @@ -1577,6 +1577,8 @@ message TTracingConfig { // time after which generated span will be discarded and will // not be sent to the collector optional uint32 SpanExportTimeoutSeconds = 5; + // maximum batch export requests being run simultaneously + optional uint64 MaxExportRequestsInflight = 6; } reserved 1 to 5; diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/blob_set.h b/ydb/core/tx/columnshard/blobs_action/abstract/blob_set.h index 73eadee2c911..febf2ca72e83 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/blob_set.h +++ b/ydb/core/tx/columnshard/blobs_action/abstract/blob_set.h @@ -367,7 +367,11 @@ class TBlobsByTablet { if (it == Data.end()) { return false; } else { - return it->second.erase(blobId); + const bool result = it->second.erase(blobId); + if (result && it->second.empty()) { + Data.erase(it); + } + return result; } } @@ -462,6 +466,12 @@ class TBlobsCategories { void AddSharing(const TTabletId tabletId, const TUnifiedBlobId& id) { AFL_VERIFY(Sharing.Add(tabletId, id)); } + void RemoveSharing(const TTabletId tabletId, const TUnifiedBlobId& id) { + Y_UNUSED(Sharing.Remove(tabletId, id)); + } + void RemoveBorrowed(const TTabletId tabletId, const TUnifiedBlobId& id) { + Y_UNUSED(Borrowed.Remove(tabletId, id)); + } TBlobsCategories(const TTabletId selfTabletId) : SelfTabletId(selfTabletId) { diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/gc.cpp b/ydb/core/tx/columnshard/blobs_action/abstract/gc.cpp index 31aa5a45ce8d..68ba52307d30 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/gc.cpp +++ b/ydb/core/tx/columnshard/blobs_action/abstract/gc.cpp @@ -6,6 +6,7 @@ namespace NKikimr::NOlap { void IBlobsGCAction::OnCompleteTxAfterCleaning(NColumnShard::TColumnShard& self, const std::shared_ptr& taskAction) { if (!AbortedFlag) { + NActors::TLogContextGuard logGuard = NActors::TLogContextBuilder::Build()("tablet_id", self.TabletID()); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "OnCompleteTxAfterCleaning")("action_guid", GetActionGuid()); auto storage = self.GetStoragesManager()->GetOperator(GetStorageId()); storage->GetSharedBlobs()->OnTransactionCompleteAfterCleaning(BlobsToRemove); @@ -22,6 +23,7 @@ void IBlobsGCAction::OnCompleteTxAfterCleaning(NColumnShard::TColumnShard& self, void IBlobsGCAction::OnExecuteTxAfterCleaning(NColumnShard::TColumnShard& self, TBlobManagerDb& dbBlobs) { if (!AbortedFlag) { + const NActors::TLogContextGuard logGuard = NActors::TLogContextBuilder::Build()("tablet_id", self.TabletID()); auto storage = self.GetStoragesManager()->GetOperator(GetStorageId()); storage->GetSharedBlobs()->OnTransactionExecuteAfterCleaning(BlobsToRemove, dbBlobs.GetDatabase()); for (auto i = BlobsToRemove.GetIterator(); i.IsValid(); ++i) { diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/gc_actor.h b/ydb/core/tx/columnshard/blobs_action/abstract/gc_actor.h index 0def89dd7926..009dee623caf 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/gc_actor.h +++ b/ydb/core/tx/columnshard/blobs_action/abstract/gc_actor.h @@ -15,13 +15,14 @@ class TSharedBlobsCollectionActor: public TActorBootstrapped { using TBase = TActorBootstrapped; const TString OperatorId; TBlobsByTablet BlobIdsByTablets; + const TTabletId SelfTabletId; virtual void DoOnSharedRemovingFinished() = 0; void OnSharedRemovingFinished() { SharedRemovingFinished = true; DoOnSharedRemovingFinished(); } void Handle(NEvents::TEvDeleteSharedBlobsFinished::TPtr& ev) { - AFL_VERIFY(BlobIdsByTablets.Remove((TTabletId)ev->Cookie)); + AFL_VERIFY(BlobIdsByTablets.Remove((TTabletId)ev->Get()->Record.GetTabletId())); if (BlobIdsByTablets.IsEmpty()) { AFL_VERIFY(!SharedRemovingFinished); OnSharedRemovingFinished(); @@ -37,9 +38,10 @@ class TSharedBlobsCollectionActor: public TActorBootstrapped { protected: bool SharedRemovingFinished = false; public: - TSharedBlobsCollectionActor(const TString& operatorId, const TBlobsByTablet& blobIds) + TSharedBlobsCollectionActor(const TString& operatorId, const TTabletId selfTabletId, const TBlobsByTablet& blobIds) : OperatorId(operatorId) , BlobIdsByTablets(blobIds) + , SelfTabletId(selfTabletId) { } @@ -58,7 +60,7 @@ class TSharedBlobsCollectionActor: public TActorBootstrapped { OnSharedRemovingFinished(); } else { for (auto&& i : BlobIdsByTablets) { - auto ev = std::make_unique(TBase::SelfId(), (ui64)i.first, OperatorId, i.second); + auto ev = std::make_unique(TBase::SelfId(), (ui64)SelfTabletId, OperatorId, i.second); NActors::TActivationContext::AsActorContext().Send(MakePipePeNodeCacheID(false), new TEvPipeCache::TEvForward(ev.release(), (ui64)i.first, true), IEventHandle::FlagTrackDelivery, (ui64)i.first); } diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/storage.h b/ydb/core/tx/columnshard/blobs_action/abstract/storage.h index 375a06b575ba..cbca2937a51e 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/storage.h +++ b/ydb/core/tx/columnshard/blobs_action/abstract/storage.h @@ -67,6 +67,7 @@ class IBlobsStorageOperator { void Stop(); + virtual TTabletsByBlob GetBlobsToDelete() const = 0; virtual std::shared_ptr GetBlobsTracker() const = 0; virtual ~IBlobsStorageOperator() = default; diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.cpp b/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.cpp index 972dfefd1ec6..738f966bc381 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.cpp +++ b/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.cpp @@ -58,4 +58,13 @@ bool IStoragesManager::LoadIdempotency(NTable::TDatabase& database) { return true; } +bool IStoragesManager::HasBlobsToDelete() const { + for (auto&& i : Constructed) { + if (!i.second->GetBlobsToDelete().IsEmpty()) { + return true; + } + } + return false; +} + } diff --git a/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h b/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h index ed76b93ea250..128c3b0b3a77 100644 --- a/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h +++ b/ydb/core/tx/columnshard/blobs_action/abstract/storages_manager.h @@ -29,6 +29,8 @@ class IStoragesManager { bool LoadIdempotency(NTable::TDatabase& database); + bool HasBlobsToDelete() const; + void Stop() { for (auto&& i : Constructed) { i.second->Stop(); diff --git a/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp b/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp index 43276070e586..379ec1361f6d 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp +++ b/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp @@ -356,7 +356,7 @@ void TBlobManager::DeleteBlobOnExecute(const TTabletId tabletId, const TUnifiedB } void TBlobManager::DeleteBlobOnComplete(const TTabletId tabletId, const TUnifiedBlobId& blobId) { - AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("to_delete_on_complete", blobId); + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("to_delete_on_complete", blobId)("tablet_id_delete", (ui64)tabletId); ++CountersUpdate.BlobsDeleted; // Check if the deletion needs to be delayed until the blob is no longer diff --git a/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.h b/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.h index 321fe225f975..f02965919df5 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.h +++ b/ydb/core/tx/columnshard/blobs_action/bs/blob_manager.h @@ -163,6 +163,12 @@ class TBlobManager : public IBlobManager, public TCommonBlobsTracker { public: TBlobManager(TIntrusivePtr tabletInfo, const ui32 gen, const TTabletId selfTabletId); + TTabletsByBlob GetBlobsToDeleteAll() const { + auto result = BlobsToDelete; + result.Add(BlobsToDeleteDelayed); + return result; + } + virtual void OnBlobFree(const TUnifiedBlobId& blobId) override; const NColumnShard::TBlobsManagerCounters& GetCounters() const { diff --git a/ydb/core/tx/columnshard/blobs_action/bs/gc_actor.h b/ydb/core/tx/columnshard/blobs_action/bs/gc_actor.h index 0abda40b1f7e..d37385323c87 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/gc_actor.h +++ b/ydb/core/tx/columnshard/blobs_action/bs/gc_actor.h @@ -20,8 +20,8 @@ class TGarbageCollectionActor: public TSharedBlobsCollectionActor& task, THashMap>&& requests, const NActors::TActorId& tabletActorId) - : TBase(task->GetStorageId(), task->GetBlobsToRemove().GetBorrowed()) + TGarbageCollectionActor(const std::shared_ptr& task, THashMap>&& requests, const NActors::TActorId& tabletActorId, const TTabletId selfTabletId) + : TBase(task->GetStorageId(), selfTabletId, task->GetBlobsToRemove().GetBorrowed()) , TabletActorId(tabletActorId) , Requests(std::move(requests)) , GCTask(task) diff --git a/ydb/core/tx/columnshard/blobs_action/bs/storage.cpp b/ydb/core/tx/columnshard/blobs_action/bs/storage.cpp index 19775f969aab..c770727184ed 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/storage.cpp +++ b/ydb/core/tx/columnshard/blobs_action/bs/storage.cpp @@ -29,7 +29,7 @@ std::shared_ptr TOperator::DoStartGCAction(const std::shared_ptr auto requests = gcTask->BuildRequests(PerGenerationCounter, Manager->GetTabletId(), Manager->GetCurrentGen()); AFL_VERIFY(requests.size()); AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "StartGC")("requests_count", requests.size()); - TActorContext::AsActorContext().Register(new TGarbageCollectionActor(gcTask, std::move(requests), TabletActorId)); + TActorContext::AsActorContext().Register(new TGarbageCollectionActor(gcTask, std::move(requests), TabletActorId, GetSelfTabletId())); return gcTask; } diff --git a/ydb/core/tx/columnshard/blobs_action/bs/storage.h b/ydb/core/tx/columnshard/blobs_action/bs/storage.h index 83def503115b..fc395fae8639 100644 --- a/ydb/core/tx/columnshard/blobs_action/bs/storage.h +++ b/ydb/core/tx/columnshard/blobs_action/bs/storage.h @@ -29,6 +29,11 @@ class TOperator: public IBlobsStorageOperator { public: TOperator(const TString& storageId, const NActors::TActorId& tabletActorId, const TIntrusivePtr& tabletInfo, const ui64 generation, const std::shared_ptr& sharedBlobs); + + virtual TTabletsByBlob GetBlobsToDelete() const override { + return Manager->GetBlobsToDeleteAll(); + } + virtual std::shared_ptr GetBlobsTracker() const override { return Manager; } diff --git a/ydb/core/tx/columnshard/blobs_action/events/delete_blobs.h b/ydb/core/tx/columnshard/blobs_action/events/delete_blobs.h index f53e2a5cf8b1..50e37bf7fe0f 100644 --- a/ydb/core/tx/columnshard/blobs_action/events/delete_blobs.h +++ b/ydb/core/tx/columnshard/blobs_action/events/delete_blobs.h @@ -1,8 +1,9 @@ #pragma once #include #include -#include #include +#include +#include namespace NKikimr::NOlap::NBlobOperations::NEvents { @@ -22,6 +23,10 @@ struct TEvDeleteSharedBlobs: public NActors::TEventPB { TEvDeleteSharedBlobsFinished() = default; + TEvDeleteSharedBlobsFinished(const TTabletId tabletId) + { + Record.SetTabletId((ui64)tabletId); + } }; } \ No newline at end of file diff --git a/ydb/core/tx/columnshard/blobs_action/protos/events.proto b/ydb/core/tx/columnshard/blobs_action/protos/events.proto index e304c8ed78a7..34ae1fc57e8b 100644 --- a/ydb/core/tx/columnshard/blobs_action/protos/events.proto +++ b/ydb/core/tx/columnshard/blobs_action/protos/events.proto @@ -10,4 +10,5 @@ message TEvDeleteSharedBlobs { } message TEvDeleteSharedBlobsFinished { + optional uint64 TabletId = 1; } diff --git a/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.cpp b/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.cpp index 9b87d88c033d..5c80c3a1d4a1 100644 --- a/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.cpp +++ b/ydb/core/tx/columnshard/blobs_action/storages_manager/manager.cpp @@ -1,7 +1,9 @@ #include "manager.h" #include #include +#ifndef KIKIMR_DISABLE_S3_OPS #include +#endif namespace NKikimr::NOlap { diff --git a/ydb/core/tx/columnshard/blobs_action/tier/gc_actor.h b/ydb/core/tx/columnshard/blobs_action/tier/gc_actor.h index 5943dcca859d..78ee58e9152d 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/gc_actor.h +++ b/ydb/core/tx/columnshard/blobs_action/tier/gc_actor.h @@ -22,8 +22,8 @@ class TGarbageCollectionActor: public TSharedBlobsCollectionActor& task, const NActors::TActorId& tabletActorId) - : TBase(task->GetStorageId(), task->GetBlobsToRemove().GetBorrowed()) + TGarbageCollectionActor(const std::shared_ptr& task, const NActors::TActorId& tabletActorId, const TTabletId selfTabletId) + : TBase(task->GetStorageId(), selfTabletId, task->GetBlobsToRemove().GetBorrowed()) , TabletActorId(tabletActorId) , GCTask(task) { diff --git a/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp b/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp index 05f8fc570fb5..a3e5e2b63c71 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp +++ b/ydb/core/tx/columnshard/blobs_action/tier/storage.cpp @@ -20,7 +20,7 @@ std::shared_ptr TOperator::DoStartDeclareRemovingAc } std::shared_ptr TOperator::DoStartWritingAction() { - return std::make_shared(GetStorageId(), GetCurrentOperator(), TabletId, GCInfo); + return std::make_shared(GetStorageId(), GetCurrentOperator(), (ui64)GetSelfTabletId(), GCInfo); } std::shared_ptr TOperator::DoStartReadingAction() { @@ -38,7 +38,7 @@ std::shared_ptr TOperator::DoStartGCAction(const std::shared_ptr categories = GetSharedBlobs()->BuildRemoveCategories(std::move(deleteBlobIds)); } auto gcTask = std::make_shared(GetStorageId(), std::move(draftBlobIds), GetCurrentOperator(), std::move(categories), counters); - TActorContext::AsActorContext().Register(new TGarbageCollectionActor(gcTask, TabletActorId)); + TActorContext::AsActorContext().Register(new TGarbageCollectionActor(gcTask, TabletActorId, GetSelfTabletId())); return gcTask; } @@ -59,7 +59,6 @@ void TOperator::InitNewExternalOperator(const NColumnShard::NTiers::TManager* ti TOperator::TOperator(const TString& storageId, const NColumnShard::TColumnShard& shard, const std::shared_ptr& storageSharedBlobsManager) : TBase(storageId, storageSharedBlobsManager) - , TabletId(shard.TabletID()) , TabletActorId(shard.SelfId()) { InitNewExternalOperator(shard.GetTierManagerPointer(storageId)); diff --git a/ydb/core/tx/columnshard/blobs_action/tier/storage.h b/ydb/core/tx/columnshard/blobs_action/tier/storage.h index 960512e7b56b..d45864f93fdd 100644 --- a/ydb/core/tx/columnshard/blobs_action/tier/storage.h +++ b/ydb/core/tx/columnshard/blobs_action/tier/storage.h @@ -11,7 +11,6 @@ namespace NKikimr::NOlap::NBlobOperations::NTier { class TOperator: public IBlobsStorageOperator { private: using TBase = IBlobsStorageOperator; - const ui64 TabletId; const NActors::TActorId TabletActorId; std::shared_ptr GCInfo = std::make_shared(); @@ -36,6 +35,12 @@ class TOperator: public IBlobsStorageOperator { public: TOperator(const TString& storageId, const NColumnShard::TColumnShard& shard, const std::shared_ptr& storageSharedBlobsManager); + virtual TTabletsByBlob GetBlobsToDelete() const override { + auto result = GCInfo->GetBlobsToDelete(); + result.Add(GCInfo->GetBlobsToDeleteInFuture()); + return result; + } + virtual std::shared_ptr GetBlobsTracker() const override { return GCInfo; } diff --git a/ydb/core/tx/columnshard/blobs_action/transaction/tx_remove_blobs.cpp b/ydb/core/tx/columnshard/blobs_action/transaction/tx_remove_blobs.cpp index 04dac000d242..251d9899bee1 100644 --- a/ydb/core/tx/columnshard/blobs_action/transaction/tx_remove_blobs.cpp +++ b/ydb/core/tx/columnshard/blobs_action/transaction/tx_remove_blobs.cpp @@ -14,7 +14,7 @@ void TTxRemoveSharedBlobs::Complete(const TActorContext& ctx) { NActors::TLogContextGuard logGuard = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", Self->TabletID())("tx_state", "complete"); RemoveAction->OnCompleteTxAfterRemoving(*Self, true); - ctx.Send(InitiatorActorId, new NOlap::NBlobOperations::NEvents::TEvDeleteSharedBlobsFinished()); + ctx.Send(InitiatorActorId, new NOlap::NBlobOperations::NEvents::TEvDeleteSharedBlobsFinished((NOlap::TTabletId)Self->TabletID())); } } diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp index 0119a63bc262..3302f7c8c3f3 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.cpp +++ b/ydb/core/tx/columnshard/columnshard_impl.cpp @@ -195,7 +195,8 @@ ui64 TColumnShard::GetOutdatedStep() const { } ui64 TColumnShard::GetMinReadStep() const { - ui64 delayMillisec = MaxReadStaleness.MilliSeconds(); + const TDuration maxReadStaleness = NYDBTest::TControllers::GetColumnShardController()->GetReadTimeoutClean(TDuration::Minutes(5)); + ui64 delayMillisec = maxReadStaleness.MilliSeconds(); ui64 passedStep = GetOutdatedStep(); ui64 minReadStep = (passedStep > delayMillisec ? passedStep - delayMillisec : 0); return minReadStep; @@ -1013,10 +1014,9 @@ void TColumnShard::Handle(NOlap::NBlobOperations::NEvents::TEvDeleteSharedBlobs: NActors::TLogContextGuard gLogging = NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", TabletID())("event", "TEvDeleteSharedBlobs"); auto removeAction = StoragesManager->GetOperator(ev->Get()->Record.GetStorageId())->StartDeclareRemovingAction("DELETE_SHARED_BLOBS"); for (auto&& i : ev->Get()->Record.GetBlobIds()) { - TLogoBlobID blobId; - TString error; - AFL_VERIFY(TLogoBlobID::Parse(blobId, i, error))("problem", error); - removeAction->DeclareRemove((NOlap::TTabletId)ev->Get()->Record.GetSourceTabletId(), NOlap::TUnifiedBlobId(Max(), blobId)); + auto blobId = NOlap::TUnifiedBlobId::BuildFromString(i, nullptr); + AFL_VERIFY(!!blobId)("problem", blobId.GetErrorMessage()); + removeAction->DeclareRemove((NOlap::TTabletId)ev->Get()->Record.GetSourceTabletId(), *blobId); } Execute(new TTxRemoveSharedBlobs(this, removeAction, NActors::ActorIdFromProto(ev->Get()->Record.GetSourceActorId())), ctx); } diff --git a/ydb/core/tx/columnshard/columnshard_impl.h b/ydb/core/tx/columnshard/columnshard_impl.h index 5e9f4c566ef0..e35e26312245 100644 --- a/ydb/core/tx/columnshard/columnshard_impl.h +++ b/ydb/core/tx/columnshard/columnshard_impl.h @@ -458,7 +458,6 @@ class TColumnShard TIntrusivePtr MediatorTimeCastEntry; bool MediatorTimeCastRegistered = false; TSet MediatorTimeCastWaitingSteps; - TDuration MaxReadStaleness = TDuration::Minutes(5); // TODO: Make configurable? const TDuration PeriodicWakeupActivationPeriod; TDuration FailActivationDelay = TDuration::Seconds(1); const TDuration StatsReportInterval; @@ -566,6 +565,10 @@ class TColumnShard return TablesManager.GetPrimaryIndexAsVerified(); } + bool HasIndex() const { + return !!TablesManager.GetPrimaryIndex(); + } + NOlap::TSnapshot GetLastPlannedSnapshot() const { return NOlap::TSnapshot(LastPlannedStep, LastPlannedTxId); } diff --git a/ydb/core/tx/columnshard/data_sharing/common/session/common.cpp b/ydb/core/tx/columnshard/data_sharing/common/session/common.cpp index 9a27eca9a030..d9fe54bda2f1 100644 --- a/ydb/core/tx/columnshard/data_sharing/common/session/common.cpp +++ b/ydb/core/tx/columnshard/data_sharing/common/session/common.cpp @@ -47,7 +47,7 @@ bool TCommonSession::Start(const NColumnShard::TColumnShard& shard) { IsStartedFlag = false; } if (IsStartedFlag) { - shard.GetDataLocksManager()->RegisterLock(GetSessionId(), portionsLock, true); + shard.GetDataLocksManager()->RegisterLock("sharing_session:" + GetSessionId(), portionsLock, true); } return IsStartedFlag; } @@ -56,7 +56,7 @@ void TCommonSession::Finish(const std::shared_ptr& dataLoc AFL_VERIFY(!IsFinishedFlag); IsFinishedFlag = true; if (IsStartedFlag) { - dataLocksManager->UnregisterLock(GetSessionId()); + dataLocksManager->UnregisterLock("sharing_session:" + GetSessionId()); } } diff --git a/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.cpp b/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.cpp index 2ff5d6862e8e..403ee00848d0 100644 --- a/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.cpp +++ b/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.cpp @@ -97,4 +97,17 @@ void TStorageSharedBlobsManager::OnTransactionExecuteAfterCleaning(const TBlobsC } } +void TStorageSharedBlobsManager::OnTransactionCompleteAfterCleaning(const TBlobsCategories& removeTask) { + for (auto i = removeTask.GetSharing().GetIterator(); i.IsValid(); ++i) { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("action", "remove_share")("tablet_id_share", i.GetTabletId())("blob_id", i.GetBlobId().ToStringNew()); + SharedBlobIds.Remove(i.GetTabletId(), i.GetBlobId()); + } + for (auto i = removeTask.GetBorrowed().GetIterator(); i.IsValid(); ++i) { + AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("action", "remove_own")("tablet_id_own", i.GetTabletId())("blob_id", i.GetBlobId().ToStringNew()); + auto it = BorrowedBlobIds.find(i.GetBlobId()); + AFL_VERIFY(it != BorrowedBlobIds.end()); + BorrowedBlobIds.erase(it); + } +} + } \ No newline at end of file diff --git a/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.h b/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.h index 22ec0dca3f21..6772061d49b6 100644 --- a/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.h +++ b/ydb/core/tx/columnshard/data_sharing/manager/shared_blobs.h @@ -32,8 +32,8 @@ class TStorageSharedBlobsManager { if (doRemove) { auto it = BorrowedBlobIds.find(blobId); if (it != BorrowedBlobIds.end()) { - AFL_VERIFY(it->second == tabletId); - blobs.AddBorrowed(tabletId, blobId); + AFL_VERIFY(it->second != tabletId); + blobs.AddBorrowed(it->second, blobId); } else { blobs.AddDirect(tabletId, blobId); } @@ -48,6 +48,9 @@ class TStorageSharedBlobsManager { } + bool IsTrivialLinks() const { + return BorrowedBlobIds.empty() && SharedBlobIds.IsEmpty(); + } TTabletId GetSelfTabletId() const { return SelfTabletId; } @@ -152,17 +155,7 @@ class TStorageSharedBlobsManager { } void OnTransactionExecuteAfterCleaning(const TBlobsCategories& removeTask, NTable::TDatabase& db); - - void OnTransactionCompleteAfterCleaning(const TBlobsCategories& removeTask) { - for (auto i = removeTask.GetSharing().GetIterator(); i.IsValid(); ++i) { - SharedBlobIds.Remove(i.GetTabletId(), i.GetBlobId()); - } - for (auto i = removeTask.GetBorrowed().GetIterator(); i.IsValid(); ++i) { - auto it = BorrowedBlobIds.find(i.GetBlobId()); - AFL_VERIFY(it != BorrowedBlobIds.end()); - BorrowedBlobIds.erase(it); - } - } + void OnTransactionCompleteAfterCleaning(const TBlobsCategories& removeTask); }; class TSharedBlobsManager { @@ -176,6 +169,15 @@ class TSharedBlobsManager { } + bool IsTrivialLinks() const { + for (auto&& i : Storages) { + if (!i.second->IsTrivialLinks()) { + return false; + } + } + return true; + } + THashMap GetBlobCategories() const { THashMap result; for (auto&& i : Storages) { diff --git a/ydb/core/tx/columnshard/data_sharing/modification/tasks/modification.h b/ydb/core/tx/columnshard/data_sharing/modification/tasks/modification.h index 1a1d653a276e..c931eb2c5d48 100644 --- a/ydb/core/tx/columnshard/data_sharing/modification/tasks/modification.h +++ b/ydb/core/tx/columnshard/data_sharing/modification/tasks/modification.h @@ -340,9 +340,7 @@ class TBlobSharing { { TStorageTabletTask task(storageId, toTabletId); task.AddLink(BlobId, selfTabletId); - if (Shared.empty()) { - task.AddLink(BlobId, toTabletId); - } + task.AddLink(BlobId, toTabletId); auto info = result.emplace(toTabletId, task); if (!info.second) { info.first->second.Merge(task); diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/columns_set.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/columns_set.h index 97fef7f317eb..0bca398204e2 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/columns_set.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/columns_set.h @@ -106,6 +106,15 @@ class TColumnsSet { return true; } + bool Cross(const TColumnsSet& columnsSet) const { + for (auto&& i : columnsSet.ColumnIds) { + if (ColumnIds.contains(i)) { + return true; + } + } + return false; + } + bool IsEqual(const TColumnsSet& columnsSet) const { if (columnsSet.GetColumnIds().size() != ColumnIds.size()) { return false; diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/context.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/context.cpp index 8c42a191d6aa..930e91ad544d 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/context.cpp +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/context.cpp @@ -51,7 +51,7 @@ std::shared_ptr TSpecialReadContext } } else if (exclusiveSource) { TColumnsSet columnsFetch = *EFColumns; - if (needSnapshots || FFColumns->Contains(SpecColumns)) { + if (needSnapshots || FFColumns->Cross(*SpecColumns)) { columnsFetch = columnsFetch + *SpecColumns; } if (partialUsageByPredicate) { @@ -60,7 +60,7 @@ std::shared_ptr TSpecialReadContext AFL_VERIFY(columnsFetch.GetColumnsCount()); current = current->AttachNext(std::make_shared(std::make_shared(columnsFetch), "ef")); - if (needSnapshots || FFColumns->Contains(SpecColumns)) { + if (needSnapshots || FFColumns->Cross(*SpecColumns)) { current = current->AttachNext(std::make_shared(SpecColumns)); current = current->AttachNext(std::make_shared()); columnsFetch = columnsFetch - *SpecColumns; diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/context.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/context.h index 17d52413e39c..ff6f17216642 100644 --- a/ydb/core/tx/columnshard/engines/reader/plain_reader/context.h +++ b/ydb/core/tx/columnshard/engines/reader/plain_reader/context.h @@ -23,9 +23,6 @@ class TSpecialReadContext { NIndexes::TIndexCheckerContainer IndexChecker; TReadMetadata::TConstPtr ReadMetadata; std::shared_ptr EmptyColumns = std::make_shared(); - std::shared_ptr PKFFColumns; - std::shared_ptr EFPKColumns; - std::shared_ptr FFMinusEFColumns; std::shared_ptr BuildColumnsFetchingPlan(const bool needSnapshotsFilter, const bool exclusiveSource, const bool partialUsageByPredicate) const; std::array, 2>, 2>, 2> CacheFetchingScripts; public: diff --git a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h index cd10c61fe14a..3d45347dbefc 100644 --- a/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h +++ b/ydb/core/tx/columnshard/engines/storage/optimizer/lbuckets/optimizer.h @@ -583,6 +583,7 @@ class TPortionsBucket: public TMoveOnly { private: std::shared_ptr MainPortion; const std::shared_ptr Counters; + mutable std::optional LastWeight; TPortionsPool Others; std::optional NextBorder; @@ -700,7 +701,16 @@ class TPortionsBucket: public TMoveOnly { } i64 GetWeight() const { - return Others.GetWeight(MainPortion, !NextBorder); + LastWeight = Others.GetWeight(MainPortion, !NextBorder); + return *LastWeight; + } + + i64 GetLastWeight() const { + if (LastWeight) { + return *LastWeight; + } else { + return GetWeight(); + } } std::shared_ptr BuildOptimizationTask(const TCompactionLimits& limits, std::shared_ptr granule, @@ -839,7 +849,7 @@ class TPortionBuckets { } void RemoveBucketFromRating(const std::shared_ptr& bucket) { - auto it = BucketsByWeight.find(bucket->GetWeight()); + auto it = BucketsByWeight.find(bucket->GetLastWeight()); AFL_VERIFY(it != BucketsByWeight.end()); AFL_VERIFY(it->second.erase(bucket.get())); if (it->second.empty()) { diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h index b84b52ffea1a..663863feddbc 100644 --- a/ydb/core/tx/columnshard/hooks/abstract/abstract.h +++ b/ydb/core/tx/columnshard/hooks/abstract/abstract.h @@ -109,6 +109,9 @@ class ICSController { } virtual void OnIndexSelectProcessed(const std::optional /*result*/) { } + virtual TDuration GetReadTimeoutClean(const TDuration def) { + return def; + } virtual EOptimizerCompactionWeightControl GetCompactionControl() const { return EOptimizerCompactionWeightControl::Force; } diff --git a/ydb/core/tx/columnshard/hooks/testing/controller.cpp b/ydb/core/tx/columnshard/hooks/testing/controller.cpp index 7c69478aee24..1a3ea8fb41d4 100644 --- a/ydb/core/tx/columnshard/hooks/testing/controller.cpp +++ b/ydb/core/tx/columnshard/hooks/testing/controller.cpp @@ -18,6 +18,10 @@ bool TController::DoOnAfterFilterAssembling(const std::shared_ptr g(Mutex); + if (SharingIds.empty()) { + CheckInvariants(); + } return true; } @@ -28,14 +32,20 @@ bool TController::DoOnStartCompaction(std::shared_ptr g(Mutex); +// if (SharingIds.empty()) { +// CheckInvariants(); +// } } void TController::CheckInvariants(const ::NKikimr::NColumnShard::TColumnShard& shard, TCheckContext& context) const { + if (!shard.HasIndex()) { + return; + } const auto& index = shard.GetIndexAs(); std::vector> granules = index.GetTables({}, {}); THashMap> ids; @@ -50,20 +60,22 @@ void TController::CheckInvariants(const ::NKikimr::NColumnShard::TColumnShard& s continue; } for (auto&& b : i.second) { - AFL_VERIFY(!it->second.contains(b)); + auto itB = it->second.find(b); + if (itB != it->second.end()) { + AFL_VERIFY(!itB->second.contains((NOlap::TTabletId)shard.TabletID())); + } } } - THashMap categories = shard.GetStoragesManager()->GetSharedBlobsManager()->GetBlobCategories(); - for (auto&& i : categories) { - auto it = ids.find(i.first); - for (auto cat = i.second.GetIterator(); cat.IsValid(); ++cat) { - AFL_VERIFY(it->second.contains(cat.GetBlobId())); + THashMap shardBlobsCategories = shard.GetStoragesManager()->GetSharedBlobsManager()->GetBlobCategories(); + for (auto&& i : shardBlobsCategories) { + auto manager = shard.GetStoragesManager()->GetOperatorVerified(i.first); + const NOlap::TTabletsByBlob blobs = manager->GetBlobsToDelete(); + for (auto b = blobs.GetIterator(); b.IsValid(); ++b) { + i.second.RemoveSharing(b.GetTabletId(), b.GetBlobId()); + } + for (auto b = blobs.GetIterator(); b.IsValid(); ++b) { + i.second.RemoveBorrowed(b.GetTabletId(), b.GetBlobId()); } - } - THashMap shardBlobsCategories; - for (auto&& i : ids) { - auto storageSharingManager = shard.GetStoragesManager()->GetSharedBlobsManager()->GetStorageManagerVerified(i.first); - shardBlobsCategories.emplace(i.first, storageSharingManager->BuildStoreCategories(i.second)); } context.AddCategories(shard.TabletID(), std::move(shardBlobsCategories)); } @@ -106,4 +118,17 @@ std::vector TController::GetPathIds(const ui64 tabletId) const { return result; } +bool TController::IsTrivialLinks() const { + TGuard g(Mutex); + for (auto&& i : ShardActuals) { + if (!i.second->GetStoragesManager()->GetSharedBlobsManager()->IsTrivialLinks()) { + return false; + } + if (i.second->GetStoragesManager()->HasBlobsToDelete()) { + return false; + } + } + return true; +} + } diff --git a/ydb/core/tx/columnshard/hooks/testing/controller.h b/ydb/core/tx/columnshard/hooks/testing/controller.h index 204018713adf..d8d4a901f433 100644 --- a/ydb/core/tx/columnshard/hooks/testing/controller.h +++ b/ydb/core/tx/columnshard/hooks/testing/controller.h @@ -21,9 +21,10 @@ class TController: public ICSController { YDB_ACCESSOR(std::optional, GuaranteeIndexationStartBytesLimit, 0); YDB_ACCESSOR(std::optional, OptimizerFreshnessCheckDuration, TDuration::Zero()); EOptimizerCompactionWeightControl CompactionControl = EOptimizerCompactionWeightControl::Force; + std::optional ReadTimeoutClean; THashMap ShardActuals; - THashMap> RemovedBlobIds; + THashMap>> RemovedBlobIds; TMutex Mutex; class TBlobInfo { @@ -113,7 +114,6 @@ class TController: public ICSController { void CheckInvariants(const ::NKikimr::NColumnShard::TColumnShard& shard, TCheckContext& context) const; - TCheckContext CheckInvariants() const; THashSet SharingIds; protected: virtual void DoOnTabletInitCompleted(const ::NKikimr::NColumnShard::TColumnShard& shard) override; @@ -138,6 +138,9 @@ class TController: public ICSController { virtual TDuration GetOptimizerFreshnessCheckDuration(const TDuration defaultValue) const override { return OptimizerFreshnessCheckDuration.value_or(defaultValue); } + virtual TDuration GetReadTimeoutClean(const TDuration def) override { + return ReadTimeoutClean.value_or(def); + } virtual EOptimizerCompactionWeightControl GetCompactionControl() const override { return CompactionControl; } @@ -155,6 +158,9 @@ class TController: public ICSController { } public: + bool IsTrivialLinks() const; + TCheckContext CheckInvariants() const; + ui32 GetShardActualsCount() const { TGuard g(Mutex); return ShardActuals.size(); @@ -183,6 +189,9 @@ class TController: public ICSController { void SetCompactionControl(const EOptimizerCompactionWeightControl value) { CompactionControl = value; } + void SetReadTimeoutClean(const TDuration d) { + ReadTimeoutClean = d; + } bool HasPKSortingOnly() const; bool HasCompactions() const { diff --git a/ydb/core/tx/datashard/datashard_ut_write.cpp b/ydb/core/tx/datashard/datashard_ut_write.cpp index a0d7e24ed04a..61b0a59c66fb 100644 --- a/ydb/core/tx/datashard/datashard_ut_write.cpp +++ b/ydb/core/tx/datashard/datashard_ut_write.cpp @@ -217,6 +217,70 @@ Y_UNIT_TEST_SUITE(DataShardWrite) { } } + + Y_UNIT_TEST(WritePreparedManyTables) { + auto [runtime, server, sender] = TestCreateServer(); + + TShardedTableOptions opts; + const TString tableName1 = "table-1"; + const TString tableName2 = "table-2"; + const auto [shards1, tableId1] = CreateShardedTable(server, sender, "/Root", tableName1, opts); + const auto [shards2, tableId2] = CreateShardedTable(server, sender, "/Root", tableName2, opts); + const ui64 tabletId1 = shards1[0]; + const ui64 tabletId2 = shards2[0]; + const ui64 rowCount = 3; + + ui64 txId = 100; + ui64 minStep1, maxStep1; + ui64 minStep2, maxStep2; + + Cerr << "===== Write prepared to table 1" << Endl; + { + const auto writeResult = Write(runtime, sender, tabletId1, tableId1, opts.Columns_, rowCount, txId, NKikimrDataEvents::TEvWrite::MODE_PREPARE); + + minStep1 = writeResult.GetMinStep(); + maxStep1 = writeResult.GetMaxStep(); + } + + Cerr << "===== Write prepared to table 2" << Endl; + { + const auto writeResult = Write(runtime, sender, tabletId2, tableId2, opts.Columns_, rowCount, txId, NKikimrDataEvents::TEvWrite::MODE_PREPARE); + + minStep2 = writeResult.GetMinStep(); + maxStep2 = writeResult.GetMaxStep(); + } + + Cerr << "========= Send propose to coordinator" << Endl; + { + SendProposeToCoordinator(server, {tabletId1, tabletId2}, Max(minStep1, minStep2), Min(maxStep1, maxStep2), txId); + } + + Cerr << "========= Wait for completed transactions" << Endl; + for (ui8 i = 0; i < 1; ++i) + { + const auto writeResult = WaitForWriteCompleted(runtime, sender); + + UNIT_ASSERT_GE(writeResult.GetStep(), Max(minStep1, minStep2)); + UNIT_ASSERT_LE(writeResult.GetStep(), Min(maxStep1, maxStep2)); + UNIT_ASSERT_VALUES_EQUAL(writeResult.GetOrderId(), txId); + UNIT_ASSERT_VALUES_EQUAL(writeResult.GetTxId(), txId); + + UNIT_ASSERT_VALUES_EQUAL(writeResult.TxLocksSize(), 0); + + const auto& tableAccessStats = writeResult.GetTxStats().GetTableAccessStats(0); + UNIT_ASSERT_VALUES_EQUAL(tableAccessStats.GetUpdateRow().GetCount(), rowCount); + UNIT_ASSERT_VALUES_EQUAL(tableAccessStats.GetTableInfo().GetName(), "/Root/" + (writeResult.GetOrigin() == tabletId1 ? tableName1 : tableName2)); + } + + Cout << "========= Read from tables" << Endl; + { + auto tableState = TReadTableState(server, MakeReadTableSettings("/Root/" + tableName1)).All(); + UNIT_ASSERT_VALUES_EQUAL(tableState, expectedTableState); + tableState = TReadTableState(server, MakeReadTableSettings("/Root/"+ tableName2)).All(); + UNIT_ASSERT_VALUES_EQUAL(tableState, expectedTableState); + } + } + Y_UNIT_TEST(WritePreparedNoTxCache) { auto [runtime, server, sender] = TestCreateServer(); diff --git a/ydb/core/tx/replication/service/table_writer.cpp b/ydb/core/tx/replication/service/table_writer.cpp index c0343c6450d1..289527c03398 100644 --- a/ydb/core/tx/replication/service/table_writer.cpp +++ b/ydb/core/tx/replication/service/table_writer.cpp @@ -105,7 +105,18 @@ class TTablePartitionWriter: public TActorBootstrapped { << ": status# " << static_cast(record.GetStatus()) << ", reason# " << static_cast(record.GetReason()) << ", error# " << record.GetErrorDescription()); - return Leave(); + return Leave(IsHardError(record.GetReason())); + } + } + + static bool IsHardError(NKikimrTxDataShard::TEvApplyReplicationChangesResult::EReason reason) { + switch (reason) { + case NKikimrTxDataShard::TEvApplyReplicationChangesResult::REASON_SCHEME_ERROR: + case NKikimrTxDataShard::TEvApplyReplicationChangesResult::REASON_BAD_REQUEST: + case NKikimrTxDataShard::TEvApplyReplicationChangesResult::REASON_UNEXPECTED_ROW_OPERATION: + return true; + default: + return false; } } @@ -115,8 +126,8 @@ class TTablePartitionWriter: public TActorBootstrapped { } } - void Leave() { - Send(Parent, new NChangeExchange::TEvChangeExchangePrivate::TEvGone(TabletId)); + void Leave(bool hardError = false) { + Send(Parent, new NChangeExchange::TEvChangeExchangePrivate::TEvGone(TabletId, hardError)); PassAway(); } @@ -451,7 +462,12 @@ class TLocalTableWriter void Handle(NChangeExchange::TEvChangeExchangePrivate::TEvGone::TPtr& ev) { LOG_D("Handle " << ev->Get()->ToString()); - OnGone(ev->Get()->PartitionId); + + if (ev->Get()->HardError) { + Leave(TEvWorker::TEvGone::SCHEME_ERROR); + } else { + OnGone(ev->Get()->PartitionId); + } } void Retry() { diff --git a/ydb/core/tx/replication/service/table_writer_ut.cpp b/ydb/core/tx/replication/service/table_writer_ut.cpp index 6b89e0159879..1763cedab9c5 100644 --- a/ydb/core/tx/replication/service/table_writer_ut.cpp +++ b/ydb/core/tx/replication/service/table_writer_ut.cpp @@ -12,6 +12,8 @@ namespace NKikimr::NReplication::NService { Y_UNIT_TEST_SUITE(LocalTableWriter) { + using namespace NTestHelpers; + Y_UNIT_TEST(WriteTable) { TEnv env; env.GetRuntime().SetLogPriority(NKikimrServices::REPLICATION_SERVICE, NLog::PRI_DEBUG); diff --git a/ydb/core/tx/replication/service/topic_reader.cpp b/ydb/core/tx/replication/service/topic_reader.cpp index 217af89bb081..a787c6c49099 100644 --- a/ydb/core/tx/replication/service/topic_reader.cpp +++ b/ydb/core/tx/replication/service/topic_reader.cpp @@ -66,11 +66,12 @@ class TRemoteTopicReader: public TActor { void Handle(TEvYdbProxy::TEvTopicReaderGone::TPtr& ev) { LOG_D("Handle " << ev->Get()->ToString()); + switch (ev->Get()->Result.GetStatus()) { - case NYdb::EStatus::SCHEME_ERROR: - return Leave(TEvWorker::TEvGone::SCHEME_ERROR); - default: - return Leave(TEvWorker::TEvGone::UNAVAILABLE); + case NYdb::EStatus::SCHEME_ERROR: + return Leave(TEvWorker::TEvGone::SCHEME_ERROR); + default: + return Leave(TEvWorker::TEvGone::UNAVAILABLE); } } diff --git a/ydb/core/tx/replication/service/topic_reader_ut.cpp b/ydb/core/tx/replication/service/topic_reader_ut.cpp index 1b66e29e0d44..5cfcf70a79c7 100644 --- a/ydb/core/tx/replication/service/topic_reader_ut.cpp +++ b/ydb/core/tx/replication/service/topic_reader_ut.cpp @@ -11,50 +11,46 @@ namespace NKikimr::NReplication::NService { Y_UNIT_TEST_SUITE(RemoteTopicReader) { + using namespace NTestHelpers; + template TActorId CreateReader(Env& env, const TEvYdbProxy::TTopicReaderSettings& settings) { - auto reader = env.GetRuntime().Register(CreateRemoteTopicReader(env.GetYdbProxy(), settings)); - env.SendAsync(reader, new TEvWorker::TEvHandshake()); + do { + auto reader = env.GetRuntime().Register(CreateRemoteTopicReader(env.GetYdbProxy(), settings)); + env.SendAsync(reader, new TEvWorker::TEvHandshake()); - while (true) { - TAutoPtr handle; - auto result = env.GetRuntime().template GrabEdgeEventsRethrow(handle); - if (handle->Sender != reader) { - continue; - } + TAutoPtr ev; + do { + env.GetRuntime().template GrabEdgeEvents(ev); + } while (ev->Sender != reader); - if (auto* ev = std::get(result)) { + switch (ev->GetTypeRewrite()) { + case TEvWorker::EvHandshake: return reader; - } else if (std::get(result)) { - reader = env.GetRuntime().Register(CreateRemoteTopicReader(env.GetYdbProxy(), settings)); - env.SendAsync(reader, new TEvWorker::TEvHandshake()); + case TEvWorker::EvGone: continue; - } else { - UNIT_ASSERT("Unexpected event"); } - } + } while (true); } template auto ReadData(Env& env, TActorId& reader, const TEvYdbProxy::TTopicReaderSettings& settings) { - reader = CreateReader(env, settings); - env.SendAsync(reader, new TEvWorker::TEvPoll()); + do { + reader = CreateReader(env, settings); + env.SendAsync(reader, new TEvWorker::TEvPoll()); - while (true) { - TAutoPtr handle; - auto result = env.GetRuntime().template GrabEdgeEventsRethrow(handle); - if (handle->Sender != reader) { - continue; - } + TAutoPtr ev; + do { + env.GetRuntime().template GrabEdgeEvents(ev); + } while (ev->Sender != reader); - if (auto* ev = std::get(result)) { - return ev->Records; - } else if (std::get(result)) { - reader = CreateReader(env, settings); - env.SendAsync(reader, new TEvWorker::TEvPoll()); + switch (ev->GetTypeRewrite()) { + case TEvWorker::EvData: + return ev->Get()->Records; + case TEvWorker::EvGone: continue; } - } + } while (true); } Y_UNIT_TEST(ReadTopic) { @@ -68,7 +64,7 @@ Y_UNIT_TEST_SUITE(RemoteTopicReader) { .ConsumerName("consumer") .EndAddConsumer(); - auto ev = env.Send( + auto ev = env.Send(env.GetYdbProxy(), new TEvYdbProxy::TEvCreateTopicRequest("/Root/topic", settings)); UNIT_ASSERT(ev); UNIT_ASSERT(ev->Get()->Result.IsSuccess()); diff --git a/ydb/core/tx/replication/service/ut_worker/ya.make b/ydb/core/tx/replication/service/ut_worker/ya.make new file mode 100644 index 000000000000..2085cf27f8bf --- /dev/null +++ b/ydb/core/tx/replication/service/ut_worker/ya.make @@ -0,0 +1,22 @@ +UNITTEST_FOR(ydb/core/tx/replication/service) + +FORK_SUBTESTS() + +SIZE(MEDIUM) + +TIMEOUT(600) + +PEERDIR( + ydb/core/tx/replication/ut_helpers + ydb/core/tx/replication/ydb_proxy + ydb/public/sdk/cpp/client/ydb_topic + library/cpp/testing/unittest +) + +SRCS( + worker_ut.cpp +) + +YQL_LAST_ABI_VERSION() + +END() diff --git a/ydb/core/tx/replication/service/worker.cpp b/ydb/core/tx/replication/service/worker.cpp index a640f2541e72..8f2be72c6aa5 100644 --- a/ydb/core/tx/replication/service/worker.cpp +++ b/ydb/core/tx/replication/service/worker.cpp @@ -23,6 +23,11 @@ TEvWorker::TEvData::TRecord::TRecord(ui64 offset, TString&& data) { } +TEvWorker::TEvData::TEvData(const TVector& records) + : Records(records) +{ +} + TEvWorker::TEvData::TEvData(TVector&& records) : Records(std::move(records)) { @@ -31,7 +36,7 @@ TEvWorker::TEvData::TEvData(TVector&& records) void TEvWorker::TEvData::TRecord::Out(IOutputStream& out) const { out << "{" << " Offset: " << Offset - << " Data: " << Data + << " Data: " << Data.size() << "b" << " }"; } @@ -53,14 +58,17 @@ TString TEvWorker::TEvGone::ToString() const { } class TWorker: public TActorBootstrapped { - struct TActorInfo { - THolder Actor; + class TActorInfo { + std::function CreateFn; TActorId ActorId; bool InitDone; + ui32 CreateAttempt; - explicit TActorInfo(THolder&& actor) - : Actor(std::move(actor)) + public: + explicit TActorInfo(std::function&& createFn) + : CreateFn(std::move(createFn)) , InitDone(false) + , CreateAttempt(0) { } @@ -71,6 +79,22 @@ class TWorker: public TActorBootstrapped { explicit operator bool() const { return InitDone; } + + void Register(IActorOps* ops) { + ActorId = ops->RegisterWithSameMailbox(CreateFn()); + ops->Send(ActorId, new TEvWorker::TEvHandshake()); + InitDone = false; + ++CreateAttempt; + } + + void Registered() { + InitDone = true; + CreateAttempt = 0; + } + + ui32 GetCreateAttempt() const { + return CreateAttempt; + } }; TStringBuf GetLogPrefix() const { @@ -83,39 +107,30 @@ class TWorker: public TActorBootstrapped { return LogPrefix.GetRef(); } - TActorId RegisterActor(TActorInfo& info) { - Y_ABORT_UNLESS(info.Actor); - info.ActorId = RegisterWithSameMailbox(info.Actor.Release()); - return info.ActorId; - } - - void InitActor(TActorInfo& info) { - Y_ABORT_UNLESS(info.ActorId); - Send(info.ActorId, new TEvWorker::TEvHandshake()); - info.InitDone = false; - } - void Handle(TEvWorker::TEvHandshake::TPtr& ev) { LOG_D("Handle " << ev->Get()->ToString()); if (ev->Sender == Reader) { LOG_I("Handshake with reader" << ": sender# " << ev->Sender); - Reader.InitDone = true; + + Reader.Registered(); + if (!InFlightRecords) { + Send(Reader, new TEvWorker::TEvPoll()); + } } else if (ev->Sender == Writer) { LOG_I("Handshake with writer" << ": sender# " << ev->Sender); - Writer.InitDone = true; + + Writer.Registered(); + if (InFlightRecords) { + Send(Writer, new TEvWorker::TEvData(InFlightRecords)); + } } else { LOG_W("Handshake from unknown actor" << ": sender# " << ev->Sender); return; } - - if (Reader && Writer) { - LOG_N("Start working"); - Send(Reader, new TEvWorker::TEvPoll()); - } } void Handle(TEvWorker::TEvPoll::TPtr& ev) { @@ -127,7 +142,10 @@ class TWorker: public TActorBootstrapped { return; } - Send(ev->Forward(Reader)); + InFlightRecords.clear(); + if (Reader) { + Send(ev->Forward(Reader)); + } } void Handle(TEvWorker::TEvData::TPtr& ev) { @@ -139,23 +157,46 @@ class TWorker: public TActorBootstrapped { return; } - Send(ev->Forward(Writer)); + Y_ABORT_UNLESS(InFlightRecords.empty()); + InFlightRecords = ev->Get()->Records; + + if (Writer) { + Send(ev->Forward(Writer)); + } } void Handle(TEvWorker::TEvGone::TPtr& ev) { - // TODO: handle status if (ev->Sender == Reader) { LOG_I("Reader has gone" << ": sender# " << ev->Sender); + MaybeRecreateActor(ev->Get()->Status, Reader); } else if (ev->Sender == Writer) { LOG_I("Writer has gone" << ": sender# " << ev->Sender); + MaybeRecreateActor(ev->Get()->Status, Writer); } else { LOG_W("Unknown actor has gone" << ": sender# " << ev->Sender); } } + void MaybeRecreateActor(TEvWorker::TEvGone::EStatus status, TActorInfo& info) { + switch (status) { + case TEvWorker::TEvGone::UNAVAILABLE: + if (info.GetCreateAttempt() < MaxAttempts) { + return info.Register(this); + } + [[fallthrough]]; + default: + return Leave(); + } + } + + void Leave() { + // TODO: signal to parent + PassAway(); + } + void PassAway() override { for (auto* actor : {&Reader, &Writer}) { Send(*actor, new TEvents::TEvPoison()); @@ -169,16 +210,15 @@ class TWorker: public TActorBootstrapped { return NKikimrServices::TActivity::REPLICATION_WORKER; } - explicit TWorker(THolder&& reader, THolder&& writer) - : Reader(std::move(reader)) - , Writer(std::move(writer)) + explicit TWorker(std::function&& createReaderFn, std::function&& createWriterFn) + : Reader(std::move(createReaderFn)) + , Writer(std::move(createWriterFn)) { } void Bootstrap() { for (auto* actor : {&Reader, &Writer}) { - RegisterActor(*actor); - InitActor(*actor); + actor->Register(this); } Become(&TThis::StateWork); @@ -195,13 +235,15 @@ class TWorker: public TActorBootstrapped { } private: + static constexpr ui32 MaxAttempts = 3; mutable TMaybe LogPrefix; TActorInfo Reader; TActorInfo Writer; + TVector InFlightRecords; }; -IActor* CreateWorker(THolder&& reader, THolder&& writer) { - return new TWorker(std::move(reader), std::move(writer)); +IActor* CreateWorker(std::function&& createReaderFn, std::function&& createWriterFn) { + return new TWorker(std::move(createReaderFn), std::move(createWriterFn)); } } diff --git a/ydb/core/tx/replication/service/worker.h b/ydb/core/tx/replication/service/worker.h index b29d0b6b45a7..54e6d351a773 100644 --- a/ydb/core/tx/replication/service/worker.h +++ b/ydb/core/tx/replication/service/worker.h @@ -5,6 +5,8 @@ #include +#include + namespace NKikimr::NReplication::NService { struct TEvWorker { @@ -36,6 +38,7 @@ struct TEvWorker { TVector Records; + explicit TEvData(const TVector& records); explicit TEvData(TVector&& records); TString ToString() const override; }; @@ -53,7 +56,7 @@ struct TEvWorker { }; }; -IActor* CreateWorker(THolder&& reader, THolder&& writer); +IActor* CreateWorker(std::function&& createReaderFn, std::function&& createWriterFn); } diff --git a/ydb/core/tx/replication/service/worker_ut.cpp b/ydb/core/tx/replication/service/worker_ut.cpp new file mode 100644 index 000000000000..d03ff144f460 --- /dev/null +++ b/ydb/core/tx/replication/service/worker_ut.cpp @@ -0,0 +1,68 @@ +#include "table_writer.h" +#include "topic_reader.h" +#include "worker.h" + +#include +#include +#include +#include +#include + +#include + +namespace NKikimr::NReplication::NService { + +Y_UNIT_TEST_SUITE(Worker) { + using namespace NTestHelpers; + + Y_UNIT_TEST(Basic) { + TEnv env; + env.GetRuntime().SetLogPriority(NKikimrServices::REPLICATION_SERVICE, NLog::PRI_DEBUG); + + { + auto ev = env.Send(env.GetYdbProxy(), + new TEvYdbProxy::TEvCreateTopicRequest("/Root/topic", + NYdb::NTopic::TCreateTopicSettings() + .BeginAddConsumer() + .ConsumerName("consumer") + .EndAddConsumer() + )); + + UNIT_ASSERT(ev); + UNIT_ASSERT(ev->Get()->Result.IsSuccess()); + } + + env.CreateTable("/Root", *MakeTableDescription(TTestTableDescription{ + .Name = "Test", + .KeyColumns = {"key"}, + .Columns = { + {.Name = "key", .Type = "Uint32"}, + {.Name = "value", .Type = "Utf8"}, + }, + })); + + auto createReaderFn = [ydbProxy = env.GetYdbProxy()]() { + return CreateRemoteTopicReader(ydbProxy, + TEvYdbProxy::TTopicReaderSettings() + .ConsumerName("consumer") + .AppendTopics(NYdb::NTopic::TTopicReadSettings() + .Path("/Root/topic") + .AppendPartitionIds(0) + ) + ); + }; + + auto createWriterFn = [tablePathId = env.GetPathId("/Root/Table")]() { + return CreateLocalTableWriter(tablePathId); + }; + + auto worker = env.GetRuntime().Register(CreateWorker(std::move(createReaderFn), std::move(createWriterFn))); + Y_UNUSED(worker); + + UNIT_ASSERT(WriteTopic(env, "/Root/topic", R"({"key":[1], "update":{"value":"10"}})")); + UNIT_ASSERT(WriteTopic(env, "/Root/topic", R"({"key":[2], "update":{"value":"20"}})")); + UNIT_ASSERT(WriteTopic(env, "/Root/topic", R"({"key":[3], "update":{"value":"30"}})")); + } +} + +} diff --git a/ydb/core/tx/replication/service/ya.make b/ydb/core/tx/replication/service/ya.make index b6dab04594c2..92ac750a9af2 100644 --- a/ydb/core/tx/replication/service/ya.make +++ b/ydb/core/tx/replication/service/ya.make @@ -31,4 +31,5 @@ END() RECURSE_FOR_TESTS( ut_table_writer ut_topic_reader + ut_worker ) diff --git a/ydb/core/tx/replication/ut_helpers/test_env.h b/ydb/core/tx/replication/ut_helpers/test_env.h index 84d17737558b..5a4a1b72c14a 100644 --- a/ydb/core/tx/replication/ut_helpers/test_env.h +++ b/ydb/core/tx/replication/ut_helpers/test_env.h @@ -8,7 +8,7 @@ #include -namespace NKikimr::NReplication { +namespace NKikimr::NReplication::NTestHelpers { template class TEnv { @@ -136,11 +136,6 @@ class TEnv { return Server.GetRuntime()->GrabEdgeEvent(Sender); } - template - auto Send(IEventBase* ev) { - return Send(YdbProxy, ev); - } - auto& GetRuntime() { return *Server.GetRuntime(); } diff --git a/ydb/core/tx/replication/ut_helpers/test_table.cpp b/ydb/core/tx/replication/ut_helpers/test_table.cpp index 1fe8d388e59d..1b762c25274c 100644 --- a/ydb/core/tx/replication/ut_helpers/test_table.cpp +++ b/ydb/core/tx/replication/ut_helpers/test_table.cpp @@ -2,7 +2,7 @@ #include -namespace NKikimr::NReplication { +namespace NKikimr::NReplication::NTestHelpers { void TTestTableDescription::TColumn::SerializeTo(NKikimrSchemeOp::TColumnDescription& proto) const { proto.SetName(Name); diff --git a/ydb/core/tx/replication/ut_helpers/test_table.h b/ydb/core/tx/replication/ut_helpers/test_table.h index c730c51df1b1..0314bc1a2be4 100644 --- a/ydb/core/tx/replication/ut_helpers/test_table.h +++ b/ydb/core/tx/replication/ut_helpers/test_table.h @@ -9,7 +9,7 @@ namespace NKikimrSchemeOp { class TTableDescription; } -namespace NKikimr::NReplication { +namespace NKikimr::NReplication::NTestHelpers { struct TTestTableDescription { struct TColumn { diff --git a/ydb/core/tx/replication/ut_helpers/write_topic.h b/ydb/core/tx/replication/ut_helpers/write_topic.h index 3fc8955d7b77..8d384a8d653a 100644 --- a/ydb/core/tx/replication/ut_helpers/write_topic.h +++ b/ydb/core/tx/replication/ut_helpers/write_topic.h @@ -1,6 +1,6 @@ #include -namespace NKikimr::NReplication { +namespace NKikimr::NReplication::NTestHelpers { template bool WriteTopic(const Env& env, const TString& topicPath, const TString& data) { diff --git a/ydb/core/tx/replication/ydb_proxy/ydb_proxy.cpp b/ydb/core/tx/replication/ydb_proxy/ydb_proxy.cpp index dad3284467ac..942fa445e524 100644 --- a/ydb/core/tx/replication/ydb_proxy/ydb_proxy.cpp +++ b/ydb/core/tx/replication/ydb_proxy/ydb_proxy.cpp @@ -193,8 +193,10 @@ class TTopicReader: public TBaseProxyActor { } void Handle(TEvPrivate::TEvTopicEventReady::TPtr& ev) { - auto event = Session->GetEvent(true); - Y_ABORT_UNLESS(event.Defined()); + auto event = Session->GetEvent(false); + if (!event) { + return WaitEvent(ev->Get()->Sender, ev->Get()->Cookie); + } if (auto* x = std::get_if(&*event)) { x->Confirm(); @@ -227,7 +229,7 @@ class TTopicReader: public TBaseProxyActor { } void PassAway() override { - Session->Close(TDuration::MilliSeconds(100)); // non-blocking if there is no inflight commits + Session->Close(TDuration::Zero()); TBaseProxyActor::PassAway(); } diff --git a/ydb/core/tx/replication/ydb_proxy/ydb_proxy_ut.cpp b/ydb/core/tx/replication/ydb_proxy/ydb_proxy_ut.cpp index 6b5a749ef4ae..c7e7aad10384 100644 --- a/ydb/core/tx/replication/ydb_proxy/ydb_proxy_ut.cpp +++ b/ydb/core/tx/replication/ydb_proxy/ydb_proxy_ut.cpp @@ -11,6 +11,20 @@ namespace NKikimr::NReplication { Y_UNIT_TEST_SUITE(YdbProxyTests) { + template + class TEnv: public NTestHelpers::TEnv { + using TBase = NTestHelpers::TEnv; + + public: + using TBase::TBase; + using TBase::Send; + + template + auto Send(IEventBase* ev) { + return TBase::template Send(this->GetYdbProxy(), ev); + } + }; + Y_UNIT_TEST(MakeDirectory) { TEnv env; // ok @@ -131,7 +145,7 @@ Y_UNIT_TEST_SUITE(YdbProxyTests) { } Y_UNIT_TEST(StaticCreds) { - TEnv env("user1", "password1"); + TEnv env("user1", "password1"); // make dir { auto ev = env.Send( @@ -623,7 +637,7 @@ Y_UNIT_TEST_SUITE(YdbProxyTests) { TActorId reader = CreateTopicReader(env, "/Root/topic"); - UNIT_ASSERT(WriteTopic(env, "/Root/topic", "message-0")); + UNIT_ASSERT(NTestHelpers::WriteTopic(env, "/Root/topic", "message-0")); { auto data = ReadTopicData(env, reader, "/Root/topic"); UNIT_ASSERT_VALUES_EQUAL(data.Messages.size(), 1); @@ -651,7 +665,7 @@ Y_UNIT_TEST_SUITE(YdbProxyTests) { env.SendAsync(reader, new TEvents::TEvPoison()); } - UNIT_ASSERT(WriteTopic(env, "/Root/topic", "message-1")); + UNIT_ASSERT(NTestHelpers::WriteTopic(env, "/Root/topic", "message-1")); { auto data = ReadTopicData(env, newReader, "/Root/topic"); UNIT_ASSERT(data.Messages.size() >= 1); @@ -669,7 +683,7 @@ Y_UNIT_TEST_SUITE(YdbProxyTests) { TEnv env; auto reader = CreateTopicReader(env, "/Root/topic"); - auto ev = env.template Send(reader, new TEvYdbProxy::TEvReadTopicRequest()); + auto ev = env.Send(reader, new TEvYdbProxy::TEvReadTopicRequest()); UNIT_ASSERT(ev); UNIT_ASSERT_VALUES_EQUAL(ev->Get()->Result.GetStatus(), NYdb::EStatus::SCHEME_ERROR); diff --git a/ydb/epilogue.cmake b/ydb/epilogue.cmake deleted file mode 100644 index 97ef638b67f2..000000000000 --- a/ydb/epilogue.cmake +++ /dev/null @@ -1,8 +0,0 @@ -add_custom_target(ydb_py_protos) - -add_custom_command( - TARGET ydb_py_protos POST_BUILD - COMMAND ${CMAKE_SOURCE_DIR}/ydb/tests/oss/launch/compile_protos.sh ${CMAKE_SOURCE_DIR} ydb library/cpp/actors - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - COMMENT "Compiling PY protos..." -) diff --git a/ydb/library/actors/core/actor.h b/ydb/library/actors/core/actor.h index 98b551f41cee..720619954341 100644 --- a/ydb/library/actors/core/actor.h +++ b/ydb/library/actors/core/actor.h @@ -352,13 +352,6 @@ namespace NActors { friend class TExecutorPoolBaseMailboxed; friend class TGenericExecutorThread; - IActor(const ui32 activityType) - : SelfActorId(TActorId()) - , ElapsedTicks(0) - , ActivityType(activityType) - , HandledEvents(0) { - } - protected: TActorCallbackBehaviour CImpl; public: @@ -367,29 +360,25 @@ namespace NActors { /// @sa services.proto NKikimrServices::TActivity::EType using EActorActivity = EInternalActorType; using EActivityType = EActorActivity; - ui32 ActivityType; + TActivityIndex ActivityType; protected: ui64 HandledEvents; - template ::value, bool>::type v = true> - IActor(const EEnum activityEnumType = EActivityType::OTHER) - : IActor(TEnumProcessKey::GetIndex(activityEnumType)) { - } + IActor(TActivityIndex activity = EActivityType::OTHER) + : SelfActorId(TActorId()) + , ElapsedTicks(0) + , ActivityType(activity) + , HandledEvents(0) + {} - IActor(TActorCallbackBehaviour&& cImpl, const ui32 activityType) + IActor(TActorCallbackBehaviour&& cImpl, TActivityIndex activity) : SelfActorId(TActorId()) , ElapsedTicks(0) , CImpl(std::move(cImpl)) - , ActivityType(activityType) + , ActivityType(activity) , HandledEvents(0) - { - } - - template ::value, bool>::type v = true> - IActor(TActorCallbackBehaviour&& cImpl, const EEnum activityEnumType = EActivityType::OTHER) - : IActor(std::move(cImpl), TEnumProcessKey::GetIndex(activityEnumType)) { - } + {} public: template @@ -441,8 +430,9 @@ namespace NActors { virtual void PassAway(); protected: - void SetActivityType(ui32 activityType) { - ActivityType = activityType; + template ::value, bool>::type v = true> + void SetActivityType(TActivityIndex activity) { + ActivityType = activity; } public: @@ -617,13 +607,11 @@ namespace NActors { protected: template IActorCallback(TReceiveFunc stateFunc, const TEnum activityType = IActor::EActivityType::OTHER) - : IActor(TActorCallbackBehaviour(stateFunc), activityType) { - + : IActor(TActorCallbackBehaviour(stateFunc), TActivityIndex{activityType}) { } - IActorCallback(TReceiveFunc stateFunc, const ui32 activityType) + IActorCallback(TReceiveFunc stateFunc, TActivityIndex activityType) : IActor(TActorCallbackBehaviour(stateFunc), activityType) { - } public: @@ -676,9 +664,9 @@ namespace NActors { } } - static ui32 GetActivityTypeIndex() { + static TActivityIndex GetActivityTypeIndex() { static const ui32 result = GetActivityTypeIndexImpl(); - return result; + return static_cast(result); } protected: @@ -690,11 +678,11 @@ namespace NActors { template TActor(TDerivedReceiveFunc func, const TEnum activityEnumType = EActivityType::OTHER) - : IActorCallback(static_cast(func), activityEnumType) { + : IActorCallback(static_cast(func), TActivityIndex{activityEnumType}) { } TActor(TDerivedReceiveFunc func, const TString& actorName) - : IActorCallback(static_cast(func), TLocalProcessKeyState::GetInstance().Register(actorName)) { + : IActorCallback(static_cast(func), TActivityIndex{TLocalProcessKeyState::GetInstance().Register(actorName)}) { } public: @@ -782,7 +770,7 @@ namespace NActors { public: TDecorator(THolder&& actor) - : IActorCallback(static_cast(&TDecorator::State), actor->GetActivityType()) + : IActorCallback(static_cast(&TDecorator::State)) , Actor(std::move(actor)) { } diff --git a/ydb/library/actors/util/local_process_key.h b/ydb/library/actors/util/local_process_key.h index bff8bef81b7b..e219475a59aa 100644 --- a/ydb/library/actors/util/local_process_key.h +++ b/ydb/library/actors/util/local_process_key.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include #include @@ -8,6 +10,23 @@ #include #include + +struct TActivityIndex { + ui32 Value; + + explicit TActivityIndex(ui32 value) + : Value(value) + {} + + template ::value, bool>::type v = true> + TActivityIndex(EEnum activityType); + + operator ui32() const { + return Value; + } +}; + + class TLocalProcessKeyStateIndexLimiter { public: static constexpr ui32 GetMaxKeysCount() { @@ -39,12 +58,12 @@ friend class TEnumProcessKey; return MaxKeysCount; } - TStringBuf GetNameByIndex(size_t index) const { + TStringBuf GetNameByIndex(ui32 index) const { Y_ABORT_UNLESS(index < Names.size()); return Names[index]; } - size_t GetIndexByName(TStringBuf name) const { + ui32 GetIndexByName(TStringBuf name) const { TGuard g(Mutex); auto it = Map.find(name); Y_ENSURE(it != Map.end()); @@ -55,7 +74,7 @@ friend class TEnumProcessKey; Names.resize(MaxKeysCount); } - size_t Register(TStringBuf name) { + ui32 Register(TStringBuf name) { TGuard g(Mutex); auto it = Map.find(name); if (it != Map.end()) { @@ -77,7 +96,7 @@ friend class TEnumProcessKey; private: TVector Names; - THashMap Map; + THashMap Map; TMutex Mutex; }; @@ -88,12 +107,12 @@ class TLocalProcessKey { return Name; } - static size_t GetIndex() { + static ui32 GetIndex() { return Index; } private: - inline static size_t Index = TLocalProcessKeyState::GetInstance().Register(Name); + inline static ui32 Index = TLocalProcessKeyState::GetInstance().Register(Name); }; template @@ -103,7 +122,7 @@ class TLocalProcessExtKey { return Name; } - static size_t GetIndex() { + static ui32 GetIndex() { return Index; } @@ -119,7 +138,7 @@ class TLocalProcessExtKey { } static const inline TString Name = TypeName(); - inline static size_t Index = TLocalProcessKeyState::GetInstance().Register(TypeNameRobust()); + inline static ui32 Index = TLocalProcessKeyState::GetInstance().Register(TypeNameRobust()); }; template @@ -129,17 +148,17 @@ class TEnumProcessKey { return TLocalProcessKeyState::GetInstance().GetNameByIndex(GetIndex(key)); } - static size_t GetIndex(const EnumT key) { + static ui32 GetIndex(const EnumT key) { ui32 index = static_cast(key); Y_ABORT_UNLESS(index < Enum2Index.size()); return Enum2Index[index]; } private: - inline static TVector RegisterAll() { + inline static TVector RegisterAll() { static_assert(std::is_enum::value, "Enum is required"); - TVector enum2Index; + TVector enum2Index; auto names = GetEnumNames(); ui32 maxId = 0; for (const auto& [k, v] : names) { @@ -153,5 +172,10 @@ class TEnumProcessKey { return enum2Index; } - inline static TVector Enum2Index = RegisterAll(); + inline static TVector Enum2Index = RegisterAll(); }; + +template ::value, bool>::type v> +TActivityIndex::TActivityIndex(EEnum activityEnumType) + : Value(TEnumProcessKey::GetIndex(activityEnumType)) +{} diff --git a/ydb/library/actors/wilson/wilson_uploader.cpp b/ydb/library/actors/wilson/wilson_uploader.cpp index 3fdaf8904d5f..35d98209cc95 100644 --- a/ydb/library/actors/wilson/wilson_uploader.cpp +++ b/ydb/library/actors/wilson/wilson_uploader.cpp @@ -84,6 +84,13 @@ namespace NWilson { } }; + struct TExportRequestData : TIntrusiveListItem { + std::unique_ptr Context; + std::unique_ptr> Reader; + grpc::Status Status; + NServiceProto::ExportTraceServiceResponse Response; + }; + class TWilsonUploader : public TActorBootstrapped { @@ -95,6 +102,7 @@ namespace NWilson { ui64 MaxBytesInBatch; TDuration MaxBatchAccumulation = TDuration::Seconds(1); TDuration MaxSpanTimeInQueue; + ui64 MaxExportInflight; bool WakeupScheduled = false; @@ -106,10 +114,6 @@ namespace NWilson { grpc::CompletionQueue CQ; std::unique_ptr GrpcSigner; - std::unique_ptr Context; - std::unique_ptr> Reader; - NServiceProto::ExportTraceServiceResponse Response; - grpc::Status Status; TBatch CurrentBatch; std::queue BatchQueue; @@ -119,6 +123,9 @@ namespace NWilson { bool BatchCompletionScheduled = false; TMonotonic NextBatchCompletion; + TIntrusiveListWithAutoDelete ExportRequests; + size_t ExportRequestsCount = 0; + public: TWilsonUploader(WilsonUploaderParams params) : MaxSpansPerSecond(params.MaxSpansPerSecond) @@ -126,6 +133,7 @@ namespace NWilson { , MaxBytesInBatch(params.MaxBytesInBatch) , MaxBatchAccumulation(params.MaxBatchAccumulation) , MaxSpanTimeInQueue(TDuration::Seconds(params.SpanExportTimeoutSeconds)) + , MaxExportInflight(params.MaxExportRequestsInflight) , CollectorUrl(std::move(params.CollectorUrl)) , ServiceName(std::move(params.ServiceName)) , GrpcSigner(std::move(params.GrpcSigner)) @@ -149,6 +157,10 @@ namespace NWilson { ALOG_WARN(WILSON_SERVICE_ID, "max_spans_in_batch shold be greater than 0, changing to 1"); MaxSpansInBatch = 1; } + if (MaxExportInflight == 0) { + ALOG_WARN(WILSON_SERVICE_ID, "max_span_export_inflight should be greater than 0, changing to 1"); + MaxExportInflight = 1; + } TStringBuf scheme; TStringBuf host; @@ -243,7 +255,7 @@ namespace NWilson { "dropped " << numSpansDropped << " span(s) due to expiration"); } - if (Context || BatchQueue.empty()) { + if (ExportRequestsCount >= MaxExportInflight || BatchQueue.empty()) { return; } else if (now < NextSendTimestamp) { ScheduleWakeup(NextSendTimestamp); @@ -267,29 +279,42 @@ namespace NWilson { SpansSizeBytes -= batch.SizeBytes; ScheduleWakeup(NextSendTimestamp); - Context = std::make_unique(); + + auto context = std::make_unique(); if (GrpcSigner) { - GrpcSigner->SignClientContext(*Context); + GrpcSigner->SignClientContext(*context); } - Reader = Stub->AsyncExport(Context.get(), std::move(batch.Request), &CQ); - Reader->Finish(&Response, &Status, nullptr); + auto reader = Stub->AsyncExport(context.get(), std::move(batch.Request), &CQ); + auto uploadData = std::unique_ptr(new TExportRequestData { + .Context = std::move(context), + .Reader = std::move(reader), + }); + uploadData->Reader->Finish(&uploadData->Response, &uploadData->Status, uploadData.get()); + ALOG_TRACE(WILSON_SERVICE_ID, "started export request " << (void*)uploadData.get()); + ExportRequests.PushBack(uploadData.release()); + ++ExportRequestsCount; } - void CheckIfDone() { - if (Context) { - void *tag; - bool ok; - if (CQ.AsyncNext(&tag, &ok, std::chrono::system_clock::now()) == grpc::CompletionQueue::GOT_EVENT) { - if (!Status.ok()) { - ALOG_ERROR(WILSON_SERVICE_ID, - "failed to commit traces: " << Status.error_message()); - } - - Reader.reset(); - Context.reset(); - } else { - ScheduleWakeup(TDuration::MilliSeconds(100)); + void ReapCompletedRequests() { + if (ExportRequests.Empty()) { + return; + } + void* tag; + bool ok; + while (CQ.AsyncNext(&tag, &ok, std::chrono::system_clock::now()) == grpc::CompletionQueue::GOT_EVENT) { + auto node = std::unique_ptr(static_cast(tag)); + ALOG_TRACE(WILSON_SERVICE_ID, "finished export request " << (void*)node.get()); + if (!node->Status.ok()) { + ALOG_ERROR(WILSON_SERVICE_ID, + "failed to commit traces: " << node->Status.error_message()); } + + --ExportRequestsCount; + node->Unlink(); + } + + if (!ExportRequests.Empty()) { + ScheduleWakeup(TDuration::MilliSeconds(100)); } } @@ -322,7 +347,7 @@ namespace NWilson { } void TryMakeProgress() { - CheckIfDone(); + ReapCompletedRequests(); TryToSend(); } diff --git a/ydb/library/actors/wilson/wilson_uploader.h b/ydb/library/actors/wilson/wilson_uploader.h index e64809e518ad..e93e70049162 100644 --- a/ydb/library/actors/wilson/wilson_uploader.h +++ b/ydb/library/actors/wilson/wilson_uploader.h @@ -36,6 +36,7 @@ namespace NWilson { ui64 MaxBatchAccumulationMilliseconds = 1'000; TDuration MaxBatchAccumulation = TDuration::Seconds(1); ui32 SpanExportTimeoutSeconds = 60 * 60 * 24 * 365; + ui64 MaxExportRequestsInflight = 1; NActors::IActor* CreateUploader() &&; }; diff --git a/ydb/library/services/services.proto b/ydb/library/services/services.proto index 9dcdd26738e6..9b28c9c0b7ed 100644 --- a/ydb/library/services/services.proto +++ b/ydb/library/services/services.proto @@ -82,7 +82,6 @@ enum EServiceKikimr { BOOTSTRAPPER = 302; TENANT_POOL = 303; LABELS_MAINTAINER = 305; - GRAPH = 306; // TABLET section @@ -172,7 +171,7 @@ enum EServiceKikimr { PQ_READ_PROXY = 442; PQ_WRITE_PROXY = 443; PQ_MIRRORER = 446; - PQ_READ_SPEED_LIMITER = 447; + PQ_RATE_LIMITER = 447; PERSQUEUE_READ_BALANCER = 448; PQ_MOVE_TOPIC = 449; PQ_FETCH_REQUEST = 451; @@ -1022,5 +1021,7 @@ message TActivity { REPLICATION_REMOTE_TOPIC_READER = 628; REPLICATION_LOCAL_TABLE_WRITER = 629; REPLICATION_TABLE_PARTITION_WRITER = 630; + PERSQUEUE_ACCOUNT_WRITE_QUOTER = 631; + PERSQUEUE_WRITE_QUOTER = 632; }; }; diff --git a/ydb/library/yql/core/type_ann/type_ann_pg.cpp b/ydb/library/yql/core/type_ann/type_ann_pg.cpp index 2ca16b6455c6..bbba4d615976 100644 --- a/ydb/library/yql/core/type_ann/type_ann_pg.cpp +++ b/ydb/library/yql/core/type_ann/type_ann_pg.cpp @@ -2103,7 +2103,7 @@ ui64 CalculateExprHash(const TExprNode& root, TNodeMap& visited) { case TExprNode::EType::List: hash = CseeHash(root.ChildrenSize(), hash); for (ui32 i = 0; i < root.ChildrenSize(); ++i) { - hash = CalculateExprHash(*root.Child(i), visited); + hash = CombineHashes(CalculateExprHash(*root.Child(i), visited), hash); } break; @@ -2112,6 +2112,19 @@ ui64 CalculateExprHash(const TExprNode& root, TNodeMap& visited) { hash = CseeHash(root.Content().Data(), root.Content().Size(), hash); hash = CseeHash(root.GetFlagsToCompare(), hash); break; + case TExprNode::EType::World: + break; + case TExprNode::EType::Lambda: + hash = CseeHash(root.ChildrenSize(), hash); + hash = CseeHash(root.Head().ChildrenSize(), hash); + for (ui32 argIndex = 0; argIndex < root.Head().ChildrenSize(); ++argIndex) { + visited.emplace(root.Head().Child(argIndex), argIndex); + } + + for (ui32 bodyIndex = 1; bodyIndex < root.ChildrenSize(); ++bodyIndex) { + hash = CombineHashes(CalculateExprHash(*root.Child(bodyIndex), visited), hash); + } + break; default: YQL_ENSURE(false, "Unexpected node type"); } @@ -2152,6 +2165,24 @@ bool ExprNodesEquals(const TExprNode& left, const TExprNode& right, TNodeSet& vi return left.Content() == right.Content() && left.GetFlagsToCompare() == right.GetFlagsToCompare(); case TExprNode::EType::Argument: return left.GetArgIndex() == right.GetArgIndex(); + case TExprNode::EType::World: + return true; + case TExprNode::EType::Lambda: + if (left.ChildrenSize() != right.ChildrenSize()) { + return false; + } + + if (left.Head().ChildrenSize() != right.Head().ChildrenSize()) { + return false; + } + + for (ui32 i = 1; i < left.ChildrenSize(); ++i) { + if (!ExprNodesEquals(*left.Child(i), *right.Child(i), visited)) { + return false; + } + } + + return true; default: YQL_ENSURE(false, "Unexpected node type"); } @@ -2625,7 +2656,8 @@ bool IsPlainMemberOverArg(const TExprNode& expr, TStringBuf& memberName) { bool ValidateSort(TInputs& inputs, TInputs& subLinkInputs, const THashSet& possibleAliases, const TExprNode& data, TExtContext& ctx, bool& hasNewSort, TExprNode::TListType& newSorts, bool scanColumnsOnly, - const TExprNode::TPtr& groupExprs, const TStringBuf& scope, const TProjectionOrders* projectionOrders) { + const TExprNode::TPtr& groupExprs, const TStringBuf& scope, const TProjectionOrders* projectionOrders, + const TExprNode::TPtr& projection) { newSorts.clear(); for (ui32 index = 0; index < data.ChildrenSize(); ++index) { auto oneSort = data.Child(index); @@ -2728,6 +2760,64 @@ bool ValidateSort(TInputs& inputs, TInputs& subLinkInputs, const THashSet> projectionHashes; + if (projectionOrders && projection) { + canReplaceProjectionExpr = true; + for (const auto& x : *projectionOrders) { + if (!x->second) { // skip stars + canReplaceProjectionExpr = false; + break; + } + } + + if (canReplaceProjectionExpr) { + auto projectionItems = projection->Tail().ChildrenSize(); + for (ui32 i = 0; i < projectionItems; ++i) { + TNodeMap hashVisited; + const auto& lambda = projection->Tail().Child(i)->Tail(); + if (lambda.Head().ChildrenSize() == 1) { + hashVisited[&lambda.Head().Head()] = 0; + ui64 hash = CalculateExprHash(lambda.Tail(), hashVisited); + projectionHashes[hash].push_back(i); + } + } + } + } + + if (canReplaceProjectionExpr && newLambda->Head().ChildrenSize() == 1) { + TNodeMap hashVisited; + hashVisited[&newLambda->Head().Head()] = 0; + ui64 hash = CalculateExprHash(newLambda->Tail(), hashVisited); + bool changedSort = false; + for (auto projectionIndex : projectionHashes[hash]) { + const auto& projectionLambda = projection->Tail().Child(projectionIndex)->Tail(); + TNodeSet equalsVisited; + if (ExprNodesEquals(newLambda->Tail(), projectionLambda.Tail(), equalsVisited)) { + auto columnName = projectionOrders->at(projectionIndex)->first.front(); + newLambda = ctx.Expr.Builder(newLambda->Pos()) + .Lambda() + .Callable("PgColumnRef") + .Atom(0, columnName) + .Seal() + .Seal() + .Build(); + + newChildren[1] = newLambda; + changedSort = true; + break; + } + } + + if (changedSort) { + newChildren[0] = ctx.Expr.NewCallable(newLambda->Pos(), "Void", {}); + auto newSort = ctx.Expr.ChangeChildren(*oneSort, std::move(newChildren)); + newSorts.push_back(newSort); + hasNewSort = true; + continue; + } + } + newSorts.push_back(data.ChildPtr(index)); } @@ -3895,7 +3985,7 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN bool hasNewSort = false; TExprNode::TListType newSorts; - if (!ValidateSort(joinInputs, joinInputs, possibleAliases, *sort, ctx, hasNewSort, newSorts, scanColumnsOnly, groupExprs, "", nullptr)) { + if (!ValidateSort(joinInputs, joinInputs, possibleAliases, *sort, ctx, hasNewSort, newSorts, scanColumnsOnly, groupExprs, "", nullptr, nullptr)) { return IGraphTransformer::TStatus::Error; } @@ -3990,7 +4080,7 @@ IGraphTransformer::TStatus PgSetItemWrapper(const TExprNode::TPtr& input, TExprN TExprNode::TListType newSortTupleItems; // no effective types yet, scan lambda bodies if (!ValidateSort(projectionInputs, joinInputs, possibleAliases, data, ctx, hasNewSort, newSortTupleItems, - scanColumnsOnly, groupExprs, "ORDER BY", &projectionOrders)) { + scanColumnsOnly, groupExprs, "ORDER BY", &projectionOrders, GetSetting(options, "result"))) { return IGraphTransformer::TStatus::Error; } @@ -4494,7 +4584,7 @@ IGraphTransformer::TStatus PgSelectWrapper(const TExprNode::TPtr& input, TExprNo projectionOrders.push_back(std::make_pair(TColumnOrder{ col }, true)); } - if (!ValidateSort(projectionInputs, projectionInputs, {}, data, ctx, hasNewSort, newSortTupleItems, false, nullptr, "ORDER BY", &projectionOrders)) { + if (!ValidateSort(projectionInputs, projectionInputs, {}, data, ctx, hasNewSort, newSortTupleItems, false, nullptr, "ORDER BY", &projectionOrders, nullptr)) { return IGraphTransformer::TStatus::Error; } diff --git a/ydb/library/yql/core/yql_execution.h b/ydb/library/yql/core/yql_execution.h index cb38d913fa67..f7cda02aa8b7 100644 --- a/ydb/library/yql/core/yql_execution.h +++ b/ydb/library/yql/core/yql_execution.h @@ -18,9 +18,19 @@ namespace NYql { YQL_OPERATION_PROGRESS_STATE_MAP(ENUM_VALUE_GEN) }; +#define YQL_OPERATION_BLOCK_STATUS_MAP(xx) \ + xx(None, 0) \ + xx(Partial, 1) \ + xx(Full, 2) + + enum class EOpBlockStatus { + YQL_OPERATION_BLOCK_STATUS_MAP(ENUM_VALUE_GEN) + }; + TString Category; ui32 Id; EState State; + TMaybe BlockStatus; using TStage = std::pair; TStage Stage; diff --git a/ydb/library/yql/dq/opt/dq_opt_hopping.cpp b/ydb/library/yql/dq/opt/dq_opt_hopping.cpp new file mode 100644 index 000000000000..ff84188dc929 --- /dev/null +++ b/ydb/library/yql/dq/opt/dq_opt_hopping.cpp @@ -0,0 +1,793 @@ +#include "dq_opt_hopping.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace NYql::NDq::NHopping { + +using namespace NYql; +using namespace NYql::NDq; +using namespace NYql::NNodes; + +struct THoppingTraits { + TString Column; + TCoHoppingTraits Traits; + ui64 Hop; + ui64 Interval; + ui64 Delay; +}; + + struct TKeysDescription { + TVector PickleKeys; + TVector MemberKeys; + TVector FakeKeys; + + TKeysDescription(const TStructExprType& rowType, const TCoAtomList& keys, const TString& hoppingColumn) { + for (const auto& key : keys) { + if (key.StringValue() == hoppingColumn) { + FakeKeys.emplace_back(key.StringValue()); + continue; + } + + const auto index = rowType.FindItem(key.StringValue()); + Y_ENSURE(index); + + auto itemType = rowType.GetItems()[*index]->GetItemType(); + if (RemoveOptionalType(itemType)->GetKind() == ETypeAnnotationKind::Data) { + MemberKeys.emplace_back(key.StringValue()); + continue; + } + + PickleKeys.emplace_back(key.StringValue()); + } + } + + TExprNode::TPtr BuildPickleLambda(TExprContext& ctx, TPositionHandle pos) const { + TCoArgument arg = Build(ctx, pos) + .Name("item") + .Done(); + + TExprBase body = arg; + + for (const auto& key : PickleKeys) { + const auto member = Build(ctx, pos) + .Name().Build(key) + .Struct(arg) + .Done() + .Ptr(); + + body = Build(ctx, pos) + .Struct(body) + .Name().Build(key) + .Item(ctx.NewCallable(pos, "StablePickle", { member })) + .Done(); + } + + return Build(ctx, pos) + .Args({arg}) + .Body(body) + .Done() + .Ptr(); + } + + TExprNode::TPtr BuildUnpickleLambda(TExprContext& ctx, TPositionHandle pos, const TStructExprType& rowType) { + TCoArgument arg = Build(ctx, pos) + .Name("item") + .Done(); + + TExprBase body = arg; + + for (const auto& key : PickleKeys) { + const auto index = rowType.FindItem(key); + Y_ENSURE(index); + + auto itemType = rowType.GetItems().at(*index)->GetItemType(); + const auto member = Build(ctx, pos) + .Name().Build(key) + .Struct(arg) + .Done() + .Ptr(); + + body = Build(ctx, pos) + .Struct(body) + .Name().Build(key) + .Item(ctx.NewCallable(pos, "Unpickle", { ExpandType(pos, *itemType, ctx), member })) + .Done(); + } + + return Build(ctx, pos) + .Args({arg}) + .Body(body) + .Done() + .Ptr(); + } + + TVector GetKeysList(TExprContext& ctx, TPositionHandle pos) const { + TVector res; + res.reserve(PickleKeys.size() + MemberKeys.size()); + + for (const auto& pickleKey : PickleKeys) { + res.emplace_back(Build(ctx, pos).Value(pickleKey).Done()); + } + for (const auto& memberKey : MemberKeys) { + res.emplace_back(Build(ctx, pos).Value(memberKey).Done()); + } + return res; + } + + TVector GetActualGroupKeys() { + TVector result; + result.reserve(PickleKeys.size() + MemberKeys.size()); + result.insert(result.end(), PickleKeys.begin(), PickleKeys.end()); + result.insert(result.end(), MemberKeys.begin(), MemberKeys.end()); + return result; + } + + bool NeedPickle() const { + return !PickleKeys.empty(); + } + + TExprNode::TPtr GetKeySelector(TExprContext& ctx, TPositionHandle pos, const TStructExprType* rowType) { + auto builder = Build(ctx, pos); + for (auto key : GetKeysList(ctx, pos)) { + builder.Add(std::move(key)); + } + return BuildKeySelector(pos, *rowType, builder.Build().Value().Ptr(), ctx); + } +}; + +TString BuildColumnName(const TExprBase& column) { + if (const auto columnName = column.Maybe()) { + return columnName.Cast().StringValue(); + } + + if (const auto columnNames = column.Maybe()) { + TStringBuilder columnNameBuilder; + for (const auto columnName : columnNames.Cast()) { + columnNameBuilder.append(columnName.StringValue()); + columnNameBuilder.append("_"); + } + return columnNameBuilder; + } + + YQL_ENSURE(false, "Invalid node. Expected Atom or AtomList, but received: " + << column.Ptr()->Dump()); +} + +bool IsLegacyHopping(const TExprNode::TPtr& hoppingSetting) { + return !hoppingSetting->Child(1)->IsList(); +} + +void EnsureNotDistinct(const TCoAggregate& aggregate) { + const auto& aggregateHandlers = aggregate.Handlers(); + + YQL_ENSURE( + AllOf(aggregateHandlers, [](const auto& t){ return !t.DistinctName(); }), + "Distinct is not supported for aggregation with hop"); +} + +TMaybe ExtractHopTraits(const TCoAggregate& aggregate, TExprContext& ctx, bool analyticsMode) { + const auto pos = aggregate.Pos(); + + const auto hopSetting = GetSetting(aggregate.Settings().Ref(), "hopping"); + if (!hopSetting) { + ctx.AddError(TIssue(ctx.GetPosition(pos), "Aggregate over stream must have 'hopping' setting")); + return Nothing(); + } + + const auto hoppingColumn = IsLegacyHopping(hopSetting) + ? "_yql_time" + : TString(hopSetting->Child(1)->Child(0)->Content()); + + const auto traitsNode = IsLegacyHopping(hopSetting) + ? hopSetting->Child(1) + : hopSetting->Child(1)->Child(1); + + const auto maybeTraits = TMaybeNode(traitsNode); + if (!maybeTraits) { + ctx.AddError(TIssue(ctx.GetPosition(pos), "Invalid 'hopping' setting in Aggregate")); + return Nothing(); + } + + const auto traits = maybeTraits.Cast(); + + const auto checkIntervalParam = [&] (TExprBase param) -> ui64 { + if (param.Maybe()) { + param = param.Cast().Input(); + } + if (!param.Maybe()) { + ctx.AddError(TIssue(ctx.GetPosition(pos), "Not an interval data ctor")); + return 0; + } + auto value = FromString(param.Cast().Literal().Value()); + if (value <= 0) { + ctx.AddError(TIssue(ctx.GetPosition(pos), "Interval value must be positive")); + return 0; + } + return (ui64)value; + }; + + const auto hop = checkIntervalParam(traits.Hop()); + if (!hop) { + return Nothing(); + } + const auto interval = checkIntervalParam(traits.Interval()); + if (!interval) { + return Nothing(); + } + const auto delay = checkIntervalParam(traits.Delay()); + if (!delay) { + return Nothing(); + } + + if (interval < hop) { + ctx.AddError(TIssue(ctx.GetPosition(pos), "Interval must be greater or equal then hop")); + return Nothing(); + } + if (delay < hop) { + ctx.AddError(TIssue(ctx.GetPosition(pos), "Delay must be greater or equal then hop")); + return Nothing(); + } + + const auto newTraits = Build(ctx, aggregate.Pos()) + .InitFrom(traits) + .DataWatermarks(analyticsMode + ? ctx.NewAtom(aggregate.Pos(), "false") + : traits.DataWatermarks().Ptr()) + .Done(); + + return THoppingTraits { + hoppingColumn, + newTraits, + hop, + interval, + delay + }; +} + +TExprNode::TPtr BuildTimeExtractor(const TCoHoppingTraits& hoppingTraits, TExprContext& ctx) { + const auto pos = hoppingTraits.Pos(); + + if (hoppingTraits.ItemType().Ref().GetTypeAnn()->Cast()->GetType()->Cast()->GetSize() == 0) { + // The case when no fields are used in lambda. F.e. when it has only DependsOn. + return ctx.DeepCopyLambda(hoppingTraits.TimeExtractor().Ref()); + } + + return Build(ctx, pos) + .Args({"item"}) + .Body() + .Apply(hoppingTraits.TimeExtractor()) + .With(0) + .Type(hoppingTraits.ItemType()) + .Value("item") + .Build() + .Build() + .Done() + .Ptr(); +} + +TExprNode::TPtr BuildInitHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { + const auto pos = aggregate.Pos(); + const auto& aggregateHandlers = aggregate.Handlers(); + + const auto initItemArg = Build(ctx, pos).Name("item").Done(); + + TVector structItems; + structItems.reserve(aggregateHandlers.Size()); + + ui32 index = 0; + for (const auto& handler : aggregateHandlers) { + const auto tuple = handler.Cast(); + + TMaybeNode applier; + if (tuple.Trait().Cast().InitHandler().Args().Size() == 1) { + applier = Build(ctx, pos) + .Apply(tuple.Trait().Cast().InitHandler()) + .With(0, initItemArg) + .Done(); + } else { + applier = Build(ctx, pos) + .Apply(tuple.Trait().Cast().InitHandler()) + .With(0, initItemArg) + .With(1) + .Literal().Build(ToString(index)) + .Build() + .Done(); + } + + structItems.push_back(Build(ctx, pos) + .Name().Build(BuildColumnName(tuple.ColumnName())) + .Value(applier) + .Done()); + ++index; + } + + return Build(ctx, pos) + .Args({initItemArg}) + .Body() + .Add(structItems) + .Build() + .Done() + .Ptr(); +} + +TExprNode::TPtr BuildUpdateHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { + const auto pos = aggregate.Pos(); + const auto aggregateHandlers = aggregate.Handlers(); + + const auto updateItemArg = Build(ctx, pos).Name("item").Done(); + const auto updateStateArg = Build(ctx, pos).Name("state").Done(); + + TVector structItems; + structItems.reserve(aggregateHandlers.Size()); + + i32 index = 0; + for (const auto& handler : aggregateHandlers) { + const auto tuple = handler.Cast(); + const TString columnName = BuildColumnName(tuple.ColumnName()); + + const auto member = Build(ctx, pos) + .Struct(updateStateArg) + .Name().Build(columnName) + .Done(); + + TMaybeNode applier; + if (tuple.Trait().Cast().UpdateHandler().Args().Size() == 2) { + applier = Build(ctx, pos) + .Apply(tuple.Trait().Cast().UpdateHandler()) + .With(0, updateItemArg) + .With(1, member) + .Done(); + } else { + applier = Build(ctx, pos) + .Apply(tuple.Trait().Cast().UpdateHandler()) + .With(0, updateItemArg) + .With(1, member) + .With(2) + .Literal().Build(ToString(index)) + .Build() + .Done(); + } + + structItems.push_back(Build(ctx, pos) + .Name().Build(columnName) + .Value(applier) + .Done()); + ++index; + } + + return Build(ctx, pos) + .Args({updateItemArg, updateStateArg}) + .Body() + .Add(structItems) + .Build() + .Done() + .Ptr(); +} + +TExprNode::TPtr WrapToShuffle( + const TKeysDescription& keysDescription, + const TCoAggregate& aggregate, + const TDqConnection& input, + TExprContext& ctx) +{ + auto pos = aggregate.Pos(); + + TDqStageBase mappedInput = input.Output().Stage(); + if (keysDescription.NeedPickle()) { + mappedInput = Build(ctx, pos) + .Inputs() + .Add() + .Output() + .Stage(input.Output().Stage()) + .Index(input.Output().Index()) + .Build() + .Build() + .Build() + .Program() + .Args({"stream"}) + .Body() + .Input("stream") + .Lambda(keysDescription.BuildPickleLambda(ctx, pos)) + .Build() + .Build() + .Settings(TDqStageSettings().BuildNode(ctx, pos)) + .Done(); + } + + return Build(ctx, pos) + .Output() + .Stage(mappedInput) + .Index().Value("0").Build() + .Build() + .KeyColumns() + .Add(keysDescription.GetKeysList(ctx, pos)) + .Build() + .Done() + .Ptr(); +} + +TExprNode::TPtr BuildMergeHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { + const auto pos = aggregate.Pos(); + const auto& aggregateHandlers = aggregate.Handlers(); + + const auto mergeState1Arg = Build(ctx, pos).Name("state1").Done(); + const auto mergeState2Arg = Build(ctx, pos).Name("state2").Done(); + + TVector structItems; + structItems.reserve(aggregateHandlers.Size()); + + for (const auto& handler : aggregateHandlers) { + const auto tuple = handler.Cast(); + const TString columnName = BuildColumnName(tuple.ColumnName()); + + const auto member1 = Build(ctx, pos) + .Struct(mergeState1Arg) + .Name().Build(columnName) + .Done(); + const auto member2 = Build(ctx, pos) + .Struct(mergeState2Arg) + .Name().Build(columnName) + .Done(); + + structItems.push_back(Build(ctx, pos) + .Name().Build(columnName) + .Value() + .Apply(tuple.Trait().Cast().MergeHandler()) + .With(0, member1) + .With(1, member2) + .Build() + .Done()); + } + + return Build(ctx, pos) + .Args({mergeState1Arg, mergeState2Arg}) + .Body() + .Add(structItems) + .Build() + .Done() + .Ptr(); +} + +TExprNode::TPtr BuildFinishHopLambda( + const TCoAggregate& aggregate, + const TVector& actualGroupKeys, + const TString& hoppingColumn, + TExprContext& ctx) +{ + const auto pos = aggregate.Pos(); + const auto aggregateHandlers = aggregate.Handlers(); + + const auto finishKeyArg = Build(ctx, pos).Name("key").Done(); + const auto finishStateArg = Build(ctx, pos).Name("state").Done(); + const auto finishTimeArg = Build(ctx, pos).Name("time").Done(); + + TVector structItems; + structItems.reserve(actualGroupKeys.size() + aggregateHandlers.Size() + 1); + + if (actualGroupKeys.size() == 1) { + structItems.push_back(Build(ctx, pos) + .Name().Build(actualGroupKeys[0]) + .Value(finishKeyArg) + .Done()); + } else { + for (size_t i = 0; i < actualGroupKeys.size(); ++i) { + structItems.push_back(Build(ctx, pos) + .Name().Build(actualGroupKeys[i]) + .Value() + .Tuple(finishKeyArg) + .Index() + .Value(ToString(i)) + .Build() + .Build() + .Done()); + } + } + + for (const auto& handler : aggregateHandlers) { + const auto tuple = handler.Cast(); + const TString compoundColumnName = BuildColumnName(tuple.ColumnName()); + + const auto member = Build(ctx, pos) + .Struct(finishStateArg) + .Name().Build(compoundColumnName) + .Done(); + + if (tuple.ColumnName().Maybe()) { + structItems.push_back(Build(ctx, pos) + .Name().Build(compoundColumnName) + .Value() + .Apply(tuple.Trait().Cast().FinishHandler()) + .With(0, member) + .Build() + .Done()); + + continue; + } + + if (const auto namesList = tuple.ColumnName().Maybe()) { + const auto expApplier = Build(ctx, pos) + .Apply(tuple.Trait().Cast().FinishHandler()) + .With(0, member) + .Done(); + + int index = 0; + for (const auto columnName : namesList.Cast()) { + const auto extracter = Build(ctx, pos) + .Tuple(expApplier) + .Index().Build(index++) + .Done(); + + structItems.push_back(Build(ctx, pos) + .Name(columnName) + .Value(extracter) + .Done()); + } + + continue; + } + + YQL_ENSURE(false, "Invalid node. Expected Atom or AtomList, but received: " + << tuple.ColumnName().Ptr()->Dump()); + } + + structItems.push_back(Build(ctx, pos) + .Name().Build(hoppingColumn) + .Value(finishTimeArg) + .Done()); + + return Build(ctx, pos) + .Args({finishKeyArg, finishStateArg, finishTimeArg}) + .Body() + .Add(structItems) + .Build() + .Done() + .Ptr(); +} + +TExprNode::TPtr BuildSaveHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { + const auto pos = aggregate.Pos(); + const auto aggregateHandlers = aggregate.Handlers(); + + const auto saveStateArg = Build(ctx, pos).Name("state").Done(); + + TVector structItems; + structItems.reserve(aggregateHandlers.Size()); + + for (const auto& handler : aggregateHandlers) { + const auto tuple = handler.Cast(); + const TString columnName = BuildColumnName(tuple.ColumnName()); + + const auto member = Build(ctx, pos) + .Struct(saveStateArg) + .Name().Build(columnName) + .Done(); + + structItems.push_back(Build(ctx, pos) + .Name().Build(columnName) + .Value() + .Apply(tuple.Trait().Cast().SaveHandler()) + .With(0, member) + .Build() + .Done()); + } + + return Build(ctx, pos) + .Args({saveStateArg}) + .Body() + .Add(structItems) + .Build() + .Done() + .Ptr(); +} + +TExprNode::TPtr BuildLoadHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { + const auto pos = aggregate.Pos(); + const auto aggregateHandlers = aggregate.Handlers(); + + TCoArgument loadStateArg = Build(ctx, pos).Name("state").Done(); + + TVector structItems; + structItems.reserve(aggregateHandlers.Size()); + + for (const auto& handler : aggregateHandlers) { + const auto tuple = handler.Cast(); + const TString columnName = BuildColumnName(tuple.ColumnName()); + + const auto member = Build(ctx, pos) + .Struct(loadStateArg) + .Name().Build(columnName) + .Done(); + + structItems.push_back(Build(ctx, pos) + .Name().Build(columnName) + .Value() + .Apply(tuple.Trait().Cast().LoadHandler()) + .With(0, member) + .Build() + .Done()); + } + + return Build(ctx, pos) + .Args({loadStateArg}) + .Body() + .Add(structItems) + .Build() + .Done() + .Ptr(); +} + +TMaybe BuildWatermarkMode( + const TCoAggregate& aggregate, + const TCoHoppingTraits& hoppingTraits, + TExprContext& ctx, + bool analyticsMode, + bool defaultWatermarksMode, + bool syncActor) +{ + const bool enableWatermarks = !analyticsMode && + defaultWatermarksMode && + hoppingTraits.Version().Cast().StringValue() == "v2"; + if (enableWatermarks && syncActor) { + ctx.AddError(TIssue(ctx.GetPosition(aggregate.Pos()), "Watermarks should be used only with async compute actor")); + return Nothing(); + } + + if (hoppingTraits.Version().Cast().StringValue() == "v2" && !enableWatermarks) { + ctx.AddError(TIssue( + ctx.GetPosition(aggregate.Pos()), + "HoppingWindow requires watermarks to be enabled. If you don't want to do that, you can use HOP instead.")); + return Nothing(); + } + + return enableWatermarks; +} + +TMaybeNode RewriteAsHoppingWindow( + const TExprBase node, + TExprContext& ctx, + const TDqConnection& input, + bool analyticsMode, + TDuration lateArrivalDelay, + bool defaultWatermarksMode, + bool syncActor) { + const auto aggregate = node.Cast(); + const auto pos = aggregate.Pos(); + + YQL_CLOG(DEBUG, ProviderDq) << "OptimizeStreamingAggregate"; + + EnsureNotDistinct(aggregate); + + const auto maybeHopTraits = ExtractHopTraits(aggregate, ctx, analyticsMode); + if (!maybeHopTraits) { + return nullptr; + } + const auto hopTraits = *maybeHopTraits; + + const auto aggregateInputType = GetSeqItemType(*node.Ptr()->Head().GetTypeAnn()).Cast(); + TKeysDescription keysDescription(*aggregateInputType, aggregate.Keys(), hopTraits.Column); + + if (keysDescription.NeedPickle()) { + return Build(ctx, pos) + .Lambda(keysDescription.BuildUnpickleLambda(ctx, pos, *aggregateInputType)) + .Input() + .InitFrom(aggregate) + .Input() + .Lambda(keysDescription.BuildPickleLambda(ctx, pos)) + .Input(input) + .Build() + .Build() + .Done(); + } + + const auto keyLambda = keysDescription.GetKeySelector(ctx, pos, aggregateInputType); + const auto timeExtractorLambda = BuildTimeExtractor(hopTraits.Traits, ctx); + const auto initLambda = BuildInitHopLambda(aggregate, ctx); + const auto updateLambda = BuildUpdateHopLambda(aggregate, ctx); + const auto saveLambda = BuildSaveHopLambda(aggregate, ctx); + const auto loadLambda = BuildLoadHopLambda(aggregate, ctx); + const auto mergeLambda = BuildMergeHopLambda(aggregate, ctx); + const auto finishLambda = BuildFinishHopLambda(aggregate, keysDescription.GetActualGroupKeys(), hopTraits.Column, ctx); + const auto enableWatermarks = BuildWatermarkMode(aggregate, hopTraits.Traits, ctx, analyticsMode, defaultWatermarksMode, syncActor); + if (!enableWatermarks) { + return nullptr; + } + + const auto streamArg = Build(ctx, pos).Name("stream").Done(); + auto multiHoppingCoreBuilder = Build(ctx, pos) + .KeyExtractor(keyLambda) + .TimeExtractor(timeExtractorLambda) + .Hop(hopTraits.Traits.Hop()) + .Interval(hopTraits.Traits.Interval()) + .DataWatermarks(hopTraits.Traits.DataWatermarks()) + .InitHandler(initLambda) + .UpdateHandler(updateLambda) + .MergeHandler(mergeLambda) + .FinishHandler(finishLambda) + .SaveHandler(saveLambda) + .LoadHandler(loadLambda) + .template WatermarkMode().Build(ToString(*enableWatermarks)); + + if (*enableWatermarks) { + const auto hop = TDuration::MicroSeconds(hopTraits.Hop); + multiHoppingCoreBuilder.template Delay() + .Literal().Build(ToString(Max(hop, lateArrivalDelay).MicroSeconds())) + .Build(); + } else { + multiHoppingCoreBuilder.Delay(hopTraits.Traits.Delay()); + } + + if (analyticsMode) { + return Build(ctx, node.Pos()) + .Input(input.Ptr()) + .KeySelectorLambda(keyLambda) + .SortDirections() + .Literal() + .Value("true") + .Build() + .Build() + .SortKeySelectorLambda(timeExtractorLambda) + .ListHandlerLambda() + .Args(streamArg) + .template Body() + .Stream(multiHoppingCoreBuilder + .template Input() + .List(streamArg) + .Build() + .Done()) + .Build() + .Build() + .Done(); + } else { + auto wrappedInput = input.Ptr(); + if (!keysDescription.MemberKeys.empty()) { + // Shuffle input connection by keys + wrappedInput = WrapToShuffle(keysDescription, aggregate, input, ctx); + if (!wrappedInput) { + return nullptr; + } + } + + const auto stage = Build(ctx, node.Pos()) + .Inputs() + .Add(wrappedInput) + .Build() + .Program() + .Args(streamArg) + .Body() + .Input(multiHoppingCoreBuilder + .template Input() + .Input(streamArg) + .Build() + .Done()) + .Lambda(keysDescription.BuildUnpickleLambda(ctx, pos, *aggregateInputType)) + .Build() + .Build() + .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) + .Done(); + + return Build(ctx, node.Pos()) + .Output() + .Stage(stage) + .Index().Build(0) + .Build() + .Done(); + } +} + + +} // NYql::NDq::NHopping diff --git a/ydb/library/yql/dq/opt/dq_opt_hopping.h b/ydb/library/yql/dq/opt/dq_opt_hopping.h new file mode 100644 index 000000000000..7d690f6ab2fa --- /dev/null +++ b/ydb/library/yql/dq/opt/dq_opt_hopping.h @@ -0,0 +1,18 @@ +#pragma once + +#include + +#include + +namespace NYql::NDq::NHopping { + +NNodes::TMaybeNode RewriteAsHoppingWindow( + const NNodes::TExprBase node, + TExprContext& ctx, + const NNodes::TDqConnection& input, + bool analyticsHopping, + TDuration lateArrivalDelay, + bool defaultWatermarksMode, + bool syncActor); + +} // namespace NYql::NDq::NHopping diff --git a/ydb/library/yql/dq/opt/dq_opt_peephole.cpp b/ydb/library/yql/dq/opt/dq_opt_peephole.cpp index 6ce322bfc502..6ace1963dfc6 100644 --- a/ydb/library/yql/dq/opt/dq_opt_peephole.cpp +++ b/ydb/library/yql/dq/opt/dq_opt_peephole.cpp @@ -484,8 +484,8 @@ NNodes::TExprBase DqPeepholeRewriteJoinDict(const NNodes::TExprBase& node, TExpr << "(" << *leftKeyType << ") and " << rightKeys[i]->Content() << "(" << *rightKeyType << ")"; break; } - castKeyLeft = (!IsSameAnnotation(*leftDryType, *commonType) || optKeyLeft); - castKeyRight = (!IsSameAnnotation(*rightDryType, *commonType) || optKeyRight); + castKeyLeft = castKeyLeft || (!IsSameAnnotation(*leftDryType, *commonType) || optKeyLeft); + castKeyRight = castKeyRight || (!IsSameAnnotation(*rightDryType, *commonType) || optKeyRight); keyTypeItems.emplace_back(commonType); } diff --git a/ydb/library/yql/dq/opt/ya.make b/ydb/library/yql/dq/opt/ya.make index e15b0112b7ea..999fbec7dc02 100644 --- a/ydb/library/yql/dq/opt/ya.make +++ b/ydb/library/yql/dq/opt/ya.make @@ -15,6 +15,7 @@ SRCS( dq_opt.cpp dq_opt_build.cpp dq_opt_join.cpp + dq_opt_hopping.cpp dq_opt_log.cpp dq_opt_peephole.cpp dq_opt_phy_finalizing.cpp diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp index f2668e993c34..105851fd4e19 100644 --- a/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp +++ b/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp @@ -6,6 +6,7 @@ #include #include +#include #include namespace NKikimr { @@ -345,32 +346,27 @@ class TRangeUnionWrapper : public TMutableComputationNode, p NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const { TUnboxedValueVector mergedLists; auto expandedLists = ExpandLists(ctx); - while (!expandedLists.empty()) { - TMaybe argMin; - bool haveEmpty = false; - for (size_t i = 0; i < expandedLists.size(); ++i) { - if (expandedLists[i].empty()) { - haveEmpty = true; - continue; - } - auto& current = expandedLists[i].front(); - if (!argMin || TypeInfos.front().RangeCompare->Less(current, expandedLists[*argMin].front())) { - argMin = i; - } + + auto comparator = [&](size_t l, size_t r) { return TypeInfos.front().RangeCompare->Less(expandedLists[r].front(), expandedLists[l].front()); }; + std::priority_queue, decltype(comparator)> queue{comparator}; + for (size_t i = 0; i < expandedLists.size(); ++i) { + if (!expandedLists[i].empty()) { + queue.push(i); } - if (argMin) { - auto& from = expandedLists[*argMin]; - if (!RangeIsEmpty(ExpandRange(from.front()), TypeInfos.front())) { - mergedLists.emplace_back(std::move(from.front())); - } - from.pop_front(); - haveEmpty = haveEmpty || from.empty(); + } + + while (!queue.empty()) { + auto argMin = queue.top(); + queue.pop(); + + auto& from = expandedLists[argMin]; + if (!RangeIsEmpty(ExpandRange(from.front()), TypeInfos.front())) { + mergedLists.emplace_back(std::move(from.front())); } + from.pop_front(); - if (haveEmpty) { - expandedLists.erase( - std::remove_if(expandedLists.begin(), expandedLists.end(), [](const auto& list) { return list.empty(); }), - expandedLists.end()); + if (!from.empty()) { + queue.push(argMin); } } diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto index fb845b8cee7b..db2781f6e13f 100644 --- a/ydb/library/yql/providers/common/proto/gateways_config.proto +++ b/ydb/library/yql/providers/common/proto/gateways_config.proto @@ -592,9 +592,11 @@ message TGenericClusterConfig { message TGenericConnectorConfig { // Connector instance network endpoint optional NYql.NConnector.NApi.TEndpoint Endpoint = 3; - // If true, GRPC Client will use TLS encryption. - // Server cert will be verified with system CA cert pool. + // If true, Connector GRPC Client will use TLS encryption. optional bool UseSsl = 4; + // Path to the custom CA certificate to verify Connector's certs. + // If empty, the default system CA certificate pool will be used. + optional string SslCaCrt = 5; reserved 1, 2; } diff --git a/ydb/library/yql/providers/dq/common/ya.make b/ydb/library/yql/providers/dq/common/ya.make index 82704ed75da4..b5a953a629ca 100644 --- a/ydb/library/yql/providers/dq/common/ya.make +++ b/ydb/library/yql/providers/dq/common/ya.make @@ -8,6 +8,7 @@ PEERDIR( ydb/library/yql/utils/log ydb/library/yql/dq/actors ydb/library/yql/dq/proto + ydb/library/yql/dq/integration ) GENERATE_ENUM_SERIALIZATION(yql_dq_settings.h) diff --git a/ydb/library/yql/providers/dq/counters/counters.h b/ydb/library/yql/providers/dq/counters/counters.h index d99722fae175..c7edf1cfa038 100644 --- a/ydb/library/yql/providers/dq/counters/counters.h +++ b/ydb/library/yql/providers/dq/counters/counters.h @@ -61,6 +61,10 @@ struct TCounters { Counters[name] = TEntry(value); } + void SetTimeCounter(const TString& name, i64 value) const { + SetCounter(name, value * 1000); // ms => us + } + THashMap& GetHistogram(const TString& name) { return Histograms[name]; } diff --git a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp index 8743ccb9524d..afb98f8a2c54 100644 --- a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp +++ b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -22,33 +23,6 @@ using namespace NYql; using namespace NYql::NDq; using namespace NYql::NNodes; -namespace { - - -TString BuildColumnName(const TExprBase column) { - if (const auto columnName = column.Maybe()) { - return columnName.Cast().StringValue(); - } - - if (const auto columnNames = column.Maybe()) { - TStringBuilder columnNameBuilder; - for (const auto columnName : columnNames.Cast()) { - columnNameBuilder.append(columnName.StringValue()); - columnNameBuilder.append("_"); - } - return columnNameBuilder; - } - - YQL_ENSURE(false, "Invalid node. Expected Atom or AtomList, but received: " - << column.Ptr()->Dump()); -} - -bool IsLegacyHopping(const TExprNode::TPtr& hoppingSetting) { - return !hoppingSetting->Child(1)->IsList(); -} - -} - class TDqsLogicalOptProposalTransformer : public TOptimizeTransformerBase { public: TDqsLogicalOptProposalTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config) @@ -139,7 +113,13 @@ class TDqsLogicalOptProposalTransformer : public TOptimizeTransformerBase { auto hopSetting = GetSetting(aggregate.Settings().Ref(), "hopping"); if (input) { if (hopSetting) { - return RewriteAsHoppingWindow(node, ctx, input.Cast()); + bool analyticsHopping = Config->AnalyticsHopping.Get().GetOrElse(false); + const auto lateArrivalDelay = TDuration::MilliSeconds(Config->WatermarksLateArrivalDelayMs + .Get() + .GetOrElse(TDqSettings::TDefault::WatermarksLateArrivalDelayMs)); + bool defaultWatermarksMode = Config->WatermarksMode.Get() == "default"; + bool syncActor = Config->ComputeActorType.Get() != "async"; + return NHopping::RewriteAsHoppingWindow(node, ctx, input.Cast(), analyticsHopping, lateArrivalDelay, defaultWatermarksMode, syncActor); } else { return DqRewriteAggregate(node, ctx, TypesCtx, true, Config->UseAggPhases.Get().GetOrElse(false), Config->UseFinalizeByKey.Get().GetOrElse(false)); } @@ -244,382 +224,6 @@ class TDqsLogicalOptProposalTransformer : public TOptimizeTransformerBase { } private: - TMaybeNode RewriteAsHoppingWindow(const TExprBase node, TExprContext& ctx, const TDqConnection& input) { - const auto aggregate = node.Cast(); - const auto pos = aggregate.Pos(); - - YQL_CLOG(DEBUG, ProviderDq) << "OptimizeStreamingAggregate"; - - EnsureNotDistinct(aggregate); - - const auto maybeHopTraits = ExtractHopTraits(aggregate, ctx); - if (!maybeHopTraits) { - return nullptr; - } - const auto hopTraits = *maybeHopTraits; - - const auto aggregateInputType = GetSeqItemType(*node.Ptr()->Head().GetTypeAnn()).Cast(); - TKeysDescription keysDescription(*aggregateInputType, aggregate.Keys(), hopTraits.Column); - - if (keysDescription.NeedPickle()) { - return Build(ctx, pos) - .Lambda(keysDescription.BuildUnpickleLambda(ctx, pos, *aggregateInputType)) - .Input() - .InitFrom(aggregate) - .Input() - .Lambda(keysDescription.BuildPickleLambda(ctx, pos)) - .Input(input) - .Build() - .Build() - .Done(); - } - - const auto keyLambda = keysDescription.GetKeySelector(ctx, pos, aggregateInputType); - const auto timeExtractorLambda = BuildTimeExtractor(hopTraits.Traits, ctx); - const auto initLambda = BuildInitHopLambda(aggregate, ctx); - const auto updateLambda = BuildUpdateHopLambda(aggregate, ctx); - const auto saveLambda = BuildSaveHopLambda(aggregate, ctx); - const auto loadLambda = BuildLoadHopLambda(aggregate, ctx); - const auto mergeLambda = BuildMergeHopLambda(aggregate, ctx); - const auto finishLambda = BuildFinishHopLambda(aggregate, keysDescription.GetActualGroupKeys(), hopTraits.Column, ctx); - const auto enableWatermarks = BuildWatermarkMode(aggregate, hopTraits.Traits, ctx); - if (!enableWatermarks) { - return nullptr; - } - - const auto streamArg = Build(ctx, pos).Name("stream").Done(); - auto multiHoppingCoreBuilder = Build(ctx, pos) - .KeyExtractor(keyLambda) - .TimeExtractor(timeExtractorLambda) - .Hop(hopTraits.Traits.Hop()) - .Interval(hopTraits.Traits.Interval()) - .DataWatermarks(hopTraits.Traits.DataWatermarks()) - .InitHandler(initLambda) - .UpdateHandler(updateLambda) - .MergeHandler(mergeLambda) - .FinishHandler(finishLambda) - .SaveHandler(saveLambda) - .LoadHandler(loadLambda) - .WatermarkMode().Build(ToString(*enableWatermarks)); - - if (*enableWatermarks) { - const auto hop = TDuration::MicroSeconds(hopTraits.Hop); - const auto lateArrivalDelay = TDuration::MilliSeconds(Config->WatermarksLateArrivalDelayMs - .Get() - .GetOrElse(TDqSettings::TDefault::WatermarksLateArrivalDelayMs)); - - multiHoppingCoreBuilder.Delay() - .Literal().Build(ToString(Max(hop, lateArrivalDelay).MicroSeconds())) - .Build(); - } else { - multiHoppingCoreBuilder.Delay(hopTraits.Traits.Delay()); - } - - if (Config->AnalyticsHopping.Get().GetOrElse(false)) { - return Build(ctx, node.Pos()) - .Input(input.Ptr()) - .KeySelectorLambda(keyLambda) - .SortDirections() - .Literal() - .Value("true") - .Build() - .Build() - .SortKeySelectorLambda(timeExtractorLambda) - .ListHandlerLambda() - .Args(streamArg) - .Body() - .Stream(multiHoppingCoreBuilder - .Input() - .List(streamArg) - .Build() - .Done()) - .Build() - .Build() - .Done(); - } else { - auto wrappedInput = input.Ptr(); - if (!keysDescription.MemberKeys.empty()) { - // Shuffle input connection by keys - wrappedInput = WrapToShuffle(keysDescription, aggregate, input, ctx); - if (!wrappedInput) { - return nullptr; - } - } - - const auto stage = Build(ctx, node.Pos()) - .Inputs() - .Add(wrappedInput) - .Build() - .Program() - .Args(streamArg) - .Body() - .Input(multiHoppingCoreBuilder - .Input() - .Input(streamArg) - .Build() - .Done()) - .Lambda(keysDescription.BuildUnpickleLambda(ctx, pos, *aggregateInputType)) - .Build() - .Build() - .Settings(TDqStageSettings().BuildNode(ctx, node.Pos())) - .Done(); - - return Build(ctx, node.Pos()) - .Output() - .Stage(stage) - .Index().Build(0) - .Build() - .Done(); - } - } - - struct THoppingTraits { - TString Column; - TCoHoppingTraits Traits; - ui64 Hop; - ui64 Interval; - ui64 Delay; - }; - - TMaybe ExtractHopTraits(const TCoAggregate& aggregate, TExprContext& ctx) { - const auto pos = aggregate.Pos(); - - const auto hopSetting = GetSetting(aggregate.Settings().Ref(), "hopping"); - if (!hopSetting) { - ctx.AddError(TIssue(ctx.GetPosition(pos), "Aggregate over stream must have 'hopping' setting")); - return Nothing(); - } - - const auto hoppingColumn = IsLegacyHopping(hopSetting) - ? "_yql_time" - : TString(hopSetting->Child(1)->Child(0)->Content()); - - const auto traitsNode = IsLegacyHopping(hopSetting) - ? hopSetting->Child(1) - : hopSetting->Child(1)->Child(1); - - const auto maybeTraits = TMaybeNode(traitsNode); - if (!maybeTraits) { - ctx.AddError(TIssue(ctx.GetPosition(pos), "Invalid 'hopping' setting in Aggregate")); - return Nothing(); - } - - const auto traits = maybeTraits.Cast(); - - const auto checkIntervalParam = [&] (TExprBase param) -> ui64 { - if (param.Maybe()) { - param = param.Cast().Input(); - } - if (!param.Maybe()) { - ctx.AddError(TIssue(ctx.GetPosition(pos), "Not an interval data ctor")); - return 0; - } - auto value = FromString(param.Cast().Literal().Value()); - if (value <= 0) { - ctx.AddError(TIssue(ctx.GetPosition(pos), "Interval value must be positive")); - return 0; - } - return (ui64)value; - }; - - const auto hop = checkIntervalParam(traits.Hop()); - if (!hop) { - return Nothing(); - } - const auto interval = checkIntervalParam(traits.Interval()); - if (!interval) { - return Nothing(); - } - const auto delay = checkIntervalParam(traits.Delay()); - if (!delay) { - return Nothing(); - } - - if (interval < hop) { - ctx.AddError(TIssue(ctx.GetPosition(pos), "Interval must be greater or equal then hop")); - return Nothing(); - } - if (delay < hop) { - ctx.AddError(TIssue(ctx.GetPosition(pos), "Delay must be greater or equal then hop")); - return Nothing(); - } - - const auto newTraits = Build(ctx, aggregate.Pos()) - .InitFrom(traits) - .DataWatermarks(Config->AnalyticsHopping.Get().GetOrElse(false) - ? ctx.NewAtom(aggregate.Pos(), "false") - : traits.DataWatermarks().Ptr()) - .Done(); - - return THoppingTraits { - hoppingColumn, - newTraits, - hop, - interval, - delay - }; - } - - struct TKeysDescription { - TVector PickleKeys; - TVector MemberKeys; - TVector FakeKeys; - - explicit TKeysDescription(const TStructExprType& rowType, const TCoAtomList& keys, const TString& hoppingColumn) { - for (const auto& key : keys) { - if (key.StringValue() == hoppingColumn) { - FakeKeys.emplace_back(key.StringValue()); - continue; - } - - const auto index = rowType.FindItem(key.StringValue()); - Y_ENSURE(index); - - auto itemType = rowType.GetItems()[*index]->GetItemType(); - if (RemoveOptionalType(itemType)->GetKind() == ETypeAnnotationKind::Data) { - MemberKeys.emplace_back(key.StringValue()); - continue; - } - - PickleKeys.emplace_back(key.StringValue()); - } - } - - TExprNode::TPtr BuildPickleLambda(TExprContext& ctx, TPositionHandle pos) const { - TCoArgument arg = Build(ctx, pos) - .Name("item") - .Done(); - - TExprBase body = arg; - - for (const auto& key : PickleKeys) { - const auto member = Build(ctx, pos) - .Name().Build(key) - .Struct(arg) - .Done() - .Ptr(); - - body = Build(ctx, pos) - .Struct(body) - .Name().Build(key) - .Item(ctx.NewCallable(pos, "StablePickle", { member })) - .Done(); - } - - return Build(ctx, pos) - .Args({arg}) - .Body(body) - .Done() - .Ptr(); - } - - TExprNode::TPtr BuildUnpickleLambda(TExprContext& ctx, TPositionHandle pos, const TStructExprType& rowType) { - TCoArgument arg = Build(ctx, pos) - .Name("item") - .Done(); - - TExprBase body = arg; - - for (const auto& key : PickleKeys) { - const auto index = rowType.FindItem(key); - Y_ENSURE(index); - - auto itemType = rowType.GetItems().at(*index)->GetItemType(); - const auto member = Build(ctx, pos) - .Name().Build(key) - .Struct(arg) - .Done() - .Ptr(); - - body = Build(ctx, pos) - .Struct(body) - .Name().Build(key) - .Item(ctx.NewCallable(pos, "Unpickle", { ExpandType(pos, *itemType, ctx), member })) - .Done(); - } - - return Build(ctx, pos) - .Args({arg}) - .Body(body) - .Done() - .Ptr(); - } - - TVector GetKeysList(TExprContext& ctx, TPositionHandle pos) const { - TVector res; - res.reserve(PickleKeys.size() + MemberKeys.size()); - - for (const auto& pickleKey : PickleKeys) { - res.emplace_back(Build(ctx, pos).Value(pickleKey).Done()); - } - for (const auto& memberKey : MemberKeys) { - res.emplace_back(Build(ctx, pos).Value(memberKey).Done()); - } - return res; - } - - TVector GetActualGroupKeys() { - TVector result; - result.reserve(PickleKeys.size() + MemberKeys.size()); - result.insert(result.end(), PickleKeys.begin(), PickleKeys.end()); - result.insert(result.end(), MemberKeys.begin(), MemberKeys.end()); - return result; - } - - bool NeedPickle() const { - return !PickleKeys.empty(); - } - - TExprNode::TPtr GetKeySelector(TExprContext& ctx, TPositionHandle pos, const TStructExprType* rowType) { - auto builder = Build(ctx, pos); - for (auto key : GetKeysList(ctx, pos)) { - builder.Add(std::move(key)); - } - return BuildKeySelector(pos, *rowType, builder.Build().Value().Ptr(), ctx); - } - }; - - TExprNode::TPtr WrapToShuffle( - const TKeysDescription& keysDescription, - const TCoAggregate& aggregate, - const TDqConnection& input, - TExprContext& ctx) - { - auto pos = aggregate.Pos(); - - TDqStageBase mappedInput = input.Output().Stage(); - if (keysDescription.NeedPickle()) { - mappedInput = Build(ctx, pos) - .Inputs() - .Add() - .Output() - .Stage(input.Output().Stage()) - .Index(input.Output().Index()) - .Build() - .Build() - .Build() - .Program() - .Args({"stream"}) - .Body() - .Input("stream") - .Lambda(keysDescription.BuildPickleLambda(ctx, pos)) - .Build() - .Build() - .Settings(TDqStageSettings().BuildNode(ctx, pos)) - .Done(); - } - - return Build(ctx, pos) - .Output() - .Stage(mappedInput) - .Index().Value("0").Build() - .Build() - .KeyColumns() - .Add(keysDescription.GetKeysList(ctx, pos)) - .Build() - .Done() - .Ptr(); - } void EnsureNotDistinct(const TCoAggregate& aggregate) { const auto& aggregateHandlers = aggregate.Handlers(); @@ -629,358 +233,14 @@ class TDqsLogicalOptProposalTransformer : public TOptimizeTransformerBase { "Distinct is not supported for aggregation with hop"); } - TExprNode::TPtr BuildTimeExtractor(const TCoHoppingTraits& hoppingTraits, TExprContext& ctx) { - const auto pos = hoppingTraits.Pos(); - - if (hoppingTraits.ItemType().Ref().GetTypeAnn()->Cast()->GetType()->Cast()->GetSize() == 0) { - // The case when no fields are used in lambda. F.e. when it has only DependsOn. - return ctx.DeepCopyLambda(hoppingTraits.TimeExtractor().Ref()); - } - - return Build(ctx, pos) - .Args({"item"}) - .Body() - .Apply(hoppingTraits.TimeExtractor()) - .With(0) - .Type(hoppingTraits.ItemType()) - .Value("item") - .Build() - .Build() - .Done() - .Ptr(); - } - - TExprNode::TPtr BuildInitHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { - const auto pos = aggregate.Pos(); - const auto& aggregateHandlers = aggregate.Handlers(); - - const auto initItemArg = Build(ctx, pos).Name("item").Done(); - - TVector structItems; - structItems.reserve(aggregateHandlers.Size()); - - ui32 index = 0; - for (const auto& handler : aggregateHandlers) { - const auto tuple = handler.Cast(); - - TMaybeNode applier; - if (tuple.Trait().Cast().InitHandler().Args().Size() == 1) { - applier = Build(ctx, pos) - .Apply(tuple.Trait().Cast().InitHandler()) - .With(0, initItemArg) - .Done(); - } else { - applier = Build(ctx, pos) - .Apply(tuple.Trait().Cast().InitHandler()) - .With(0, initItemArg) - .With(1) - .Literal().Build(ToString(index)) - .Build() - .Done(); - } - - structItems.push_back(Build(ctx, pos) - .Name().Build(BuildColumnName(tuple.ColumnName())) - .Value(applier) - .Done()); - ++index; + IDqOptimization* GetDqOptCallback(const TExprBase& providerRead) const { + if (providerRead.Ref().ChildrenSize() > 1 && TCoDataSource::Match(providerRead.Ref().Child(1))) { + auto dataSourceName = providerRead.Ref().Child(1)->Child(0)->Content(); + auto datasource = TypesCtx.DataSourceMap.FindPtr(dataSourceName); + YQL_ENSURE(datasource); + return (*datasource)->GetDqOptimization(); } - - return Build(ctx, pos) - .Args({initItemArg}) - .Body() - .Add(structItems) - .Build() - .Done() - .Ptr(); - } - - TExprNode::TPtr BuildUpdateHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { - const auto pos = aggregate.Pos(); - const auto aggregateHandlers = aggregate.Handlers(); - - const auto updateItemArg = Build(ctx, pos).Name("item").Done(); - const auto updateStateArg = Build(ctx, pos).Name("state").Done(); - - TVector structItems; - structItems.reserve(aggregateHandlers.Size()); - - i32 index = 0; - for (const auto& handler : aggregateHandlers) { - const auto tuple = handler.Cast(); - const TString columnName = BuildColumnName(tuple.ColumnName()); - - const auto member = Build(ctx, pos) - .Struct(updateStateArg) - .Name().Build(columnName) - .Done(); - - TMaybeNode applier; - if (tuple.Trait().Cast().UpdateHandler().Args().Size() == 2) { - applier = Build(ctx, pos) - .Apply(tuple.Trait().Cast().UpdateHandler()) - .With(0, updateItemArg) - .With(1, member) - .Done(); - } else { - applier = Build(ctx, pos) - .Apply(tuple.Trait().Cast().UpdateHandler()) - .With(0, updateItemArg) - .With(1, member) - .With(2) - .Literal().Build(ToString(index)) - .Build() - .Done(); - } - - structItems.push_back(Build(ctx, pos) - .Name().Build(columnName) - .Value(applier) - .Done()); - ++index; - } - - return Build(ctx, pos) - .Args({updateItemArg, updateStateArg}) - .Body() - .Add(structItems) - .Build() - .Done() - .Ptr(); - } - - TExprNode::TPtr BuildMergeHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { - const auto pos = aggregate.Pos(); - const auto& aggregateHandlers = aggregate.Handlers(); - - const auto mergeState1Arg = Build(ctx, pos).Name("state1").Done(); - const auto mergeState2Arg = Build(ctx, pos).Name("state2").Done(); - - TVector structItems; - structItems.reserve(aggregateHandlers.Size()); - - for (const auto& handler : aggregateHandlers) { - const auto tuple = handler.Cast(); - const TString columnName = BuildColumnName(tuple.ColumnName()); - - const auto member1 = Build(ctx, pos) - .Struct(mergeState1Arg) - .Name().Build(columnName) - .Done(); - const auto member2 = Build(ctx, pos) - .Struct(mergeState2Arg) - .Name().Build(columnName) - .Done(); - - structItems.push_back(Build(ctx, pos) - .Name().Build(columnName) - .Value() - .Apply(tuple.Trait().Cast().MergeHandler()) - .With(0, member1) - .With(1, member2) - .Build() - .Done()); - } - - return Build(ctx, pos) - .Args({mergeState1Arg, mergeState2Arg}) - .Body() - .Add(structItems) - .Build() - .Done() - .Ptr(); - } - - TExprNode::TPtr BuildFinishHopLambda( - const TCoAggregate& aggregate, - const TVector& actualGroupKeys, - const TString& hoppingColumn, - TExprContext& ctx) - { - const auto pos = aggregate.Pos(); - const auto aggregateHandlers = aggregate.Handlers(); - - const auto finishKeyArg = Build(ctx, pos).Name("key").Done(); - const auto finishStateArg = Build(ctx, pos).Name("state").Done(); - const auto finishTimeArg = Build(ctx, pos).Name("time").Done(); - - TVector structItems; - structItems.reserve(actualGroupKeys.size() + aggregateHandlers.Size() + 1); - - if (actualGroupKeys.size() == 1) { - structItems.push_back(Build(ctx, pos) - .Name().Build(actualGroupKeys[0]) - .Value(finishKeyArg) - .Done()); - } else { - for (size_t i = 0; i < actualGroupKeys.size(); ++i) { - structItems.push_back(Build(ctx, pos) - .Name().Build(actualGroupKeys[i]) - .Value() - .Tuple(finishKeyArg) - .Index() - .Value(ToString(i)) - .Build() - .Build() - .Done()); - } - } - - for (const auto& handler : aggregateHandlers) { - const auto tuple = handler.Cast(); - const TString compoundColumnName = BuildColumnName(tuple.ColumnName()); - - const auto member = Build(ctx, pos) - .Struct(finishStateArg) - .Name().Build(compoundColumnName) - .Done(); - - if (tuple.ColumnName().Maybe()) { - structItems.push_back(Build(ctx, pos) - .Name().Build(compoundColumnName) - .Value() - .Apply(tuple.Trait().Cast().FinishHandler()) - .With(0, member) - .Build() - .Done()); - - continue; - } - - if (const auto namesList = tuple.ColumnName().Maybe()) { - const auto expApplier = Build(ctx, pos) - .Apply(tuple.Trait().Cast().FinishHandler()) - .With(0, member) - .Done(); - - int index = 0; - for (const auto columnName : namesList.Cast()) { - const auto extracter = Build(ctx, pos) - .Tuple(expApplier) - .Index().Build(index++) - .Done(); - - structItems.push_back(Build(ctx, pos) - .Name(columnName) - .Value(extracter) - .Done()); - } - - continue; - } - - YQL_ENSURE(false, "Invalid node. Expected Atom or AtomList, but received: " - << tuple.ColumnName().Ptr()->Dump()); - } - - structItems.push_back(Build(ctx, pos) - .Name().Build(hoppingColumn) - .Value(finishTimeArg) - .Done()); - - return Build(ctx, pos) - .Args({finishKeyArg, finishStateArg, finishTimeArg}) - .Body() - .Add(structItems) - .Build() - .Done() - .Ptr(); - } - - TExprNode::TPtr BuildSaveHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { - const auto pos = aggregate.Pos(); - const auto aggregateHandlers = aggregate.Handlers(); - - const auto saveStateArg = Build(ctx, pos).Name("state").Done(); - - TVector structItems; - structItems.reserve(aggregateHandlers.Size()); - - for (const auto& handler : aggregateHandlers) { - const auto tuple = handler.Cast(); - const TString columnName = BuildColumnName(tuple.ColumnName()); - - const auto member = Build(ctx, pos) - .Struct(saveStateArg) - .Name().Build(columnName) - .Done(); - - structItems.push_back(Build(ctx, pos) - .Name().Build(columnName) - .Value() - .Apply(tuple.Trait().Cast().SaveHandler()) - .With(0, member) - .Build() - .Done()); - } - - return Build(ctx, pos) - .Args({saveStateArg}) - .Body() - .Add(structItems) - .Build() - .Done() - .Ptr(); - } - - TExprNode::TPtr BuildLoadHopLambda(const TCoAggregate& aggregate, TExprContext& ctx) { - const auto pos = aggregate.Pos(); - const auto aggregateHandlers = aggregate.Handlers(); - - TCoArgument loadStateArg = Build(ctx, pos).Name("state").Done(); - - TVector structItems; - structItems.reserve(aggregateHandlers.Size()); - - for (const auto& handler : aggregateHandlers) { - const auto tuple = handler.Cast(); - const TString columnName = BuildColumnName(tuple.ColumnName()); - - const auto member = Build(ctx, pos) - .Struct(loadStateArg) - .Name().Build(columnName) - .Done(); - - structItems.push_back(Build(ctx, pos) - .Name().Build(columnName) - .Value() - .Apply(tuple.Trait().Cast().LoadHandler()) - .With(0, member) - .Build() - .Done()); - } - - return Build(ctx, pos) - .Args({loadStateArg}) - .Body() - .Add(structItems) - .Build() - .Done() - .Ptr(); - } - - TMaybe BuildWatermarkMode( - const TCoAggregate& aggregate, - const TCoHoppingTraits& hoppingTraits, - TExprContext& ctx) - { - const auto analyticsMode = Config->AnalyticsHopping.Get().GetOrElse(false); - const bool enableWatermarks = !analyticsMode && - Config->WatermarksMode.Get() == "default" && - hoppingTraits.Version().Cast().StringValue() == "v2"; - if (enableWatermarks && Config->ComputeActorType.Get() != "async") { - ctx.AddError(TIssue(ctx.GetPosition(aggregate.Pos()), "Watermarks should be used only with async compute actor")); - return Nothing(); - } - - if (hoppingTraits.Version().Cast().StringValue() == "v2" && !enableWatermarks) { - ctx.AddError(TIssue( - ctx.GetPosition(aggregate.Pos()), - "HoppingWindow requires watermarks to be enabled. If you don't want to do that, you can use HOP instead.")); - return Nothing(); - } - - return enableWatermarks; + return nullptr; } private: diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp index 1d9e2351bb8f..067c839543e9 100644 --- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp +++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp @@ -305,7 +305,7 @@ struct TDqsPipelineConfigurator : public IPipelineConfigurator { std::unordered_set UniqIntegrations_; }; -TExprNode::TPtr DqMarkBlockStage(const TDqPhyStage& stage, TExprContext& ctx) { +TExprNode::TPtr DqMarkBlockStage(const TDqStatePtr& state, const TPublicIds::TPtr& publicIds, const TDqPhyStage& stage, TExprContext& ctx) { using NDq::TDqStageSettings; TDqStageSettings settings = NDq::TDqStageSettings::Parse(stage); if (settings.BlockStatus.Defined()) { @@ -359,7 +359,20 @@ TExprNode::TPtr DqMarkBlockStage(const TDqPhyStage& stage, TExprContext& ctx) { return true; }); - YQL_CLOG(INFO, CoreDq) << "Setting block status for stage #" << settings.LogicalId << " = " << ToString(blockStatus); + auto publicId = publicIds->Stage2publicId.find(settings.LogicalId); + TString publicIdMsg; + if (publicId != publicIds->Stage2publicId.end()) { + publicIdMsg = TStringBuilder() << " (public id #" << publicId->second << ")"; + auto p = TOperationProgress(TString(DqProviderName), publicId->second, TOperationProgress::EState::InProgress); + switch (blockStatus) { + case TDqStageSettings::EBlockStatus::None: p.BlockStatus = TOperationProgress::EOpBlockStatus::None; break; + case TDqStageSettings::EBlockStatus::Partial: p.BlockStatus = TOperationProgress::EOpBlockStatus::Partial; break; + case TDqStageSettings::EBlockStatus::Full: p.BlockStatus = TOperationProgress::EOpBlockStatus::Full; break; + } + state->ProgressWriter(p); + } + + YQL_CLOG(INFO, CoreDq) << "Setting block status for stage #" << settings.LogicalId << publicIdMsg << " to " << ToString(blockStatus); return Build(ctx, stage.Pos()) .InitFrom(stage) .Settings(settings.SetBlockStatus(blockStatus).BuildNode(ctx, stage.Settings().Pos())) @@ -368,8 +381,9 @@ TExprNode::TPtr DqMarkBlockStage(const TDqPhyStage& stage, TExprContext& ctx) { struct TDqsFinalPipelineConfigurator : public IPipelineConfigurator { public: - explicit TDqsFinalPipelineConfigurator(const TDqStatePtr& state) + explicit TDqsFinalPipelineConfigurator(const TDqStatePtr& state, const TPublicIds::TPtr& publicIds) : State_(state) + , PublicIds_(publicIds) { } private: @@ -384,13 +398,13 @@ struct TDqsFinalPipelineConfigurator : public IPipelineConfigurator { "DqBuildWideBlockChannels", TIssuesIds::DEFAULT_ERROR); pipeline->Add(CreateFunctorTransformer( - [typeCtx](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { + [typeCtx, state = State_, publicIds = PublicIds_](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) { TOptimizeExprSettings optSettings{typeCtx.Get()}; optSettings.VisitLambdas = false; - return OptimizeExpr(input, output, [](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { + return OptimizeExpr(input, output, [state, publicIds](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr { TExprBase expr{node}; if (auto stage = expr.Maybe()) { - return DqMarkBlockStage(stage.Cast(), ctx); + return DqMarkBlockStage(state, publicIds, stage.Cast(), ctx); } return node; }, ctx, optSettings); @@ -399,7 +413,8 @@ struct TDqsFinalPipelineConfigurator : public IPipelineConfigurator { "DqMarkBlockStages", TIssuesIds::DEFAULT_ERROR); } - TDqStatePtr State_; + const TDqStatePtr State_; + const TPublicIds::TPtr PublicIds_; }; class TSimpleSkiffConverter : public ISkiffConverter { @@ -884,11 +899,12 @@ class TDqExecTransformer: public TExecTransformerBase, TCounters settings->_AllResultsBytesLimit = 64_MB; } + TPublicIds::TPtr publicIds = std::make_shared(); int level; TExprNode::TPtr resInput = WrapLambdaBody(level, result.Input().Ptr(), ctx); { auto block = MeasureBlock("PeepHole"); - if (const auto status = PeepHole(resInput, resInput, ctx, resSettings); status.Level != TStatus::Ok) { + if (const auto status = PeepHole(resInput, resInput, ctx, resSettings, publicIds); status.Level != TStatus::Ok) { return SyncStatus(status); } } @@ -903,7 +919,6 @@ class TDqExecTransformer: public TExecTransformerBase, TCounters TVector columns; GetResultType(&type, &columns, result.Ref(), result.Input().Ref()); - TPublicIds::TPtr publicIds = std::make_shared(); VisitExpr(result.Ptr(), [&](const TExprNode::TPtr& node) { const TExprBase expr(node); if (expr.Maybe()) { @@ -1281,7 +1296,7 @@ class TDqExecTransformer: public TExecTransformerBase, TCounters optimizedInput->SetTypeAnn(pull.Input().Ref().GetTypeAnn()); optimizedInput->CopyConstraints(pull.Input().Ref()); - auto status = PeepHole(optimizedInput, optimizedInput, ctx, pullSettings); + auto status = PeepHole(optimizedInput, optimizedInput, ctx, pullSettings, publicIds); if (status.Level != TStatus::Ok) { return SyncStatus(status); } @@ -1813,7 +1828,7 @@ class TDqExecTransformer: public TExecTransformerBase, TCounters auto optimizedInput = input; optimizedInput->SetState(TExprNode::EState::ConstrComplete); - auto status = PeepHole(optimizedInput, optimizedInput, ctx, providerParams); + auto status = PeepHole(optimizedInput, optimizedInput, ctx, providerParams, publicIds); if (status.Level != TStatus::Ok) { return combinedStatus.Combine(status); } @@ -2033,9 +2048,11 @@ class TDqExecTransformer: public TExecTransformerBase, TCounters return combinedStatus; } - IGraphTransformer::TStatus PeepHole(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx, const THashMap& providerParams) const { + IGraphTransformer::TStatus PeepHole(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx, + const THashMap& providerParams, const TPublicIds::TPtr& publicIds) const + { TDqsPipelineConfigurator peepholeConfig(State, providerParams); - TDqsFinalPipelineConfigurator finalPeepholeConfg(State); + TDqsFinalPipelineConfigurator finalPeepholeConfg(State, publicIds); TPeepholeSettings peepholeSettings; peepholeSettings.CommonConfig = &peepholeConfig; peepholeSettings.FinalConfig = &finalPeepholeConfg; diff --git a/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp b/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp index e92dc952a3d9..37d899baae6d 100644 --- a/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp +++ b/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp @@ -127,13 +127,11 @@ class TTaskCommandExecutor { "TaskRunner", labels, name); - auto& old = CurrentJobStats[counterName]; if (name.EndsWith("Time")) { - QueryStat.AddTimeCounter(counterName, value - old); + QueryStat.SetTimeCounter(counterName, value); } else { - QueryStat.AddCounter(counterName, value - old); + QueryStat.SetCounter(counterName, value); } - old = value; } }); } @@ -766,7 +764,6 @@ class TTaskCommandExecutor { std::unique_ptr Alloc; NKikimr::NMiniKQL::TComputationNodeFactory ComputationFactory; TTaskTransformFactory TaskTransformFactory; - THashMap CurrentJobStats; NKikimr::NMiniKQL::IStatsRegistry* JobStats; bool TerminateOnError; TIntrusivePtr Runner; diff --git a/ydb/library/yql/providers/generic/connector/libcpp/client.cpp b/ydb/library/yql/providers/generic/connector/libcpp/client.cpp index 9d6237808377..e0fe558b7ae1 100644 --- a/ydb/library/yql/providers/generic/connector/libcpp/client.cpp +++ b/ydb/library/yql/providers/generic/connector/libcpp/client.cpp @@ -1,3 +1,5 @@ +#include + #include "client.h" namespace NYql::NConnector { @@ -21,10 +23,18 @@ namespace NYql::NConnector { public: TClientGRPC() = delete; TClientGRPC(const TGenericConnectorConfig& config) { - TString endpoint = TStringBuilder() << config.GetEndpoint().host() << ":" << ToString(config.GetEndpoint().port()); - GrpcConfig_ = NYdbGrpc::TGRpcClientConfig(endpoint); + GrpcConfig_ = NYdbGrpc::TGRpcClientConfig(); + GrpcConfig_.Locator = TStringBuilder() << config.GetEndpoint().host() << ":" << config.GetEndpoint().port(); GrpcConfig_.EnableSsl = config.GetUseSsl(); + // Read content of CA cert + TString rootCertData; + if (config.GetSslCaCrt()) { + rootCertData = TFileInput(config.GetSslCaCrt()).ReadAll(); + } + + GrpcConfig_.SslCredentials = grpc::SslCredentialsOptions{.pem_root_certs = rootCertData, .pem_private_key = "", .pem_cert_chain = ""}; + GrpcClient_ = std::make_unique(); // FIXME: is it OK to use single connection during the client lifetime? diff --git a/ydb/library/yql/providers/yt/comp_nodes/yql_mkql_file_input_state.h b/ydb/library/yql/providers/yt/comp_nodes/yql_mkql_file_input_state.h index e83859abe65e..e20261562834 100644 --- a/ydb/library/yql/providers/yt/comp_nodes/yql_mkql_file_input_state.h +++ b/ydb/library/yql/providers/yt/comp_nodes/yql_mkql_file_input_state.h @@ -60,12 +60,12 @@ class TFileInputState: public IInputState { TVector RawReaders_; const size_t BlockCount_; const size_t BlockSize_; - TMkqlReaderImpl MkqlReader_; size_t CurrentInput_ = 0; size_t CurrentRecord_ = 0; bool Valid_ = true; NUdf::TUnboxedValue CurrentValue_; NYT::TRawTableReaderPtr CurrentReader_; + TMkqlReaderImpl MkqlReader_; // Should be deleted before CurrentReader_ std::function OnNextBlockCallback_; }; diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h index c49327b3308d..e0245ed911dc 100644 --- a/ydb/library/yql/sql/v1/node.h +++ b/ydb/library/yql/sql/v1/node.h @@ -500,8 +500,12 @@ namespace NSQLTranslationV1 { TAstNode* Translate(TContext& ctx) const override; }; + enum class ESampleClause { + TableSample, //from SQL standard, percantage rate (0..100) + Sample //simplified (implied Bernulli mode), fraction (0..1) + }; + enum class ESampleMode { - Auto, Bernoulli, System }; diff --git a/ydb/library/yql/sql/v1/select.cpp b/ydb/library/yql/sql/v1/select.cpp index 4d2acfc8d000..1932643ed002 100644 --- a/ydb/library/yql/sql/v1/select.cpp +++ b/ydb/library/yql/sql/v1/select.cpp @@ -605,6 +605,7 @@ class TSubqueryRefNode: public IRealSource { bool SetSamplingOptions( TContext& ctx, TPosition pos, + ESampleClause sampleClause, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed) override { @@ -616,7 +617,7 @@ class TSubqueryRefNode: public IRealSource { ctx.Error(pos) << "'Repeatable' keyword is not supported for subqueries"; return false; } - return SetSamplingRate(ctx, samplingRate); + return SetSamplingRate(ctx, sampleClause, samplingRate); } bool IsStream() const override { @@ -708,40 +709,33 @@ class TTableSource: public IRealSource { bool SetSamplingOptions( TContext& ctx, TPosition pos, + ESampleClause sampleClause, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed) override { + Y_UNUSED(pos); TString modeName; if (!samplingSeed) { samplingSeed = Y("Int32", Q("0")); } - + if (ESampleClause::Sample == sampleClause) { + YQL_ENSURE(ESampleMode::Bernoulli == mode, "Internal logic error"); + } switch (mode) { - case ESampleMode::Auto: - modeName = "bernoulli"; - samplingRate = Y("*", samplingRate, Y("Double", Q("100"))); - break; - case ESampleMode::Bernoulli: - modeName = "bernoulli"; - break; - case ESampleMode::System: - modeName = "system"; - break; + case ESampleMode::Bernoulli: + modeName = "bernoulli"; + break; + case ESampleMode::System: + modeName = "system"; + break; } if (!samplingRate->Init(ctx, FakeSource.Get())) { return false; } - auto ensureLow = Y("Ensure", "samplingRate", Y(">=", "samplingRate", Y("Double", Q("0"))), Y("String", BuildQuotedAtom(pos, "Expected sampling rate to be nonnegative"))); - auto ensureHigh = Y("Ensure", "samplingRate", Y("<=", "samplingRate", Y("Double", Q("100"))), Y("String", BuildQuotedAtom(pos, "Sampling rate is over 100%"))); - - auto block(Y(Y("let", "samplingRate", samplingRate))); - block = L(block, Y("let", "samplingRate", ensureLow)); - block = L(block, Y("let", "samplingRate", ensureHigh)); - - samplingRate = Y("block", Q(L(block, Y("return", "samplingRate")))); + samplingRate = PrepareSamplingRate(pos, sampleClause, samplingRate, true); auto sampleSettings = Q(Y(Q(modeName), Y("EvaluateAtom", Y("ToString", samplingRate)), Y("EvaluateAtom", Y("ToString", samplingSeed)))); auto sampleOption = Q(Y(Q("sample"), sampleSettings)); @@ -807,9 +801,10 @@ class TInnerSource: public IProxySource { SetLabel(label); } - bool SetSamplingOptions(TContext& ctx, TPosition pos, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed) override { + bool SetSamplingOptions(TContext& ctx, TPosition pos, ESampleClause sampleClause, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed) override { Y_UNUSED(ctx); SamplingPos = pos; + SamplingClause = sampleClause; SamplingMode = mode; SamplingRate = samplingRate; SamplingSeed = samplingSeed; @@ -858,7 +853,7 @@ class TInnerSource: public IProxySource { } if (SamplingPos) { - if (!source->SetSamplingOptions(ctx, *SamplingPos, SamplingMode, SamplingRate, SamplingSeed)) { + if (!source->SetSamplingOptions(ctx, *SamplingPos, SamplingClause, SamplingMode, SamplingRate, SamplingSeed)) { return false; } } @@ -918,7 +913,8 @@ class TInnerSource: public IProxySource { private: TMaybe SamplingPos; - ESampleMode SamplingMode = ESampleMode::Auto; + ESampleClause SamplingClause; + ESampleMode SamplingMode; TNodePtr SamplingRate; TNodePtr SamplingSeed; @@ -2963,6 +2959,7 @@ class TSelect: public IProxySource { bool SetSamplingOptions( TContext& ctx, TPosition pos, + ESampleClause sampleClause, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed) override { @@ -2974,7 +2971,7 @@ class TSelect: public IProxySource { ctx.Error(pos) << "'Repeatable' keyword is not supported for subqueries"; return false; } - return SetSamplingRate(ctx, samplingRate); + return SetSamplingRate(ctx, sampleClause, samplingRate); } bool IsSelect() const override { diff --git a/ydb/library/yql/sql/v1/source.cpp b/ydb/library/yql/sql/v1/source.cpp index 1ee414400c1a..a7d8dfe625d1 100644 --- a/ydb/library/yql/sql/v1/source.cpp +++ b/ydb/library/yql/sql/v1/source.cpp @@ -394,12 +394,38 @@ TWriteSettings ISource::GetWriteSettings() const { return {}; } +TNodePtr ISource::PrepareSamplingRate(TPosition pos, ESampleClause clause, TNodePtr samplingRate, bool tempUseBlockExpr) { + if (ESampleClause::Sample == clause) { + samplingRate = Y("*", samplingRate, Y("Double", Q("100"))); + } + //TODO YQL-17526 temp for minimizing diff in canonized tests + if (tempUseBlockExpr) { + auto ensureLow = Y("Ensure", "samplingRate", Y(">=", "samplingRate", Y("Double", Q("0"))), Y("String", BuildQuotedAtom(pos, "Expected sampling rate to be nonnegative"))); + auto ensureHigh = Y("Ensure", "samplingRate", Y("<=", "samplingRate", Y("Double", Q("100"))), Y("String", BuildQuotedAtom(pos, "Sampling rate is over 100%"))); + + auto block(Y(Y("let", "samplingRate", samplingRate))); + block = L(block, Y("let", "samplingRate", ensureLow)); + block = L(block, Y("let", "samplingRate", ensureHigh)); + + samplingRate = Y("block", Q(L(block, Y("return", "samplingRate")))); + } else { + samplingRate = Y("Ensure", samplingRate, Y(">=", samplingRate, Y("Double", Q("0"))), + Y("String", Q("\"Expected sampling rate to be nonnegative\""))); + samplingRate = Y("Ensure", samplingRate, Y("<=", samplingRate, Y("Double", Q("100"))), + Y("String", Q("\"Sampling rate is over 100%\""))); + } + return samplingRate; +} + + bool ISource::SetSamplingOptions(TContext& ctx, TPosition pos, + ESampleClause sampleClause, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed) { Y_UNUSED(pos); + Y_UNUSED(sampleClause); Y_UNUSED(mode); Y_UNUSED(samplingRate); Y_UNUSED(samplingSeed); @@ -529,13 +555,12 @@ bool ISource::BuildSamplingLambda(TNodePtr& node) { return !!node; } -bool ISource::SetSamplingRate(TContext& ctx, TNodePtr samplingRate) { +bool ISource::SetSamplingRate(TContext& ctx, ESampleClause clause, TNodePtr samplingRate) { if (samplingRate) { if (!samplingRate->Init(ctx, this)) { return false; } - SamplingRate = Y("Ensure", samplingRate, Y(">=", samplingRate, Y("Double", Q("0"))), Y("String", Q("\"Expected sampling rate to be nonnegative\""))); - SamplingRate = Y("Ensure", SamplingRate, Y("<=", SamplingRate, Y("Double", Q("100"))), Y("String", Q("\"Sampling rate is over 100%\""))); + SamplingRate = PrepareSamplingRate(Pos, clause, samplingRate, false); } return true; } diff --git a/ydb/library/yql/sql/v1/source.h b/ydb/library/yql/sql/v1/source.h index f1a91dec12f0..58f04284b740 100644 --- a/ydb/library/yql/sql/v1/source.h +++ b/ydb/library/yql/sql/v1/source.h @@ -71,7 +71,8 @@ namespace NSQLTranslationV1 { virtual bool IsStream() const; virtual EOrderKind GetOrderKind() const; virtual TWriteSettings GetWriteSettings() const; - virtual bool SetSamplingOptions(TContext& ctx, TPosition pos, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed); + TNodePtr PrepareSamplingRate(TPosition pos, ESampleClause clause, TNodePtr samplingRate, bool tempUseBlockExpr); //TODO fixme YQL-17526 + virtual bool SetSamplingOptions(TContext& ctx, TPosition pos, ESampleClause clause, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed); virtual bool SetTableHints(TContext& ctx, TPosition pos, const TTableHints& hints, const TTableHints& contextHints); virtual bool CalculateGroupingHint(TContext& ctx, const TVector& columns, ui64& hint) const; virtual TNodePtr BuildFilter(TContext& ctx, const TString& label); @@ -86,7 +87,7 @@ namespace NSQLTranslationV1 { virtual TNodePtr BuildSort(TContext& ctx, const TString& label); virtual TNodePtr BuildCleanupColumns(TContext& ctx, const TString& label); virtual bool BuildSamplingLambda(TNodePtr& node); - virtual bool SetSamplingRate(TContext& ctx, TNodePtr samplingRate); + virtual bool SetSamplingRate(TContext& ctx, ESampleClause clause, TNodePtr samplingRate); virtual IJoin* GetJoin(); virtual ISource* GetCompositeSource(); virtual bool IsSelect() const; diff --git a/ydb/library/yql/sql/v1/sql_select.cpp b/ydb/library/yql/sql/v1/sql_select.cpp index 1fb347a88cb3..40041f13f6d2 100644 --- a/ydb/library/yql/sql/v1/sql_select.cpp +++ b/ydb/library/yql/sql/v1/sql_select.cpp @@ -552,7 +552,8 @@ TSourcePtr TSqlSelect::NamedSingleSource(const TRule_named_single_source& node, singleSource->SetLabel(label); } if (node.HasBlock4()) { - ESampleMode mode = ESampleMode::Auto; + ESampleClause sampleClause; + ESampleMode mode; TSqlExpression expr(Ctx, Mode); TNodePtr samplingRateNode; TNodePtr samplingSeedNode; @@ -561,6 +562,8 @@ TSourcePtr TSqlSelect::NamedSingleSource(const TRule_named_single_source& node, switch (sampleBlock.Alt_case()) { case TRule_named_single_source::TBlock4::kAlt1: { + sampleClause = ESampleClause::Sample; + mode = ESampleMode::Bernoulli; const auto& sampleExpr = sampleBlock.GetAlt1().GetRule_sample_clause1().GetRule_expr2(); samplingRateNode = expr.Build(sampleExpr); if (!samplingRateNode) { @@ -572,6 +575,7 @@ TSourcePtr TSqlSelect::NamedSingleSource(const TRule_named_single_source& node, break; case TRule_named_single_source::TBlock4::kAlt2: { + sampleClause = ESampleClause::TableSample; const auto& tableSampleClause = sampleBlock.GetAlt2().GetRule_tablesample_clause1(); const auto& modeToken = tableSampleClause.GetRule_sampling_mode2().GetToken1(); const TCiString& token = Token(modeToken); @@ -603,7 +607,7 @@ TSourcePtr TSqlSelect::NamedSingleSource(const TRule_named_single_source& node, case TRule_named_single_source::TBlock4::ALT_NOT_SET: Y_ABORT("SampleClause: does not corresond to grammar changes"); } - if (!singleSource->SetSamplingOptions(Ctx, pos, mode, samplingRateNode, samplingSeedNode)) { + if (!singleSource->SetSamplingOptions(Ctx, pos, sampleClause, mode, samplingRateNode, samplingSeedNode)) { Ctx.IncrementMonCounter("sql_errors", "IncorrectSampleClause"); return nullptr; } diff --git a/ydb/library/yql/tests/sql/dq_file/part1/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part1/canondata/result.json index 929cc751ac8d..c8ac667b4d22 100644 --- a/ydb/library/yql/tests/sql/dq_file/part1/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part1/canondata/result.json @@ -2139,9 +2139,9 @@ ], "test.test[pg-tpcds-q19-default.txt-Debug]": [ { - "checksum": "fbd511118372d98be6f026eafdfee1ac", - "size": 1251, - "uri": "https://{canondata_backend}/1937367/26ffadbe955b9e88125bb0a27831ce1640a50e2a/resource.tar.gz#test.test_pg-tpcds-q19-default.txt-Debug_/opt.yql_patched" + "checksum": "2975432e3f3a636ba8d371e867810457", + "size": 858, + "uri": "https://{canondata_backend}/212715/1628fc53f3f62f8d00cdc3a2832cfcfea9a015c3/resource.tar.gz#test.test_pg-tpcds-q19-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q19-default.txt-Plan]": [ @@ -2249,9 +2249,9 @@ ], "test.test[pg-tpcds-q95-default.txt-Debug]": [ { - "checksum": "d8dff9d6e720e0d7a4c8f3619689168e", - "size": 3305, - "uri": "https://{canondata_backend}/1924537/23c5fb7a06e21c00a5ca2293cc03bc27b0578850/resource.tar.gz#test.test_pg-tpcds-q95-default.txt-Debug_/opt.yql_patched" + "checksum": "f27adf3fafc1d572c4751abf4ae0ec01", + "size": 3117, + "uri": "https://{canondata_backend}/212715/1628fc53f3f62f8d00cdc3a2832cfcfea9a015c3/resource.tar.gz#test.test_pg-tpcds-q95-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q95-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part10/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part10/canondata/result.json index 7fabcbd11064..4ab0bc2bb32c 100644 --- a/ydb/library/yql/tests/sql/dq_file/part10/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part10/canondata/result.json @@ -1763,6 +1763,28 @@ } ], "test.test[pg-aggregate_combine--Results]": [], + "test.test[pg-order_by_distinct_same_expr_agg_sublink-default.txt-Analyze]": [ + { + "checksum": "c1f2d837c3623c81dd596a9877913fb8", + "size": 948, + "uri": "https://{canondata_backend}/1937027/bae649e6896209dbfb01462c67cd54c0f971d262/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg_sublink-default.txt-Analyze_/plan.txt" + } + ], + "test.test[pg-order_by_distinct_same_expr_agg_sublink-default.txt-Debug]": [ + { + "checksum": "365cedc84ffbf398ca3253a63fe595fa", + "size": 1678, + "uri": "https://{canondata_backend}/1937027/bae649e6896209dbfb01462c67cd54c0f971d262/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg_sublink-default.txt-Debug_/opt.yql_patched" + } + ], + "test.test[pg-order_by_distinct_same_expr_agg_sublink-default.txt-Plan]": [ + { + "checksum": "c1f2d837c3623c81dd596a9877913fb8", + "size": 948, + "uri": "https://{canondata_backend}/1937027/bae649e6896209dbfb01462c67cd54c0f971d262/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg_sublink-default.txt-Plan_/plan.txt" + } + ], + "test.test[pg-order_by_distinct_same_expr_agg_sublink-default.txt-Results]": [], "test.test[pg-select_agg_gs_rollup-default.txt-Analyze]": [ { "checksum": "b4dd508a329723c74293d80f0278c705", @@ -2080,9 +2102,9 @@ ], "test.test[pg-tpcds-q96-default.txt-Debug]": [ { - "checksum": "38c335a8237fa80e161dc20531a772d0", - "size": 1585, - "uri": "https://{canondata_backend}/1600758/c778849b827ee6b7243d05d28b4a3f467d62ba71/resource.tar.gz#test.test_pg-tpcds-q96-default.txt-Debug_/opt.yql_patched" + "checksum": "a6297e77d6e74dbc2df58a9193ac60a7", + "size": 1491, + "uri": "https://{canondata_backend}/1942525/7de1fbc5f1b7918aec7094b41384bc4c27fc7953/resource.tar.gz#test.test_pg-tpcds-q96-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q96-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part11/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part11/canondata/result.json index 84eaa0ec2a97..3aea639b6822 100644 --- a/ydb/library/yql/tests/sql/dq_file/part11/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part11/canondata/result.json @@ -1935,9 +1935,9 @@ ], "test.test[pg-tpcds-q31-default.txt-Debug]": [ { - "checksum": "3ff4c64d5d2d5e107e874990f8b4593d", - "size": 881, - "uri": "https://{canondata_backend}/1936947/a99026e839b7e22714c2a9a81971a3b5e3ed1eb4/resource.tar.gz#test.test_pg-tpcds-q31-default.txt-Debug_/opt.yql_patched" + "checksum": "4b024bbca419b517c370d5aeb9ed9d0f", + "size": 760, + "uri": "https://{canondata_backend}/1889210/a25c62064c6b3aebb0148e4cc231d4df4bb7bd7c/resource.tar.gz#test.test_pg-tpcds-q31-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q31-default.txt-Plan]": [ @@ -1979,9 +1979,9 @@ ], "test.test[pg-tpcds-q71-default.txt-Debug]": [ { - "checksum": "22016c037eee83779d1f190992a53087", - "size": 837, - "uri": "https://{canondata_backend}/1936947/a99026e839b7e22714c2a9a81971a3b5e3ed1eb4/resource.tar.gz#test.test_pg-tpcds-q71-default.txt-Debug_/opt.yql_patched" + "checksum": "3f59ba59811853377a33a7004b8b52b2", + "size": 710, + "uri": "https://{canondata_backend}/1889210/a25c62064c6b3aebb0148e4cc231d4df4bb7bd7c/resource.tar.gz#test.test_pg-tpcds-q71-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q71-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part12/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part12/canondata/result.json index e971cff0f660..6725ec3dc937 100644 --- a/ydb/library/yql/tests/sql/dq_file/part12/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part12/canondata/result.json @@ -2361,9 +2361,9 @@ ], "test.test[pg-tpcds-q03-default.txt-Debug]": [ { - "checksum": "01a13c3a811a8ec79fbcf3fd60235a2a", - "size": 1060, - "uri": "https://{canondata_backend}/1880306/234eadcde1cd54bffae64f4516628981e02b093d/resource.tar.gz#test.test_pg-tpcds-q03-default.txt-Debug_/opt.yql_patched" + "checksum": "fba9e5eba8b28dab0f43a7f3525af85e", + "size": 728, + "uri": "https://{canondata_backend}/1889210/1220fbc43e6c9913dd69b912a91a021b32a209aa/resource.tar.gz#test.test_pg-tpcds-q03-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q03-default.txt-Plan]": [ @@ -2383,9 +2383,9 @@ ], "test.test[pg-tpcds-q14-default.txt-Debug]": [ { - "checksum": "2febf9bad1f9bd61418056d6dcf4320a", - "size": 4022, - "uri": "https://{canondata_backend}/1936997/15647165297f31fc500aaeb8506af785801000d5/resource.tar.gz#test.test_pg-tpcds-q14-default.txt-Debug_/opt.yql_patched" + "checksum": "e18e245d4516f1ada5f9b8095a80dc2a", + "size": 3624, + "uri": "https://{canondata_backend}/1889210/1220fbc43e6c9913dd69b912a91a021b32a209aa/resource.tar.gz#test.test_pg-tpcds-q14-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q14-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part13/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part13/canondata/result.json index ffae0f6a10c0..529b19c314e5 100644 --- a/ydb/library/yql/tests/sql/dq_file/part13/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part13/canondata/result.json @@ -2029,9 +2029,9 @@ ], "test.test[pg-tpcds-q55-default.txt-Debug]": [ { - "checksum": "1449d820faba78f5cf1c88591cf03645", - "size": 979, - "uri": "https://{canondata_backend}/1936997/93899b3de50fae3f9677baacc98094a7a629590a/resource.tar.gz#test.test_pg-tpcds-q55-default.txt-Debug_/opt.yql_patched" + "checksum": "575c4ffb87c9cc96da1d85661ca0e271", + "size": 639, + "uri": "https://{canondata_backend}/1924537/994204c85c8f656606cca064cdae9e3d22058188/resource.tar.gz#test.test_pg-tpcds-q55-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q55-default.txt-Plan]": [ @@ -2073,9 +2073,9 @@ ], "test.test[pg-tpcds-q64-default.txt-Debug]": [ { - "checksum": "99ffc52417bd2639128d4147db016ba8", - "size": 1743, - "uri": "https://{canondata_backend}/1936997/93899b3de50fae3f9677baacc98094a7a629590a/resource.tar.gz#test.test_pg-tpcds-q64-default.txt-Debug_/opt.yql_patched" + "checksum": "dd7b79f5730916625921dc3a053cb9aa", + "size": 1327, + "uri": "https://{canondata_backend}/1924537/994204c85c8f656606cca064cdae9e3d22058188/resource.tar.gz#test.test_pg-tpcds-q64-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q64-default.txt-Plan]": [ @@ -2095,9 +2095,9 @@ ], "test.test[pg-tpcds-q99-default.txt-Debug]": [ { - "checksum": "4cfd531a65c370b5aa4468b946dfeaa3", - "size": 1248, - "uri": "https://{canondata_backend}/1942525/32a558f2f82aa94ed548a77eaea0657193500581/resource.tar.gz#test.test_pg-tpcds-q99-default.txt-Debug_/opt.yql_patched" + "checksum": "6330d07570b598b290f25600f41dff27", + "size": 851, + "uri": "https://{canondata_backend}/1924537/994204c85c8f656606cca064cdae9e3d22058188/resource.tar.gz#test.test_pg-tpcds-q99-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q99-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part14/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part14/canondata/result.json index 851ae95084ca..fb426f12ade0 100644 --- a/ydb/library/yql/tests/sql/dq_file/part14/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part14/canondata/result.json @@ -2319,9 +2319,9 @@ ], "test.test[pg-tpcds-q16-default.txt-Debug]": [ { - "checksum": "5f7bcf588f17a8bec04edc7e70ca6509", - "size": 3305, - "uri": "https://{canondata_backend}/1936997/ad7538cf8edf8e81865f7eee42c2de851daf1211/resource.tar.gz#test.test_pg-tpcds-q16-default.txt-Debug_/opt.yql_patched" + "checksum": "5248ac897649d8227ebdd9e281cfa540", + "size": 3117, + "uri": "https://{canondata_backend}/1773845/461e7989a09a65be78c660f6a49d876212096306/resource.tar.gz#test.test_pg-tpcds-q16-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q16-default.txt-Plan]": [ @@ -2385,9 +2385,9 @@ ], "test.test[pg-tpcds-q78-default.txt-Debug]": [ { - "checksum": "675bb53485999ee028e10065a1ea4c8e", - "size": 1726, - "uri": "https://{canondata_backend}/1936997/ad7538cf8edf8e81865f7eee42c2de851daf1211/resource.tar.gz#test.test_pg-tpcds-q78-default.txt-Debug_/opt.yql_patched" + "checksum": "b9a70464875d7439e6cddd96dd16babc", + "size": 1257, + "uri": "https://{canondata_backend}/1773845/461e7989a09a65be78c660f6a49d876212096306/resource.tar.gz#test.test_pg-tpcds-q78-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q78-default.txt-Plan]": [ @@ -2407,9 +2407,9 @@ ], "test.test[pg-tpcds-q85-default.txt-Debug]": [ { - "checksum": "8e6960f3af5cbb71740ec70cf5265017", - "size": 1358, - "uri": "https://{canondata_backend}/1871002/616290c68f4557542b5fbf174765825f6519b81f/resource.tar.gz#test.test_pg-tpcds-q85-default.txt-Debug_/opt.yql_patched" + "checksum": "74984502a5242e562f466d4b2e3e97f4", + "size": 740, + "uri": "https://{canondata_backend}/1773845/461e7989a09a65be78c660f6a49d876212096306/resource.tar.gz#test.test_pg-tpcds-q85-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q85-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json index 0376b3052047..8544fc015080 100644 --- a/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part15/canondata/result.json @@ -2140,9 +2140,9 @@ ], "test.test[pg-tpcds-q44-default.txt-Debug]": [ { - "checksum": "1b8db1c163c9bca5c6857cfe41419aba", - "size": 975, - "uri": "https://{canondata_backend}/1600758/aad142702907f13e911494c1a7b312bad34f692a/resource.tar.gz#test.test_pg-tpcds-q44-default.txt-Debug_/opt.yql_patched" + "checksum": "f90a90094a5162fe392caebe6a98aea5", + "size": 610, + "uri": "https://{canondata_backend}/212715/2cb1b5139d83aa48f3466b8892464b93f89797d0/resource.tar.gz#test.test_pg-tpcds-q44-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q44-default.txt-Plan]": [ @@ -2206,9 +2206,9 @@ ], "test.test[pg-tpcds-q62-default.txt-Debug]": [ { - "checksum": "a94fcf62f87342d4297350674292d4b5", - "size": 1252, - "uri": "https://{canondata_backend}/1936273/eb0d560ee0c1c4dad301a9afd809cba4153d5d6c/resource.tar.gz#test.test_pg-tpcds-q62-default.txt-Debug_/opt.yql_patched" + "checksum": "de40dc88cf3cebf52d96498021006320", + "size": 854, + "uri": "https://{canondata_backend}/212715/2cb1b5139d83aa48f3466b8892464b93f89797d0/resource.tar.gz#test.test_pg-tpcds-q62-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q62-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part16/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part16/canondata/result.json index 52bb9b8126e9..e9042cd4f522 100644 --- a/ydb/library/yql/tests/sql/dq_file/part16/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part16/canondata/result.json @@ -2075,9 +2075,9 @@ ], "test.test[pg-order_by_agg_no_extra_for_aggs-default.txt-Debug]": [ { - "checksum": "8bf84c2c63eb8ce00647567b21b09290", - "size": 1429, - "uri": "https://{canondata_backend}/1599023/6ea95a71ae6e3995d639ef495d263a106e521882/resource.tar.gz#test.test_pg-order_by_agg_no_extra_for_aggs-default.txt-Debug_/opt.yql_patched" + "checksum": "5e4d77346ee91b7f4a2a7e3dcee98c2f", + "size": 1847, + "uri": "https://{canondata_backend}/1777230/ad26fa2df1ca85e707984d82325c42cb09d4b3da/resource.tar.gz#test.test_pg-order_by_agg_no_extra_for_aggs-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-order_by_agg_no_extra_for_aggs-default.txt-Plan]": [ @@ -2229,9 +2229,9 @@ ], "test.test[pg-tpcds-q83-default.txt-Debug]": [ { - "checksum": "e9ac7df08620e5268a21b7d192af93c8", - "size": 1242, - "uri": "https://{canondata_backend}/1599023/6ea95a71ae6e3995d639ef495d263a106e521882/resource.tar.gz#test.test_pg-tpcds-q83-default.txt-Debug_/opt.yql_patched" + "checksum": "81a6ab71950789fadcd861689a2b3fc8", + "size": 826, + "uri": "https://{canondata_backend}/1784826/fb40e5e64a539be6615e07cc5f6e889bcc1e4564/resource.tar.gz#test.test_pg-tpcds-q83-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q83-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part17/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part17/canondata/result.json index 1c33c6bfae6a..b7012b135ec4 100644 --- a/ydb/library/yql/tests/sql/dq_file/part17/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part17/canondata/result.json @@ -1792,9 +1792,9 @@ ], "test.test[pg-order_by_agg_input_columns_prj_and_aggr-default.txt-Debug]": [ { - "checksum": "fc39f6af5d125db2b77661a579e7e872", - "size": 1569, - "uri": "https://{canondata_backend}/1871002/6626d4d4db6c797eb56d1419294da5e803196055/resource.tar.gz#test.test_pg-order_by_agg_input_columns_prj_and_aggr-default.txt-Debug_/opt.yql_patched" + "checksum": "614a96ba26dbe0aedc8c4859cfe375d0", + "size": 1599, + "uri": "https://{canondata_backend}/1599023/aac9f84343918a59a2e7eb28e31ca1d36cf30297/resource.tar.gz#test.test_pg-order_by_agg_input_columns_prj_and_aggr-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-order_by_agg_input_columns_prj_and_aggr-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part3/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part3/canondata/result.json index a07768684262..b1eae8484479 100644 --- a/ydb/library/yql/tests/sql/dq_file/part3/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part3/canondata/result.json @@ -1839,9 +1839,9 @@ ], "test.test[pg-tpcds-q79-default.txt-Debug]": [ { - "checksum": "036c154d6e9ef1ed64974819b9b4d997", - "size": 1340, - "uri": "https://{canondata_backend}/1889210/c22e1c869ce53a1ee053fd324047313c59e6f45d/resource.tar.gz#test.test_pg-tpcds-q79-default.txt-Debug_/opt.yql_patched" + "checksum": "64703e6a81fa325c14c728c5afebe013", + "size": 847, + "uri": "https://{canondata_backend}/1889210/718540831bd2dba15f12341f6611010d7d655169/resource.tar.gz#test.test_pg-tpcds-q79-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q79-default.txt-Plan]": [ @@ -1883,9 +1883,9 @@ ], "test.test[pg-tpcds-q84-default.txt-Debug]": [ { - "checksum": "d316218e6fd0a6d01828ddfe0499523f", - "size": 863, - "uri": "https://{canondata_backend}/1936947/0d1daf9062d6c8b5dd4dc9b1f73c97791c044137/resource.tar.gz#test.test_pg-tpcds-q84-default.txt-Debug_/opt.yql_patched" + "checksum": "8a9feddb4919ead064e22fe897098a6f", + "size": 556, + "uri": "https://{canondata_backend}/1889210/718540831bd2dba15f12341f6611010d7d655169/resource.tar.gz#test.test_pg-tpcds-q84-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q84-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part4/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part4/canondata/result.json index c6c221865566..f99a5aa42596 100644 --- a/ydb/library/yql/tests/sql/dq_file/part4/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part4/canondata/result.json @@ -2294,6 +2294,28 @@ } ], "test.test[sampling-bind_expr_subquery-default.txt-Results]": [], + "test.test[sampling-sample-default.txt-Analyze]": [ + { + "checksum": "6d0155b136969d7ed6b123616f4c5b6e", + "size": 7470, + "uri": "https://{canondata_backend}/1946324/271078e1009104f40f86a9564ea5abedf4eb2729/resource.tar.gz#test.test_sampling-sample-default.txt-Analyze_/plan.txt" + } + ], + "test.test[sampling-sample-default.txt-Debug]": [ + { + "checksum": "9521d3d2dc37d905d2db63f0f4adad46", + "size": 2446, + "uri": "https://{canondata_backend}/1946324/271078e1009104f40f86a9564ea5abedf4eb2729/resource.tar.gz#test.test_sampling-sample-default.txt-Debug_/opt.yql_patched" + } + ], + "test.test[sampling-sample-default.txt-Plan]": [ + { + "checksum": "6d0155b136969d7ed6b123616f4c5b6e", + "size": 7470, + "uri": "https://{canondata_backend}/1946324/271078e1009104f40f86a9564ea5abedf4eb2729/resource.tar.gz#test.test_sampling-sample-default.txt-Plan_/plan.txt" + } + ], + "test.test[sampling-sample-default.txt-Results]": [], "test.test[sampling-sort-default.txt-Analyze]": [ { "checksum": "52153b39639de8b55659620856cfe364", diff --git a/ydb/library/yql/tests/sql/dq_file/part5/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part5/canondata/result.json index d85ba1b89527..b34b90648c06 100644 --- a/ydb/library/yql/tests/sql/dq_file/part5/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part5/canondata/result.json @@ -2153,9 +2153,9 @@ ], "test.test[pg-order_by_agg_input_columns_prj_and_keys_and_aggr-default.txt-Debug]": [ { - "checksum": "3fc4d16e575f8f344570a74b172edd25", - "size": 1608, - "uri": "https://{canondata_backend}/1600758/bba12fda8a5a68a3753c70d51907e240b2e6a66b/resource.tar.gz#test.test_pg-order_by_agg_input_columns_prj_and_keys_and_aggr-default.txt-Debug_/opt.yql_patched" + "checksum": "2df9a63b4ce1d31e7bea35802758a2e4", + "size": 1678, + "uri": "https://{canondata_backend}/1775059/c78334a5a54c55b78c6157e0006c3af42c43b3aa/resource.tar.gz#test.test_pg-order_by_agg_input_columns_prj_and_keys_and_aggr-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-order_by_agg_input_columns_prj_and_keys_and_aggr-default.txt-Plan]": [ @@ -2351,9 +2351,9 @@ ], "test.test[pg-tpcds-q06-default.txt-Debug]": [ { - "checksum": "f090478f9f226e9ab4fb84725548dea2", - "size": 933, - "uri": "https://{canondata_backend}/1936842/73fe0e78069055b4c244798fc9c15ebb1173a692/resource.tar.gz#test.test_pg-tpcds-q06-default.txt-Debug_/opt.yql_patched" + "checksum": "6ba39ef366ee7bdb49bfcc18b80dfe40", + "size": 560, + "uri": "https://{canondata_backend}/1784826/25cbabce687b21eded79fabb140f901221253ab9/resource.tar.gz#test.test_pg-tpcds-q06-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q06-default.txt-Plan]": [ @@ -2373,9 +2373,9 @@ ], "test.test[pg-tpcds-q11-default.txt-Debug]": [ { - "checksum": "d8043d7b043a6d77959b059377a63b94", - "size": 1292, - "uri": "https://{canondata_backend}/1937429/5bcfa7fa889e048eab4fac33f32363a8c63e5b0b/resource.tar.gz#test.test_pg-tpcds-q11-default.txt-Debug_/opt.yql_patched" + "checksum": "c5dcf7a0751b7d76b990242adbad84ae", + "size": 763, + "uri": "https://{canondata_backend}/1784826/25cbabce687b21eded79fabb140f901221253ab9/resource.tar.gz#test.test_pg-tpcds-q11-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q11-default.txt-Plan]": [ @@ -2505,9 +2505,9 @@ ], "test.test[pg-tpcds-q91-default.txt-Debug]": [ { - "checksum": "9fffca4a98fa962cd63c6bb8963e9c3e", - "size": 713, - "uri": "https://{canondata_backend}/1936842/73fe0e78069055b4c244798fc9c15ebb1173a692/resource.tar.gz#test.test_pg-tpcds-q91-default.txt-Debug_/opt.yql_patched" + "checksum": "c6ab533a7d180142e58ae57260045ec1", + "size": 661, + "uri": "https://{canondata_backend}/1784826/25cbabce687b21eded79fabb140f901221253ab9/resource.tar.gz#test.test_pg-tpcds-q91-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q91-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part6/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part6/canondata/result.json index 7168def9d71b..375bb0ba9f27 100644 --- a/ydb/library/yql/tests/sql/dq_file/part6/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part6/canondata/result.json @@ -2182,9 +2182,9 @@ ], "test.test[pg-tpcds-q52-default.txt-Debug]": [ { - "checksum": "0b5c4d5b931f7997385c82c90d63d127", - "size": 1068, - "uri": "https://{canondata_backend}/1130705/bafe275fa937679d2b25012fec947db4686c5a93/resource.tar.gz#test.test_pg-tpcds-q52-default.txt-Debug_/opt.yql_patched" + "checksum": "aa998a6244b4460827d79bcb92acec8c", + "size": 734, + "uri": "https://{canondata_backend}/1871182/18e14cc850154a330057b23fc8c6576e30e17147/resource.tar.gz#test.test_pg-tpcds-q52-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q52-default.txt-Plan]": [ @@ -2248,9 +2248,9 @@ ], "test.test[pg-tpcds-q92-default.txt-Debug]": [ { - "checksum": "deab2cc6bbe9945492c3b80637f2e40c", - "size": 1718, - "uri": "https://{canondata_backend}/1942415/97b13545b1a9a1a90731e3bb7b5c3ac8ea4c7f0b/resource.tar.gz#test.test_pg-tpcds-q92-default.txt-Debug_/opt.yql_patched" + "checksum": "9a2f6f7779a8976f75a376ca8c437f83", + "size": 1619, + "uri": "https://{canondata_backend}/1871182/18e14cc850154a330057b23fc8c6576e30e17147/resource.tar.gz#test.test_pg-tpcds-q92-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q92-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part7/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part7/canondata/result.json index f4a9220b4e59..6e52350e0d4b 100644 --- a/ydb/library/yql/tests/sql/dq_file/part7/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part7/canondata/result.json @@ -1888,6 +1888,28 @@ } ], "test.test[pg-def_column_name_func-default.txt-Results]": [], + "test.test[pg-order_by_distinct_same_expr-default.txt-Analyze]": [ + { + "checksum": "b4dd508a329723c74293d80f0278c705", + "size": 505, + "uri": "https://{canondata_backend}/1931696/83451cbd5c05baf359743802ffeb9d68445da80c/resource.tar.gz#test.test_pg-order_by_distinct_same_expr-default.txt-Analyze_/plan.txt" + } + ], + "test.test[pg-order_by_distinct_same_expr-default.txt-Debug]": [ + { + "checksum": "afbdf0a98f4472c58bb19a4b64acb26f", + "size": 644, + "uri": "https://{canondata_backend}/1931696/83451cbd5c05baf359743802ffeb9d68445da80c/resource.tar.gz#test.test_pg-order_by_distinct_same_expr-default.txt-Debug_/opt.yql_patched" + } + ], + "test.test[pg-order_by_distinct_same_expr-default.txt-Plan]": [ + { + "checksum": "b4dd508a329723c74293d80f0278c705", + "size": 505, + "uri": "https://{canondata_backend}/1931696/83451cbd5c05baf359743802ffeb9d68445da80c/resource.tar.gz#test.test_pg-order_by_distinct_same_expr-default.txt-Plan_/plan.txt" + } + ], + "test.test[pg-order_by_distinct_same_expr-default.txt-Results]": [], "test.test[pg-pg_array_literal-default.txt-Analyze]": [ { "checksum": "b4dd508a329723c74293d80f0278c705", @@ -2073,9 +2095,9 @@ ], "test.test[pg-tpcds-q04-default.txt-Debug]": [ { - "checksum": "d8043d7b043a6d77959b059377a63b94", - "size": 1292, - "uri": "https://{canondata_backend}/1937429/73979ae5fc764dbe9181f487de92a666371a6e2a/resource.tar.gz#test.test_pg-tpcds-q04-default.txt-Debug_/opt.yql_patched" + "checksum": "c5dcf7a0751b7d76b990242adbad84ae", + "size": 763, + "uri": "https://{canondata_backend}/1773845/06fab929582e640fdce3e7cdf48ad02f2a7fe75f/resource.tar.gz#test.test_pg-tpcds-q04-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q04-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part8/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part8/canondata/result.json index 0dadae8b4153..37e8f6e964b6 100644 --- a/ydb/library/yql/tests/sql/dq_file/part8/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part8/canondata/result.json @@ -2524,9 +2524,9 @@ ], "test.test[pg-tpcds-q94-default.txt-Debug]": [ { - "checksum": "d8dff9d6e720e0d7a4c8f3619689168e", - "size": 3305, - "uri": "https://{canondata_backend}/995452/4daffa0e509c37c55ffb6e303121fefa54adef18/resource.tar.gz#test.test_pg-tpcds-q94-default.txt-Debug_/opt.yql_patched" + "checksum": "f27adf3fafc1d572c4751abf4ae0ec01", + "size": 3117, + "uri": "https://{canondata_backend}/1924537/67045fc137b7aae73c1137f6fa60894b964dec45/resource.tar.gz#test.test_pg-tpcds-q94-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q94-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/dq_file/part9/canondata/result.json b/ydb/library/yql/tests/sql/dq_file/part9/canondata/result.json index 003fcea0ad9f..ee67dec90190 100644 --- a/ydb/library/yql/tests/sql/dq_file/part9/canondata/result.json +++ b/ydb/library/yql/tests/sql/dq_file/part9/canondata/result.json @@ -1617,6 +1617,28 @@ } ], "test.test[pg-aggregate_factory-default.txt-Results]": [], + "test.test[pg-order_by_distinct_same_expr_agg-default.txt-Analyze]": [ + { + "checksum": "b4dd508a329723c74293d80f0278c705", + "size": 505, + "uri": "https://{canondata_backend}/1942173/8cb2c995567808fa2edb42fbcac76f18f5beb954/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg-default.txt-Analyze_/plan.txt" + } + ], + "test.test[pg-order_by_distinct_same_expr_agg-default.txt-Debug]": [ + { + "checksum": "724c93091a74022e59bbdd09938460bc", + "size": 1655, + "uri": "https://{canondata_backend}/1942173/8cb2c995567808fa2edb42fbcac76f18f5beb954/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg-default.txt-Debug_/opt.yql_patched" + } + ], + "test.test[pg-order_by_distinct_same_expr_agg-default.txt-Plan]": [ + { + "checksum": "b4dd508a329723c74293d80f0278c705", + "size": 505, + "uri": "https://{canondata_backend}/1942173/8cb2c995567808fa2edb42fbcac76f18f5beb954/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg-default.txt-Plan_/plan.txt" + } + ], + "test.test[pg-order_by_distinct_same_expr_agg-default.txt-Results]": [], "test.test[pg-order_by_shadow_input_columns-default.txt-Analyze]": [ { "checksum": "b4dd508a329723c74293d80f0278c705", @@ -1890,9 +1912,9 @@ ], "test.test[pg-tpcds-q42-default.txt-Debug]": [ { - "checksum": "e2c6493eb3fc5d716bbe847313bf657d", - "size": 1315, - "uri": "https://{canondata_backend}/1777230/91a147d17132375579c4f4ec545f8c73b8b69ee1/resource.tar.gz#test.test_pg-tpcds-q42-default.txt-Debug_/opt.yql_patched" + "checksum": "c5d0d262b3106c11978378ba436a2bd4", + "size": 771, + "uri": "https://{canondata_backend}/1942525/b372d3c86a68f2b33a18c3b61b0b7b8f739a0353/resource.tar.gz#test.test_pg-tpcds-q42-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q42-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/hybrid_file/part0/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part0/canondata/result.json index ecbed83bbd05..2a97a4be6fc8 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part0/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part0/canondata/result.json @@ -1975,9 +1975,9 @@ ], "test.test[pg-tpcds-q16-default.txt-Debug]": [ { - "checksum": "b771609d7da28f4f521c33e40c9486e8", - "size": 3304, - "uri": "https://{canondata_backend}/1936947/581aa6d896ffe57e25bdb8006459e912860e61fa/resource.tar.gz#test.test_pg-tpcds-q16-default.txt-Debug_/opt.yql_patched" + "checksum": "6538b40a83560deba940cba4454ad68b", + "size": 3116, + "uri": "https://{canondata_backend}/1784826/885e3ecf7da6faaa6a93df31c27314e793907f63/resource.tar.gz#test.test_pg-tpcds-q16-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q16-default.txt-Plan]": [ @@ -2059,9 +2059,9 @@ ], "test.test[pg-tpcds-q94-default.txt-Debug]": [ { - "checksum": "e0ebb78858a1d398ff17bc7cc09aee00", - "size": 3304, - "uri": "https://{canondata_backend}/1936947/581aa6d896ffe57e25bdb8006459e912860e61fa/resource.tar.gz#test.test_pg-tpcds-q94-default.txt-Debug_/opt.yql_patched" + "checksum": "3e4ace5df8e79fb55df47a2c0eba55e8", + "size": 3116, + "uri": "https://{canondata_backend}/1784826/885e3ecf7da6faaa6a93df31c27314e793907f63/resource.tar.gz#test.test_pg-tpcds-q94-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q94-default.txt-Plan]": [ @@ -2073,9 +2073,9 @@ ], "test.test[pg-tpcds-q99-default.txt-Debug]": [ { - "checksum": "7d9c7c0f045e43aaab9598c8c5f2094e", - "size": 1247, - "uri": "https://{canondata_backend}/1600758/acf43cdc6344357508eb4fb692b6408060f2de43/resource.tar.gz#test.test_pg-tpcds-q99-default.txt-Debug_/opt.yql_patched" + "checksum": "31c00a3f721c0ae6e8e83cff8566e967", + "size": 850, + "uri": "https://{canondata_backend}/1784826/885e3ecf7da6faaa6a93df31c27314e793907f63/resource.tar.gz#test.test_pg-tpcds-q99-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q99-default.txt-Plan]": [ @@ -2281,6 +2281,20 @@ "uri": "https://{canondata_backend}/1936947/581aa6d896ffe57e25bdb8006459e912860e61fa/resource.tar.gz#test.test_sampling-map--Plan_/plan.txt" } ], + "test.test[sampling-sample-default.txt-Debug]": [ + { + "checksum": "f9541034157625e736e785993c0104b8", + "size": 2445, + "uri": "https://{canondata_backend}/1937027/3bd1fc08639e36f4fa08237761765cdc2c954fed/resource.tar.gz#test.test_sampling-sample-default.txt-Debug_/opt.yql_patched" + } + ], + "test.test[sampling-sample-default.txt-Plan]": [ + { + "checksum": "6d0155b136969d7ed6b123616f4c5b6e", + "size": 7470, + "uri": "https://{canondata_backend}/1937027/3bd1fc08639e36f4fa08237761765cdc2c954fed/resource.tar.gz#test.test_sampling-sample-default.txt-Plan_/plan.txt" + } + ], "test.test[sampling-subquery_filter-default.txt-Debug]": [ { "checksum": "d23cdf655427e8a32c2b3b98db6d70a5", diff --git a/ydb/library/yql/tests/sql/hybrid_file/part1/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part1/canondata/result.json index a4cc9a91d73e..87c5d6be91b4 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part1/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part1/canondata/result.json @@ -1919,9 +1919,9 @@ ], "test.test[pg-tpcds-q95-default.txt-Debug]": [ { - "checksum": "e0ebb78858a1d398ff17bc7cc09aee00", - "size": 3304, - "uri": "https://{canondata_backend}/1936273/85968a675c17dd0728c8d7ba5fd43bd0b237dc65/resource.tar.gz#test.test_pg-tpcds-q95-default.txt-Debug_/opt.yql_patched" + "checksum": "3e4ace5df8e79fb55df47a2c0eba55e8", + "size": 3116, + "uri": "https://{canondata_backend}/212715/1d5e3cd59753ff0c77fb4968cc3520790b529523/resource.tar.gz#test.test_pg-tpcds-q95-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q95-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/hybrid_file/part10/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part10/canondata/result.json index 380364166cb8..1d9bbabb13c3 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part10/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part10/canondata/result.json @@ -2045,9 +2045,9 @@ ], "test.test[pg-tpcds-q14-default.txt-Debug]": [ { - "checksum": "123071746f26a8865d38826efcdbab7c", - "size": 4021, - "uri": "https://{canondata_backend}/1903885/4a2231cd1ba622089d05eed9128996b4045aa435/resource.tar.gz#test.test_pg-tpcds-q14-default.txt-Debug_/opt.yql_patched" + "checksum": "7e7270248d5556a9cf7f7739c746d880", + "size": 3623, + "uri": "https://{canondata_backend}/1773845/ebbc0e7a6553d487ca6f9443345b87dc94e5ba64/resource.tar.gz#test.test_pg-tpcds-q14-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q14-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/hybrid_file/part3/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part3/canondata/result.json index 8e6ae23def02..5134ee3d9dc4 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part3/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part3/canondata/result.json @@ -2143,9 +2143,9 @@ ], "test.test[pg-tpcds-q06-default.txt-Debug]": [ { - "checksum": "be8eeae82bfe526d601dbf6d45cea9fd", - "size": 932, - "uri": "https://{canondata_backend}/1936842/11d23d4a39031af80d6dc470ce99f9427771e7d4/resource.tar.gz#test.test_pg-tpcds-q06-default.txt-Debug_/opt.yql_patched" + "checksum": "a7bc15659ffd8a1eacb6abcd79a435ee", + "size": 559, + "uri": "https://{canondata_backend}/1889210/b3ec54c8ba5425d52cf7fa3db3638fad22de7e87/resource.tar.gz#test.test_pg-tpcds-q06-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q06-default.txt-Plan]": [ @@ -2185,9 +2185,9 @@ ], "test.test[pg-tpcds-q52-default.txt-Debug]": [ { - "checksum": "b6e545813751c4ac6f6f2bc945e6bb24", - "size": 1067, - "uri": "https://{canondata_backend}/1936842/11d23d4a39031af80d6dc470ce99f9427771e7d4/resource.tar.gz#test.test_pg-tpcds-q52-default.txt-Debug_/opt.yql_patched" + "checksum": "186d960359fbd9da77d3d00a2b2a1a0c", + "size": 733, + "uri": "https://{canondata_backend}/1889210/b3ec54c8ba5425d52cf7fa3db3638fad22de7e87/resource.tar.gz#test.test_pg-tpcds-q52-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q52-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/hybrid_file/part4/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part4/canondata/result.json index 945a04062abb..eb55a2975d47 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part4/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part4/canondata/result.json @@ -2157,9 +2157,9 @@ ], "test.test[pg-tpcds-q19-default.txt-Debug]": [ { - "checksum": "7549abb7930bc167abd7e874c79ac133", - "size": 1250, - "uri": "https://{canondata_backend}/1889210/431569691fa60b20bf9ef4cc94610d8f1b1518e2/resource.tar.gz#test.test_pg-tpcds-q19-default.txt-Debug_/opt.yql_patched" + "checksum": "9cac439e81df2eb6e82b4cf1c3d6aff3", + "size": 857, + "uri": "https://{canondata_backend}/1946324/f0844b7187f1db0315c7ba22b24ff34c0bddf188/resource.tar.gz#test.test_pg-tpcds-q19-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q19-default.txt-Plan]": [ @@ -2199,9 +2199,9 @@ ], "test.test[pg-tpcds-q62-default.txt-Debug]": [ { - "checksum": "5d500e9d320389352fe91735335ff665", - "size": 1251, - "uri": "https://{canondata_backend}/1600758/20bb0998dc12a8e0bd202fe7a254431e3192f87d/resource.tar.gz#test.test_pg-tpcds-q62-default.txt-Debug_/opt.yql_patched" + "checksum": "9d6d6259dcd7a03580548c8e1ce762b1", + "size": 853, + "uri": "https://{canondata_backend}/1946324/f0844b7187f1db0315c7ba22b24ff34c0bddf188/resource.tar.gz#test.test_pg-tpcds-q62-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q62-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/hybrid_file/part5/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part5/canondata/result.json index eb49531f96a0..e8b8a7a35740 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part5/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part5/canondata/result.json @@ -1609,6 +1609,20 @@ "uri": "https://{canondata_backend}/1936947/a5f83e5d38179c14126d53519dc062cef98113ec/resource.tar.gz#test.test_pg-distinct_on_single_projection_order_expr-default.txt-Plan_/plan.txt" } ], + "test.test[pg-order_by_distinct_same_expr_agg-default.txt-Debug]": [ + { + "checksum": "ea5a217f33689d11e87534ca5b50b484", + "size": 1654, + "uri": "https://{canondata_backend}/1942415/2b3301623e3d03377711deea0f47238c2650379b/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg-default.txt-Debug_/opt.yql_patched" + } + ], + "test.test[pg-order_by_distinct_same_expr_agg-default.txt-Plan]": [ + { + "checksum": "b4dd508a329723c74293d80f0278c705", + "size": 505, + "uri": "https://{canondata_backend}/1942415/2b3301623e3d03377711deea0f47238c2650379b/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg-default.txt-Plan_/plan.txt" + } + ], "test.test[pg-parametrized--Debug]": [ { "checksum": "7c3b35a4bb3aefdc17e5933622f91573", @@ -1863,9 +1877,9 @@ ], "test.test[pg-tpcds-q04-default.txt-Debug]": [ { - "checksum": "73b9bacd7d18a6d7dbf7042975b64157", - "size": 1291, - "uri": "https://{canondata_backend}/1936947/a5f83e5d38179c14126d53519dc062cef98113ec/resource.tar.gz#test.test_pg-tpcds-q04-default.txt-Debug_/opt.yql_patched" + "checksum": "4de0f58cb00459715e12d1634035ffa6", + "size": 762, + "uri": "https://{canondata_backend}/1773845/6676eb441f225906913d6af3ed308493a06ab168/resource.tar.gz#test.test_pg-tpcds-q04-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q04-default.txt-Plan]": [ @@ -1975,9 +1989,9 @@ ], "test.test[pg-tpcds-q64-default.txt-Debug]": [ { - "checksum": "07ee327f231e3a4a0a67b1a25971e965", - "size": 1742, - "uri": "https://{canondata_backend}/1936947/a5f83e5d38179c14126d53519dc062cef98113ec/resource.tar.gz#test.test_pg-tpcds-q64-default.txt-Debug_/opt.yql_patched" + "checksum": "09b2f318cf367d2420b3a9b1825ee14a", + "size": 1326, + "uri": "https://{canondata_backend}/1773845/6676eb441f225906913d6af3ed308493a06ab168/resource.tar.gz#test.test_pg-tpcds-q64-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q64-default.txt-Plan]": [ @@ -1989,9 +2003,9 @@ ], "test.test[pg-tpcds-q84-default.txt-Debug]": [ { - "checksum": "dbbb4f306db8d843059e3635dc2bf10a", - "size": 862, - "uri": "https://{canondata_backend}/1936947/a5f83e5d38179c14126d53519dc062cef98113ec/resource.tar.gz#test.test_pg-tpcds-q84-default.txt-Debug_/opt.yql_patched" + "checksum": "4ee299495bc3986389b2272b1b6343fd", + "size": 555, + "uri": "https://{canondata_backend}/1773845/6676eb441f225906913d6af3ed308493a06ab168/resource.tar.gz#test.test_pg-tpcds-q84-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q84-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/hybrid_file/part6/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part6/canondata/result.json index c5d5a9783466..5e17ce9462a9 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part6/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part6/canondata/result.json @@ -1623,6 +1623,34 @@ "uri": "https://{canondata_backend}/1775059/3cb7d014d70b84dbcb84645fa987dd9d47d7fd6c/resource.tar.gz#test.test_pg-long_ident-default.txt-Plan_/plan.txt" } ], + "test.test[pg-order_by_distinct_same_expr-default.txt-Debug]": [ + { + "checksum": "e01341f460d0a6d02f36a145b54a8e87", + "size": 643, + "uri": "https://{canondata_backend}/1942173/a88e613f98b9308632d7651072259231cab1e791/resource.tar.gz#test.test_pg-order_by_distinct_same_expr-default.txt-Debug_/opt.yql_patched" + } + ], + "test.test[pg-order_by_distinct_same_expr-default.txt-Plan]": [ + { + "checksum": "b4dd508a329723c74293d80f0278c705", + "size": 505, + "uri": "https://{canondata_backend}/1942173/a88e613f98b9308632d7651072259231cab1e791/resource.tar.gz#test.test_pg-order_by_distinct_same_expr-default.txt-Plan_/plan.txt" + } + ], + "test.test[pg-order_by_distinct_same_expr_agg_sublink-default.txt-Debug]": [ + { + "checksum": "85fd092bc15e55ed161b0f3e3fb6f953", + "size": 1677, + "uri": "https://{canondata_backend}/1936842/c62861d65748dca3fd75e9393720a48de8395467/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg_sublink-default.txt-Debug_/opt.yql_patched" + } + ], + "test.test[pg-order_by_distinct_same_expr_agg_sublink-default.txt-Plan]": [ + { + "checksum": "c1f2d837c3623c81dd596a9877913fb8", + "size": 948, + "uri": "https://{canondata_backend}/1936842/c62861d65748dca3fd75e9393720a48de8395467/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg_sublink-default.txt-Plan_/plan.txt" + } + ], "test.test[pg-palloc_big_string-default.txt-Debug]": [ { "checksum": "19df6ef8cda70ffb5c1931a200a773e4", @@ -1919,9 +1947,9 @@ ], "test.test[pg-tpcds-q11-default.txt-Debug]": [ { - "checksum": "73b9bacd7d18a6d7dbf7042975b64157", - "size": 1291, - "uri": "https://{canondata_backend}/1775059/3cb7d014d70b84dbcb84645fa987dd9d47d7fd6c/resource.tar.gz#test.test_pg-tpcds-q11-default.txt-Debug_/opt.yql_patched" + "checksum": "4de0f58cb00459715e12d1634035ffa6", + "size": 762, + "uri": "https://{canondata_backend}/1889210/3d889b385570041dbbd2165e00510547b2c1144d/resource.tar.gz#test.test_pg-tpcds-q11-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q11-default.txt-Plan]": [ @@ -1989,9 +2017,9 @@ ], "test.test[pg-tpcds-q91-default.txt-Debug]": [ { - "checksum": "eb943643b97a4fe30d9111594be695fe", - "size": 712, - "uri": "https://{canondata_backend}/1775059/3cb7d014d70b84dbcb84645fa987dd9d47d7fd6c/resource.tar.gz#test.test_pg-tpcds-q91-default.txt-Debug_/opt.yql_patched" + "checksum": "cd391fcb8b0d8a122f43f514295993dc", + "size": 660, + "uri": "https://{canondata_backend}/1889210/3d889b385570041dbbd2165e00510547b2c1144d/resource.tar.gz#test.test_pg-tpcds-q91-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q91-default.txt-Plan]": [ @@ -2003,9 +2031,9 @@ ], "test.test[pg-tpcds-q96-default.txt-Debug]": [ { - "checksum": "790cf34741e0a8108283e1f7beb10cf2", - "size": 1584, - "uri": "https://{canondata_backend}/1937027/ada6abcf938e1f8d16f0582e4d91ef5e675daa49/resource.tar.gz#test.test_pg-tpcds-q96-default.txt-Debug_/opt.yql_patched" + "checksum": "77304ccb181c6aa009b1ee4620a15512", + "size": 1490, + "uri": "https://{canondata_backend}/1889210/3d889b385570041dbbd2165e00510547b2c1144d/resource.tar.gz#test.test_pg-tpcds-q96-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q96-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/hybrid_file/part7/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part7/canondata/result.json index e78da67066f4..3b9c290040f2 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part7/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part7/canondata/result.json @@ -1555,9 +1555,9 @@ ], "test.test[pg-order_by_agg_input_columns_prj_and_keys_and_aggr-default.txt-Debug]": [ { - "checksum": "cd7a1e26e7d3cd6ae8e89b4cf69e7a40", - "size": 1607, - "uri": "https://{canondata_backend}/1600758/2bae720d354fef176d7a7ae70957b1a227ff538a/resource.tar.gz#test.test_pg-order_by_agg_input_columns_prj_and_keys_and_aggr-default.txt-Debug_/opt.yql_patched" + "checksum": "0f058b9426b775c5becfd678c07f0854", + "size": 1677, + "uri": "https://{canondata_backend}/1600758/14d5560c6b6df65b25a7d0e4e072602b1a2a8743/resource.tar.gz#test.test_pg-order_by_agg_input_columns_prj_and_keys_and_aggr-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-order_by_agg_input_columns_prj_and_keys_and_aggr-default.txt-Plan]": [ @@ -1807,9 +1807,9 @@ ], "test.test[pg-tpcds-q03-default.txt-Debug]": [ { - "checksum": "9622d526d4972e896d6046ac9ac71c9a", - "size": 1059, - "uri": "https://{canondata_backend}/1781765/028f42f897160b53900546b39900217bb2eb9fb1/resource.tar.gz#test.test_pg-tpcds-q03-default.txt-Debug_/opt.yql_patched" + "checksum": "644eef35976320bb8290b90e23065094", + "size": 727, + "uri": "https://{canondata_backend}/1781765/8061b1bca1dbb79a1a8135dd98452c011059369f/resource.tar.gz#test.test_pg-tpcds-q03-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q03-default.txt-Plan]": [ @@ -1877,9 +1877,9 @@ ], "test.test[pg-tpcds-q31-default.txt-Debug]": [ { - "checksum": "b536ee2eb598a4152fd661311ccb3f6c", - "size": 880, - "uri": "https://{canondata_backend}/1781765/028f42f897160b53900546b39900217bb2eb9fb1/resource.tar.gz#test.test_pg-tpcds-q31-default.txt-Debug_/opt.yql_patched" + "checksum": "0acee3524e12581b10548bc72dc05c53", + "size": 759, + "uri": "https://{canondata_backend}/1781765/8061b1bca1dbb79a1a8135dd98452c011059369f/resource.tar.gz#test.test_pg-tpcds-q31-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q31-default.txt-Plan]": [ @@ -1919,9 +1919,9 @@ ], "test.test[pg-tpcds-q44-default.txt-Debug]": [ { - "checksum": "4ddf42fa3c3e960e82ee9a66b125381f", - "size": 974, - "uri": "https://{canondata_backend}/1781765/028f42f897160b53900546b39900217bb2eb9fb1/resource.tar.gz#test.test_pg-tpcds-q44-default.txt-Debug_/opt.yql_patched" + "checksum": "62d13159cf9bfd4987607f5189566427", + "size": 609, + "uri": "https://{canondata_backend}/1781765/8061b1bca1dbb79a1a8135dd98452c011059369f/resource.tar.gz#test.test_pg-tpcds-q44-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q44-default.txt-Plan]": [ @@ -1989,9 +1989,9 @@ ], "test.test[pg-tpcds-q78-default.txt-Debug]": [ { - "checksum": "37b04062c1d7361087b3983c23c7ec36", - "size": 1725, - "uri": "https://{canondata_backend}/1781765/028f42f897160b53900546b39900217bb2eb9fb1/resource.tar.gz#test.test_pg-tpcds-q78-default.txt-Debug_/opt.yql_patched" + "checksum": "b94d3f17b2683cba8d60d11a30808ca2", + "size": 1256, + "uri": "https://{canondata_backend}/1781765/8061b1bca1dbb79a1a8135dd98452c011059369f/resource.tar.gz#test.test_pg-tpcds-q78-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q78-default.txt-Plan]": [ @@ -2003,9 +2003,9 @@ ], "test.test[pg-tpcds-q85-default.txt-Debug]": [ { - "checksum": "61a79875b8d10cc038dc34e8ed8d3139", - "size": 1357, - "uri": "https://{canondata_backend}/1871002/2a28301cd702f47961195a0e9d71a1a846884662/resource.tar.gz#test.test_pg-tpcds-q85-default.txt-Debug_/opt.yql_patched" + "checksum": "b64b77214651e72022fef9f1e87aacde", + "size": 739, + "uri": "https://{canondata_backend}/1781765/8061b1bca1dbb79a1a8135dd98452c011059369f/resource.tar.gz#test.test_pg-tpcds-q85-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q85-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json index 86fbf9f786e0..e5c7d7858bf7 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part8/canondata/result.json @@ -1933,9 +1933,9 @@ ], "test.test[pg-order_by_agg_input_columns_prj_and_aggr-default.txt-Debug]": [ { - "checksum": "0baeb3619c993f0bb1a5c299b7c0bb50", - "size": 1568, - "uri": "https://{canondata_backend}/1781765/9c2a131db7c99a7c31942aabf8738f3bac969d91/resource.tar.gz#test.test_pg-order_by_agg_input_columns_prj_and_aggr-default.txt-Debug_/opt.yql_patched" + "checksum": "666b13e951ff59ae081fdda076199c1b", + "size": 1598, + "uri": "https://{canondata_backend}/1600758/fa72a23c77bab9a775b9e8e822e0be1a9841d508/resource.tar.gz#test.test_pg-order_by_agg_input_columns_prj_and_aggr-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-order_by_agg_input_columns_prj_and_aggr-default.txt-Plan]": [ @@ -1947,9 +1947,9 @@ ], "test.test[pg-order_by_agg_no_extra_for_aggs-default.txt-Debug]": [ { - "checksum": "a0b18b59219dce68a356a4014fc3a8de", - "size": 1428, - "uri": "https://{canondata_backend}/1923547/94f377eaa1d93890e1345ac4940cc6fa07bddd4f/resource.tar.gz#test.test_pg-order_by_agg_no_extra_for_aggs-default.txt-Debug_/opt.yql_patched" + "checksum": "62fa3aa85e5fe6dd04bf4afeba403a49", + "size": 1846, + "uri": "https://{canondata_backend}/1775319/16e55349c1d8a123c91f7d512b301ac22c034701/resource.tar.gz#test.test_pg-order_by_agg_no_extra_for_aggs-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-order_by_agg_no_extra_for_aggs-default.txt-Plan]": [ @@ -2283,9 +2283,9 @@ ], "test.test[pg-tpcds-q42-default.txt-Debug]": [ { - "checksum": "bde6fe798f250cfc235d460622ca423e", - "size": 1314, - "uri": "https://{canondata_backend}/1781765/9c2a131db7c99a7c31942aabf8738f3bac969d91/resource.tar.gz#test.test_pg-tpcds-q42-default.txt-Debug_/opt.yql_patched" + "checksum": "5cbcf08d68f10e18fd6157290d3b3f35", + "size": 770, + "uri": "https://{canondata_backend}/1773845/151e1e36181dc4f51864bb618bfd0ac1b52111fc/resource.tar.gz#test.test_pg-tpcds-q42-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q42-default.txt-Plan]": [ @@ -2311,9 +2311,9 @@ ], "test.test[pg-tpcds-q55-default.txt-Debug]": [ { - "checksum": "91bcb9cd382166d7a9d35989232509ee", - "size": 978, - "uri": "https://{canondata_backend}/1923547/94f377eaa1d93890e1345ac4940cc6fa07bddd4f/resource.tar.gz#test.test_pg-tpcds-q55-default.txt-Debug_/opt.yql_patched" + "checksum": "15df7e059a3819f0f4eba761ac8d2104", + "size": 638, + "uri": "https://{canondata_backend}/1773845/151e1e36181dc4f51864bb618bfd0ac1b52111fc/resource.tar.gz#test.test_pg-tpcds-q55-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q55-default.txt-Plan]": [ @@ -2339,9 +2339,9 @@ ], "test.test[pg-tpcds-q83-default.txt-Debug]": [ { - "checksum": "2a403226f1755e1e2a7ea0caacc8441d", - "size": 1241, - "uri": "https://{canondata_backend}/1923547/94f377eaa1d93890e1345ac4940cc6fa07bddd4f/resource.tar.gz#test.test_pg-tpcds-q83-default.txt-Debug_/opt.yql_patched" + "checksum": "ddf63a289a7c21c255315a6bb63d883f", + "size": 825, + "uri": "https://{canondata_backend}/1773845/151e1e36181dc4f51864bb618bfd0ac1b52111fc/resource.tar.gz#test.test_pg-tpcds-q83-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q83-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/hybrid_file/part9/canondata/result.json b/ydb/library/yql/tests/sql/hybrid_file/part9/canondata/result.json index d31be34c731f..0208954fdb69 100644 --- a/ydb/library/yql/tests/sql/hybrid_file/part9/canondata/result.json +++ b/ydb/library/yql/tests/sql/hybrid_file/part9/canondata/result.json @@ -2073,9 +2073,9 @@ ], "test.test[pg-tpcds-q71-default.txt-Debug]": [ { - "checksum": "ad7349122106589e6654e515f90ac939", - "size": 836, - "uri": "https://{canondata_backend}/1889210/796baf28896eb5aaad8828a0b6000e7d17563447/resource.tar.gz#test.test_pg-tpcds-q71-default.txt-Debug_/opt.yql_patched" + "checksum": "12c3cfd5c9809917a4279ff9926e5759", + "size": 709, + "uri": "https://{canondata_backend}/1773845/9450e6a9e418f128c33ccd34fc163a655ef7efb6/resource.tar.gz#test.test_pg-tpcds-q71-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q71-default.txt-Plan]": [ @@ -2101,9 +2101,9 @@ ], "test.test[pg-tpcds-q79-default.txt-Debug]": [ { - "checksum": "2c3797a2708d161b85d5af3c08a9baa4", - "size": 1339, - "uri": "https://{canondata_backend}/1600758/945d8bd5a89c655f23736d13a6248011860cd506/resource.tar.gz#test.test_pg-tpcds-q79-default.txt-Debug_/opt.yql_patched" + "checksum": "a3c50395bf4d1a6f3a21bca6d80e6b95", + "size": 846, + "uri": "https://{canondata_backend}/1773845/9450e6a9e418f128c33ccd34fc163a655ef7efb6/resource.tar.gz#test.test_pg-tpcds-q79-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q79-default.txt-Plan]": [ @@ -2129,9 +2129,9 @@ ], "test.test[pg-tpcds-q92-default.txt-Debug]": [ { - "checksum": "2cc8e0919bd4729d929e17d1807e1e36", - "size": 1717, - "uri": "https://{canondata_backend}/1889210/796baf28896eb5aaad8828a0b6000e7d17563447/resource.tar.gz#test.test_pg-tpcds-q92-default.txt-Debug_/opt.yql_patched" + "checksum": "6be6567e78f01793829008a7960076d1", + "size": 1618, + "uri": "https://{canondata_backend}/1773845/9450e6a9e418f128c33ccd34fc163a655ef7efb6/resource.tar.gz#test.test_pg-tpcds-q92-default.txt-Debug_/opt.yql_patched" } ], "test.test[pg-tpcds-q92-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json index 5a7f86ff0b93..925907e3e924 100644 --- a/ydb/library/yql/tests/sql/sql2yql/canondata/result.json +++ b/ydb/library/yql/tests/sql/sql2yql/canondata/result.json @@ -11103,23 +11103,44 @@ ], "test_sql2yql.test[pg-order_by_agg_input_columns_prj_and_aggr]": [ { - "checksum": "36c1e4e40ba1dd6ed249b6429d5ea14d", - "size": 966, - "uri": "https://{canondata_backend}/1871002/aa5733774deffaee951006652c15a43754548f46/resource.tar.gz#test_sql2yql.test_pg-order_by_agg_input_columns_prj_and_aggr_/sql.yql" + "checksum": "76ec779b7b2f023561b8a62e51c355c4", + "size": 1006, + "uri": "https://{canondata_backend}/1942173/abecf46358d847c78f2117cec17ad429699ca53c/resource.tar.gz#test_sql2yql.test_pg-order_by_agg_input_columns_prj_and_aggr_/sql.yql" } ], "test_sql2yql.test[pg-order_by_agg_input_columns_prj_and_keys_and_aggr]": [ { - "checksum": "980c2a8690e2eb9909532d7e4f7d7ef4", - "size": 1019, - "uri": "https://{canondata_backend}/1871002/aa5733774deffaee951006652c15a43754548f46/resource.tar.gz#test_sql2yql.test_pg-order_by_agg_input_columns_prj_and_keys_and_aggr_/sql.yql" + "checksum": "4a54c9777ccfde975fd898b4f28b9112", + "size": 1059, + "uri": "https://{canondata_backend}/1942173/abecf46358d847c78f2117cec17ad429699ca53c/resource.tar.gz#test_sql2yql.test_pg-order_by_agg_input_columns_prj_and_keys_and_aggr_/sql.yql" } ], "test_sql2yql.test[pg-order_by_agg_no_extra_for_aggs]": [ { - "checksum": "d1c7af7efe24ea6c11b217c66589cfef", - "size": 715, - "uri": "https://{canondata_backend}/1599023/af9c2f81df0601cf266a0926b5ce73b6101b9115/resource.tar.gz#test_sql2yql.test_pg-order_by_agg_no_extra_for_aggs_/sql.yql" + "checksum": "737a30b260d808fbe1ff5da1b76ce528", + "size": 713, + "uri": "https://{canondata_backend}/1871002/a5ffe5219921c8daea4d9e5fcde3aae3fa42d29c/resource.tar.gz#test_sql2yql.test_pg-order_by_agg_no_extra_for_aggs_/sql.yql" + } + ], + "test_sql2yql.test[pg-order_by_distinct_same_expr]": [ + { + "checksum": "90ff11d3ea24565554ca875062effe44", + "size": 977, + "uri": "https://{canondata_backend}/1871182/be7e0db544f9cfa977e36aa7ea0951073e129335/resource.tar.gz#test_sql2yql.test_pg-order_by_distinct_same_expr_/sql.yql" + } + ], + "test_sql2yql.test[pg-order_by_distinct_same_expr_agg]": [ + { + "checksum": "d9fa181c807070e8f4b1e98f2a046170", + "size": 1546, + "uri": "https://{canondata_backend}/1871182/be7e0db544f9cfa977e36aa7ea0951073e129335/resource.tar.gz#test_sql2yql.test_pg-order_by_distinct_same_expr_agg_/sql.yql" + } + ], + "test_sql2yql.test[pg-order_by_distinct_same_expr_agg_sublink]": [ + { + "checksum": "fb935ca78b8c8fc5446676ecf11414c4", + "size": 1593, + "uri": "https://{canondata_backend}/1923547/a5628d49e504286f75a25bfc2053145eaa2cfec1/resource.tar.gz#test_sql2yql.test_pg-order_by_distinct_same_expr_agg_sublink_/sql.yql" } ], "test_sql2yql.test[pg-order_by_input_columns]": [ @@ -14286,6 +14307,13 @@ "uri": "https://{canondata_backend}/1871182/78dc92fb0f0a949ef8d6e955f47dd0843494dee6/resource.tar.gz#test_sql2yql.test_sampling-reduce_with_presort_/sql.yql" } ], + "test_sql2yql.test[sampling-sample]": [ + { + "checksum": "61f9507fd003d17c8b5b6a6d4eb609b2", + "size": 3654, + "uri": "https://{canondata_backend}/1936947/d0bdc5228243912c5ef95425f5566fe38b25c878/resource.tar.gz#test_sql2yql.test_sampling-sample_/sql.yql" + } + ], "test_sql2yql.test[sampling-sort]": [ { "checksum": "92f5eb70dbf34ed367159f17ed1a8fb2", @@ -29329,6 +29357,13 @@ "uri": "https://{canondata_backend}/1880306/64654158d6bfb1289c66c626a8162239289559d0/resource.tar.gz#test_sql_format.test_sampling-reduce_with_presort_/formatted.sql" } ], + "test_sql_format.test[sampling-sample]": [ + { + "checksum": "f5f4a3bf510f31d7a781b95f4b16d6e9", + "size": 165, + "uri": "https://{canondata_backend}/1936947/d0bdc5228243912c5ef95425f5566fe38b25c878/resource.tar.gz#test_sql_format.test_sampling-sample_/formatted.sql" + } + ], "test_sql_format.test[sampling-sort]": [ { "checksum": "517d416d6aead392dd8b5d98afc8fbbb", diff --git a/ydb/library/yql/tests/sql/suites/pg/order_by_agg_input_columns_prj_and_aggr.sql b/ydb/library/yql/tests/sql/suites/pg/order_by_agg_input_columns_prj_and_aggr.sql index 0a277bfc025e..73e2f8efcfe3 100644 --- a/ydb/library/yql/tests/sql/suites/pg/order_by_agg_input_columns_prj_and_aggr.sql +++ b/ydb/library/yql/tests/sql/suites/pg/order_by_agg_input_columns_prj_and_aggr.sql @@ -2,4 +2,4 @@ select x, count(*) from (values (1,1),(3,5),(3,4)) a(x,y) group by x -order by count(*) desc +order by count(*)+1 desc diff --git a/ydb/library/yql/tests/sql/suites/pg/order_by_agg_input_columns_prj_and_keys_and_aggr.sql b/ydb/library/yql/tests/sql/suites/pg/order_by_agg_input_columns_prj_and_keys_and_aggr.sql index 439ce87d900b..076e8ea1864c 100644 --- a/ydb/library/yql/tests/sql/suites/pg/order_by_agg_input_columns_prj_and_keys_and_aggr.sql +++ b/ydb/library/yql/tests/sql/suites/pg/order_by_agg_input_columns_prj_and_keys_and_aggr.sql @@ -2,4 +2,4 @@ select x, count(*) from (values (1,1),(3,5),(3,4)) a(x,y) group by x -order by count(*) desc,x +order by count(*)+1 desc,x diff --git a/ydb/library/yql/tests/sql/suites/pg/order_by_agg_no_extra_for_aggs.sql b/ydb/library/yql/tests/sql/suites/pg/order_by_agg_no_extra_for_aggs.sql index 21ec294e7d0a..5c6cf52019df 100644 --- a/ydb/library/yql/tests/sql/suites/pg/order_by_agg_no_extra_for_aggs.sql +++ b/ydb/library/yql/tests/sql/suites/pg/order_by_agg_no_extra_for_aggs.sql @@ -1,4 +1,4 @@ --!syntax_pg select count(x) as y from (values (1)) a(x) -order by count(x) \ No newline at end of file +order by sum(x) diff --git a/ydb/library/yql/tests/sql/suites/pg/order_by_distinct_same_expr.sql b/ydb/library/yql/tests/sql/suites/pg/order_by_distinct_same_expr.sql new file mode 100644 index 000000000000..56a03ea834d6 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/pg/order_by_distinct_same_expr.sql @@ -0,0 +1,3 @@ +--!syntax_pg +select distinct x+1 as y,x-1 from (select 1 as x) a order by x+1,x-1 + diff --git a/ydb/library/yql/tests/sql/suites/pg/order_by_distinct_same_expr_agg.sql b/ydb/library/yql/tests/sql/suites/pg/order_by_distinct_same_expr_agg.sql new file mode 100644 index 000000000000..2ea98f75e5da --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/pg/order_by_distinct_same_expr_agg.sql @@ -0,0 +1,2 @@ +--!syntax_pg +select distinct (x + 1) * 2 as y,(x + 1) * 3,min(z) - 1 from (select 1 as x,2 as z) a group by x + 1 order by (x + 1) * 2, (x + 1) * 3, min(z) - 1 diff --git a/ydb/library/yql/tests/sql/suites/pg/order_by_distinct_same_expr_agg_sublink.sql b/ydb/library/yql/tests/sql/suites/pg/order_by_distinct_same_expr_agg_sublink.sql new file mode 100644 index 000000000000..3f1d1169d330 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/pg/order_by_distinct_same_expr_agg_sublink.sql @@ -0,0 +1,2 @@ +--!syntax_pg +select distinct x + 1/(select count(*) from pg_type) as y from (select 1 as x) a order by x + 1/(select count(*) from pg_type) diff --git a/ydb/library/yql/tests/sql/suites/sampling/sample.sql b/ydb/library/yql/tests/sql/suites/sampling/sample.sql new file mode 100644 index 000000000000..39d1b4f40c18 --- /dev/null +++ b/ydb/library/yql/tests/sql/suites/sampling/sample.sql @@ -0,0 +1,9 @@ +USE plato; + +SELECT * FROM Input +SAMPLE(0.5) +ORDER BY key; + +SELECT * FROM (SELECT* from Input) +SAMPLE(0.5) +ORDER BY key; diff --git a/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json index 1dd37ea37107..2530ce5c45f1 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part1/canondata/result.json @@ -1952,9 +1952,9 @@ ], "test.test[pg-tpcds-q19-default.txt-Debug]": [ { - "checksum": "88436f9a9ad0f22a3eec991409c85419", - "size": 1191, - "uri": "https://{canondata_backend}/1871102/0b211a9063647b0dc256212e231671bd17334519/resource.tar.gz#test.test_pg-tpcds-q19-default.txt-Debug_/opt.yql" + "checksum": "215a34d5cb30e46f945e47c79f8af19d", + "size": 798, + "uri": "https://{canondata_backend}/1942525/d135026d740e9844a43c97c2b91cb8c6502d994f/resource.tar.gz#test.test_pg-tpcds-q19-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q19-default.txt-Plan]": [ @@ -2057,9 +2057,9 @@ ], "test.test[pg-tpcds-q95-default.txt-Debug]": [ { - "checksum": "bc8a00ea43d88fec0663cb0c2dc3338c", - "size": 3242, - "uri": "https://{canondata_backend}/1937424/c3efe635504410bffb41db03fb22dc866e802fb7/resource.tar.gz#test.test_pg-tpcds-q95-default.txt-Debug_/opt.yql" + "checksum": "6739e519f2684c6306ba2c3f563b24a4", + "size": 3055, + "uri": "https://{canondata_backend}/1942525/d135026d740e9844a43c97c2b91cb8c6502d994f/resource.tar.gz#test.test_pg-tpcds-q95-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q95-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part10/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part10/canondata/result.json index ccf8aaba1270..6ae580a3f659 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part10/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part10/canondata/result.json @@ -1672,6 +1672,27 @@ "uri": "https://{canondata_backend}/1924537/852bcba5756c333f9484e9976048bf4b9c984664/resource.tar.gz#test.test_pg-aggregate_combine--Results_/results.txt" } ], + "test.test[pg-order_by_distinct_same_expr_agg_sublink-default.txt-Debug]": [ + { + "checksum": "add068636ef7667f017803290484353e", + "size": 1618, + "uri": "https://{canondata_backend}/1923547/14766d8020386b740f1aeffdb33d3c8fa323317d/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg_sublink-default.txt-Debug_/opt.yql" + } + ], + "test.test[pg-order_by_distinct_same_expr_agg_sublink-default.txt-Plan]": [ + { + "checksum": "c1f2d837c3623c81dd596a9877913fb8", + "size": 948, + "uri": "https://{canondata_backend}/1923547/14766d8020386b740f1aeffdb33d3c8fa323317d/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg_sublink-default.txt-Plan_/plan.txt" + } + ], + "test.test[pg-order_by_distinct_same_expr_agg_sublink-default.txt-Results]": [ + { + "checksum": "3cae0e6a8d7ba51d860a87cf87668351", + "size": 656, + "uri": "https://{canondata_backend}/1923547/14766d8020386b740f1aeffdb33d3c8fa323317d/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg_sublink-default.txt-Results_/results.txt" + } + ], "test.test[pg-select_agg_gs_rollup-default.txt-Debug]": [ { "checksum": "a365d2ec54c3121b00ba8e9ca2c22e58", @@ -1968,9 +1989,9 @@ ], "test.test[pg-tpcds-q96-default.txt-Debug]": [ { - "checksum": "1693671785137d36f74b5ae6d2b39dd4", - "size": 1525, - "uri": "https://{canondata_backend}/1924537/852bcba5756c333f9484e9976048bf4b9c984664/resource.tar.gz#test.test_pg-tpcds-q96-default.txt-Debug_/opt.yql" + "checksum": "c66dadb171103431e2c31d3274c7d0c7", + "size": 1431, + "uri": "https://{canondata_backend}/1942100/4de42de49ca3e2a70a76090e61fb9d9646e439b6/resource.tar.gz#test.test_pg-tpcds-q96-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q96-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json index 7b2d000b2362..d67f0d7e9134 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part11/canondata/result.json @@ -1716,9 +1716,9 @@ ], "test.test[pg-tpcds-q31-default.txt-Debug]": [ { - "checksum": "53dde57ad9441c1678a88c2a78959a79", - "size": 817, - "uri": "https://{canondata_backend}/1937027/642fd2ff53bdb0fed32ca89598d70c9c5848ac20/resource.tar.gz#test.test_pg-tpcds-q31-default.txt-Debug_/opt.yql" + "checksum": "49d36a0e91fdeccb7ee86b6561e29c59", + "size": 700, + "uri": "https://{canondata_backend}/1784826/3d6df48227ac00ed24da6a750e2d910a761ca66e/resource.tar.gz#test.test_pg-tpcds-q31-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q31-default.txt-Plan]": [ @@ -1758,9 +1758,9 @@ ], "test.test[pg-tpcds-q71-default.txt-Debug]": [ { - "checksum": "ee133457341822ef8a4bf29f8e4b272a", - "size": 777, - "uri": "https://{canondata_backend}/1937027/642fd2ff53bdb0fed32ca89598d70c9c5848ac20/resource.tar.gz#test.test_pg-tpcds-q71-default.txt-Debug_/opt.yql" + "checksum": "c5564e064be6764aaa6eba8589ca39da", + "size": 651, + "uri": "https://{canondata_backend}/1784826/3d6df48227ac00ed24da6a750e2d910a761ca66e/resource.tar.gz#test.test_pg-tpcds-q71-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q71-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part12/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part12/canondata/result.json index 09c4933a63ff..0dfcb502ac74 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part12/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part12/canondata/result.json @@ -2201,9 +2201,9 @@ ], "test.test[pg-tpcds-q03-default.txt-Debug]": [ { - "checksum": "3ac50c4c1ef96d1236f1d0b25c2e336d", - "size": 1000, - "uri": "https://{canondata_backend}/1942415/14c2050022adcb6379d90c4a2a09abf0847e7643/resource.tar.gz#test.test_pg-tpcds-q03-default.txt-Debug_/opt.yql" + "checksum": "f6d25fabeb2fa71598ca6345835d9ff7", + "size": 668, + "uri": "https://{canondata_backend}/1600758/9ff6757e840d90ecce64fb26653247d8d8d6eaff/resource.tar.gz#test.test_pg-tpcds-q03-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q03-default.txt-Plan]": [ @@ -2222,9 +2222,9 @@ ], "test.test[pg-tpcds-q14-default.txt-Debug]": [ { - "checksum": "fea41cb10452c787e505c5e83cdb9c96", - "size": 3961, - "uri": "https://{canondata_backend}/1871182/f42651a4c8505a845b01023d9d7e8e03f2e50c2d/resource.tar.gz#test.test_pg-tpcds-q14-default.txt-Debug_/opt.yql" + "checksum": "c33aaa27acdcf9a09906eee6a052acb4", + "size": 3563, + "uri": "https://{canondata_backend}/1600758/9ff6757e840d90ecce64fb26653247d8d8d6eaff/resource.tar.gz#test.test_pg-tpcds-q14-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q14-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part13/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part13/canondata/result.json index fcaeca9e4f37..537b79ad915f 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part13/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part13/canondata/result.json @@ -2012,9 +2012,9 @@ ], "test.test[pg-tpcds-q55-default.txt-Debug]": [ { - "checksum": "a1b945f3628fbcf948b284192337f16e", - "size": 919, - "uri": "https://{canondata_backend}/212715/e1bc1d6e31fa656365a738e65224a7c3f774bae6/resource.tar.gz#test.test_pg-tpcds-q55-default.txt-Debug_/opt.yql" + "checksum": "c2e917ab28ce7f11d6234bd1c066f9bd", + "size": 583, + "uri": "https://{canondata_backend}/1784826/430783c5a39647d2c72741870fc87989c43c193e/resource.tar.gz#test.test_pg-tpcds-q55-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q55-default.txt-Plan]": [ @@ -2054,9 +2054,9 @@ ], "test.test[pg-tpcds-q64-default.txt-Debug]": [ { - "checksum": "1548365cd0be2e1f05af05770daf0bf4", - "size": 1678, - "uri": "https://{canondata_backend}/212715/e1bc1d6e31fa656365a738e65224a7c3f774bae6/resource.tar.gz#test.test_pg-tpcds-q64-default.txt-Debug_/opt.yql" + "checksum": "422d0307a15d9d875496a51bd69e9ecd", + "size": 1267, + "uri": "https://{canondata_backend}/1784826/430783c5a39647d2c72741870fc87989c43c193e/resource.tar.gz#test.test_pg-tpcds-q64-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q64-default.txt-Plan]": [ @@ -2075,9 +2075,9 @@ ], "test.test[pg-tpcds-q99-default.txt-Debug]": [ { - "checksum": "2840ad79cb45be55a8f10a376ef0c8d0", - "size": 1188, - "uri": "https://{canondata_backend}/1942525/ac46fec4341e8dbe9297ea09bff7c34708fa9891/resource.tar.gz#test.test_pg-tpcds-q99-default.txt-Debug_/opt.yql" + "checksum": "c5bf2b07fc6b46ffebf6e6b4bf57ab4a", + "size": 791, + "uri": "https://{canondata_backend}/1784826/430783c5a39647d2c72741870fc87989c43c193e/resource.tar.gz#test.test_pg-tpcds-q99-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q99-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part14/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part14/canondata/result.json index 226c33eefaa3..ae0257a32b7b 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part14/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part14/canondata/result.json @@ -2249,9 +2249,9 @@ ], "test.test[pg-tpcds-q16-default.txt-Debug]": [ { - "checksum": "01f80fcb5a2952e6a83e664347c5c7ef", - "size": 3242, - "uri": "https://{canondata_backend}/1923547/5154c8bd8ef9ead4f609771f831f20c15e795571/resource.tar.gz#test.test_pg-tpcds-q16-default.txt-Debug_/opt.yql" + "checksum": "38e0bbc7d8fa1f222108a21b878db94d", + "size": 3055, + "uri": "https://{canondata_backend}/1784826/178f49acc304b019ce38e349b0691fbac8944b22/resource.tar.gz#test.test_pg-tpcds-q16-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q16-default.txt-Plan]": [ @@ -2312,9 +2312,9 @@ ], "test.test[pg-tpcds-q78-default.txt-Debug]": [ { - "checksum": "25f5affbf92cee380147f7f85dcb279d", - "size": 1662, - "uri": "https://{canondata_backend}/1923547/5154c8bd8ef9ead4f609771f831f20c15e795571/resource.tar.gz#test.test_pg-tpcds-q78-default.txt-Debug_/opt.yql" + "checksum": "bac70b45f47b371b280013636df218d5", + "size": 1193, + "uri": "https://{canondata_backend}/1784826/178f49acc304b019ce38e349b0691fbac8944b22/resource.tar.gz#test.test_pg-tpcds-q78-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q78-default.txt-Plan]": [ @@ -2333,9 +2333,9 @@ ], "test.test[pg-tpcds-q85-default.txt-Debug]": [ { - "checksum": "204a874bd83f38be5cc43f618789c13f", - "size": 1296, - "uri": "https://{canondata_backend}/1871182/b611d0aee8ff3d0602fca491ccb2628c604785cf/resource.tar.gz#test.test_pg-tpcds-q85-default.txt-Debug_/opt.yql" + "checksum": "a150e29c354394820554b4736c2210f1", + "size": 680, + "uri": "https://{canondata_backend}/1784826/178f49acc304b019ce38e349b0691fbac8944b22/resource.tar.gz#test.test_pg-tpcds-q85-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q85-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part15/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part15/canondata/result.json index d66e6c4b28f7..01f87b364b35 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part15/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part15/canondata/result.json @@ -1916,9 +1916,9 @@ ], "test.test[pg-tpcds-q44-default.txt-Debug]": [ { - "checksum": "f781de5129e203e7afd029f9b696163d", - "size": 915, - "uri": "https://{canondata_backend}/1923547/61458c9c64b8429a1ff4c80acb29f295ac160173/resource.tar.gz#test.test_pg-tpcds-q44-default.txt-Debug_/opt.yql" + "checksum": "abde66ed3cf02637a0ec8ea6b3e3b48b", + "size": 552, + "uri": "https://{canondata_backend}/1942525/2e8c824a03404fb440cfd46e8109912264a486c5/resource.tar.gz#test.test_pg-tpcds-q44-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q44-default.txt-Plan]": [ @@ -1979,9 +1979,9 @@ ], "test.test[pg-tpcds-q62-default.txt-Debug]": [ { - "checksum": "1e17d4cd2338e7b5d6ae1cb9e58cc01f", - "size": 1192, - "uri": "https://{canondata_backend}/1937027/a40aacc25fa4e098523057cb2182143fb4f8e6c5/resource.tar.gz#test.test_pg-tpcds-q62-default.txt-Debug_/opt.yql" + "checksum": "a7c9b132432eefa74e2fa51483012323", + "size": 794, + "uri": "https://{canondata_backend}/1942525/2e8c824a03404fb440cfd46e8109912264a486c5/resource.tar.gz#test.test_pg-tpcds-q62-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q62-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part16/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part16/canondata/result.json index 65da0ff8a20d..8d6dc9206cb5 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part16/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part16/canondata/result.json @@ -1706,9 +1706,9 @@ ], "test.test[pg-order_by_agg_no_extra_for_aggs-default.txt-Debug]": [ { - "checksum": "ff75878b9f1ea5a060e5f5e70911f1e3", - "size": 1369, - "uri": "https://{canondata_backend}/1924537/d66bfe69aa802f6a81aadbac897621b721f31b4b/resource.tar.gz#test.test_pg-order_by_agg_no_extra_for_aggs-default.txt-Debug_/opt.yql" + "checksum": "bdfc023514c7b4e456376c4a5eb934ec", + "size": 1786, + "uri": "https://{canondata_backend}/1775319/5b279c997511dd7fcf1a1378154cb556c4e140e3/resource.tar.gz#test.test_pg-order_by_agg_no_extra_for_aggs-default.txt-Debug_/opt.yql" } ], "test.test[pg-order_by_agg_no_extra_for_aggs-default.txt-Plan]": [ @@ -1853,9 +1853,9 @@ ], "test.test[pg-tpcds-q83-default.txt-Debug]": [ { - "checksum": "f14f80a4f82ec02d4ca272c0a7def24f", - "size": 1182, - "uri": "https://{canondata_backend}/1924537/d66bfe69aa802f6a81aadbac897621b721f31b4b/resource.tar.gz#test.test_pg-tpcds-q83-default.txt-Debug_/opt.yql" + "checksum": "ce0aafc4a5a04056f7a7bb426f6444db", + "size": 766, + "uri": "https://{canondata_backend}/1773845/103dfa47dc89a09820dbbf93b83e760c81718f14/resource.tar.gz#test.test_pg-tpcds-q83-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q83-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json index 76ea24072521..1321cb67aa00 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part17/canondata/result.json @@ -1466,9 +1466,9 @@ ], "test.test[pg-order_by_agg_input_columns_prj_and_aggr-default.txt-Debug]": [ { - "checksum": "788cb8868f36367af1f04d5a68977375", - "size": 1506, - "uri": "https://{canondata_backend}/1871002/41480761fc88d03cb31c849ad940528337d0527e/resource.tar.gz#test.test_pg-order_by_agg_input_columns_prj_and_aggr-default.txt-Debug_/opt.yql" + "checksum": "63291ef082824d652a812d5f07cab745", + "size": 1536, + "uri": "https://{canondata_backend}/937458/0d496f55667fc85c1c096d3350a82ebeff0342bb/resource.tar.gz#test.test_pg-order_by_agg_input_columns_prj_and_aggr-default.txt-Debug_/opt.yql" } ], "test.test[pg-order_by_agg_input_columns_prj_and_aggr-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json index f3183d7310c0..b5efe2a5f6a4 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part3/canondata/result.json @@ -1531,9 +1531,9 @@ ], "test.test[pg-tpcds-q79-default.txt-Debug]": [ { - "checksum": "ed197497e067d749f703639cae711943", - "size": 1280, - "uri": "https://{canondata_backend}/1600758/4bccd141523a28df31ef68c43bfce4e16b831213/resource.tar.gz#test.test_pg-tpcds-q79-default.txt-Debug_/opt.yql" + "checksum": "4621819a480fc9db86d1b1c5e965277c", + "size": 787, + "uri": "https://{canondata_backend}/1773845/d550090c4ec31d83d30327ecec7b3ed650bd4d50/resource.tar.gz#test.test_pg-tpcds-q79-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q79-default.txt-Plan]": [ @@ -1573,9 +1573,9 @@ ], "test.test[pg-tpcds-q84-default.txt-Debug]": [ { - "checksum": "acf217080022c3718a299d663a59723f", - "size": 803, - "uri": "https://{canondata_backend}/1942278/dcf51f87e7494622f2c5e661f1ef8639433dd314/resource.tar.gz#test.test_pg-tpcds-q84-default.txt-Debug_/opt.yql" + "checksum": "e99f7e00b21b522ddba0c2c1c39a8ed5", + "size": 500, + "uri": "https://{canondata_backend}/1773845/d550090c4ec31d83d30327ecec7b3ed650bd4d50/resource.tar.gz#test.test_pg-tpcds-q84-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q84-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part4/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part4/canondata/result.json index 5142306f014c..aa912861e968 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part4/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part4/canondata/result.json @@ -1867,6 +1867,27 @@ "uri": "https://{canondata_backend}/1946324/2fb8464e4bbcec8b36db266be55bc3df5e6c0f9d/resource.tar.gz#test.test_sampling-bind_expr_subquery-default.txt-Results_/results.txt" } ], + "test.test[sampling-sample-default.txt-Debug]": [ + { + "checksum": "23d2be55fde66e15d2ff0aaac1698ccc", + "size": 2586, + "uri": "https://{canondata_backend}/1871002/a516d461b35f408b4209e3090c9eba63de2e26fd/resource.tar.gz#test.test_sampling-sample-default.txt-Debug_/opt.yql" + } + ], + "test.test[sampling-sample-default.txt-Plan]": [ + { + "checksum": "7c71c97a650faa5f2d391aee871cc4a3", + "size": 7322, + "uri": "https://{canondata_backend}/1871002/a516d461b35f408b4209e3090c9eba63de2e26fd/resource.tar.gz#test.test_sampling-sample-default.txt-Plan_/plan.txt" + } + ], + "test.test[sampling-sample-default.txt-Results]": [ + { + "checksum": "62a1b10336d75805df5fa86407aaaeb6", + "size": 3850, + "uri": "https://{canondata_backend}/1871002/a516d461b35f408b4209e3090c9eba63de2e26fd/resource.tar.gz#test.test_sampling-sample-default.txt-Results_/results.txt" + } + ], "test.test[sampling-sort-default.txt-Debug]": [ { "checksum": "fa6ec83007ee8120eed239c7511ad4b7", diff --git a/ydb/library/yql/tests/sql/yt_native_file/part5/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part5/canondata/result.json index ea4c34f7935e..18128d2d9cab 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part5/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part5/canondata/result.json @@ -2014,9 +2014,9 @@ ], "test.test[pg-order_by_agg_input_columns_prj_and_keys_and_aggr-default.txt-Debug]": [ { - "checksum": "9307836482ee1436af14528f1e6a50d9", - "size": 1545, - "uri": "https://{canondata_backend}/1924537/a216c900c9110aa2e7a4c99e87da2ae19ca0813a/resource.tar.gz#test.test_pg-order_by_agg_input_columns_prj_and_keys_and_aggr-default.txt-Debug_/opt.yql" + "checksum": "d07a26a8659feba81ee43b48dd1d6f74", + "size": 1615, + "uri": "https://{canondata_backend}/1942525/17dd5a27a5a62c4bb16e33971d7a3b163c0c8216/resource.tar.gz#test.test_pg-order_by_agg_input_columns_prj_and_keys_and_aggr-default.txt-Debug_/opt.yql" } ], "test.test[pg-order_by_agg_input_columns_prj_and_keys_and_aggr-default.txt-Plan]": [ @@ -2203,9 +2203,9 @@ ], "test.test[pg-tpcds-q06-default.txt-Debug]": [ { - "checksum": "f934537b05e7f69a90d6779d0bb6778c", - "size": 873, - "uri": "https://{canondata_backend}/1784826/067f0538333fbac0ecdfa1460dd6f368fda04e05/resource.tar.gz#test.test_pg-tpcds-q06-default.txt-Debug_/opt.yql" + "checksum": "0e1b3c5fd5f26480bd0477abc5a8005d", + "size": 504, + "uri": "https://{canondata_backend}/1889210/2a9582ee728110f97ddab03e6301b3d60352b05e/resource.tar.gz#test.test_pg-tpcds-q06-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q06-default.txt-Plan]": [ @@ -2224,9 +2224,9 @@ ], "test.test[pg-tpcds-q11-default.txt-Debug]": [ { - "checksum": "370f002acc351f5acae85dea2c3d148b", - "size": 1228, - "uri": "https://{canondata_backend}/1814674/9a988bc8430652cef5b9d26ea0030620b6cb96f7/resource.tar.gz#test.test_pg-tpcds-q11-default.txt-Debug_/opt.yql" + "checksum": "79e46486f98144a3637143603ae24b04", + "size": 703, + "uri": "https://{canondata_backend}/1889210/2a9582ee728110f97ddab03e6301b3d60352b05e/resource.tar.gz#test.test_pg-tpcds-q11-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q11-default.txt-Plan]": [ @@ -2350,9 +2350,9 @@ ], "test.test[pg-tpcds-q91-default.txt-Debug]": [ { - "checksum": "936033f9091dae3cd007696ff112ff3a", - "size": 653, - "uri": "https://{canondata_backend}/1784826/067f0538333fbac0ecdfa1460dd6f368fda04e05/resource.tar.gz#test.test_pg-tpcds-q91-default.txt-Debug_/opt.yql" + "checksum": "62e6b80afed99165d2d612238e144c08", + "size": 603, + "uri": "https://{canondata_backend}/1889210/2a9582ee728110f97ddab03e6301b3d60352b05e/resource.tar.gz#test.test_pg-tpcds-q91-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q91-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part6/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part6/canondata/result.json index dd305b77a4d3..4ba9dea363ba 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part6/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part6/canondata/result.json @@ -1903,9 +1903,9 @@ ], "test.test[pg-tpcds-q52-default.txt-Debug]": [ { - "checksum": "0abd38ef2d6d532608c27c9b48dcd778", - "size": 1008, - "uri": "https://{canondata_backend}/1936842/5d6d393872cda475b6b05e0b26199ad4ccdbcef7/resource.tar.gz#test.test_pg-tpcds-q52-default.txt-Debug_/opt.yql" + "checksum": "14317455f4ca9be1725a5b9844f5468a", + "size": 674, + "uri": "https://{canondata_backend}/1600758/c5ea854dabc84a4e630a2d65d117d9d6d55a5034/resource.tar.gz#test.test_pg-tpcds-q52-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q52-default.txt-Plan]": [ @@ -1966,9 +1966,9 @@ ], "test.test[pg-tpcds-q92-default.txt-Debug]": [ { - "checksum": "f04da7838bbf61d9c3f3e023f6d1ca73", - "size": 1657, - "uri": "https://{canondata_backend}/1689644/4f33cb0e4ea8c1f718492ab219f9586a69123ad0/resource.tar.gz#test.test_pg-tpcds-q92-default.txt-Debug_/opt.yql" + "checksum": "75d0c437159be5511d63aba39dd14b0b", + "size": 1557, + "uri": "https://{canondata_backend}/1600758/c5ea854dabc84a4e630a2d65d117d9d6d55a5034/resource.tar.gz#test.test_pg-tpcds-q92-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q92-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part7/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part7/canondata/result.json index fb75d2cce4cb..bd825efc1c32 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part7/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part7/canondata/result.json @@ -1590,6 +1590,27 @@ "uri": "https://{canondata_backend}/1871002/96825674a144d70463fcbff8c8e43e73b4243cee/resource.tar.gz#test.test_pg-def_column_name_func-default.txt-Results_/results.txt" } ], + "test.test[pg-order_by_distinct_same_expr-default.txt-Debug]": [ + { + "checksum": "11462e4fa00fdd30ef84dabcef1f78dd", + "size": 584, + "uri": "https://{canondata_backend}/1871182/908119bbb1a3765078e40e5b34c5001ed9c06152/resource.tar.gz#test.test_pg-order_by_distinct_same_expr-default.txt-Debug_/opt.yql" + } + ], + "test.test[pg-order_by_distinct_same_expr-default.txt-Plan]": [ + { + "checksum": "b4dd508a329723c74293d80f0278c705", + "size": 505, + "uri": "https://{canondata_backend}/1871182/908119bbb1a3765078e40e5b34c5001ed9c06152/resource.tar.gz#test.test_pg-order_by_distinct_same_expr-default.txt-Plan_/plan.txt" + } + ], + "test.test[pg-order_by_distinct_same_expr-default.txt-Results]": [ + { + "checksum": "3e7b812a98a46b478e9625c31f3f37f1", + "size": 946, + "uri": "https://{canondata_backend}/1871182/908119bbb1a3765078e40e5b34c5001ed9c06152/resource.tar.gz#test.test_pg-order_by_distinct_same_expr-default.txt-Results_/results.txt" + } + ], "test.test[pg-pg_array_literal-default.txt-Debug]": [ { "checksum": "8e2d2873e527f3f86e85b8871162a38b", @@ -1760,9 +1781,9 @@ ], "test.test[pg-tpcds-q04-default.txt-Debug]": [ { - "checksum": "370f002acc351f5acae85dea2c3d148b", - "size": 1228, - "uri": "https://{canondata_backend}/1925842/001f28c83813fe9804830e9051ae0e479ec3973a/resource.tar.gz#test.test_pg-tpcds-q04-default.txt-Debug_/opt.yql" + "checksum": "79e46486f98144a3637143603ae24b04", + "size": 703, + "uri": "https://{canondata_backend}/1773845/136823a96bd1f6b8c50a5ab89981023976f42353/resource.tar.gz#test.test_pg-tpcds-q04-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q04-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part8/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part8/canondata/result.json index 1ee78733070d..dbd45fd1ab34 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part8/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part8/canondata/result.json @@ -2242,9 +2242,9 @@ ], "test.test[pg-tpcds-q94-default.txt-Debug]": [ { - "checksum": "bc8a00ea43d88fec0663cb0c2dc3338c", - "size": 3242, - "uri": "https://{canondata_backend}/1942173/229bba45f52f90f7fcfe71ac254ac6d4bb0114fe/resource.tar.gz#test.test_pg-tpcds-q94-default.txt-Debug_/opt.yql" + "checksum": "6739e519f2684c6306ba2c3f563b24a4", + "size": 3055, + "uri": "https://{canondata_backend}/1924537/127b61f15a1857cd28e79bf429fba30043232745/resource.tar.gz#test.test_pg-tpcds-q94-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q94-default.txt-Plan]": [ diff --git a/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json b/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json index 2a119ccde2d9..7dee8617f8ca 100644 --- a/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json +++ b/ydb/library/yql/tests/sql/yt_native_file/part9/canondata/result.json @@ -1559,6 +1559,27 @@ "uri": "https://{canondata_backend}/1942525/e650a9108a46215c1dbecfcc5ac1f7eea9025945/resource.tar.gz#test.test_pg-aggregate_factory-default.txt-Results_/results.txt" } ], + "test.test[pg-order_by_distinct_same_expr_agg-default.txt-Debug]": [ + { + "checksum": "a11be06aa510df5c4426828d173ae2db", + "size": 1595, + "uri": "https://{canondata_backend}/1775059/3dc32f45e46d436c6117f9365f68660a04a7a9ef/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg-default.txt-Debug_/opt.yql" + } + ], + "test.test[pg-order_by_distinct_same_expr_agg-default.txt-Plan]": [ + { + "checksum": "b4dd508a329723c74293d80f0278c705", + "size": 505, + "uri": "https://{canondata_backend}/1775059/3dc32f45e46d436c6117f9365f68660a04a7a9ef/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg-default.txt-Plan_/plan.txt" + } + ], + "test.test[pg-order_by_distinct_same_expr_agg-default.txt-Results]": [ + { + "checksum": "56933ce9c5e062393a84cf42021ed093", + "size": 1236, + "uri": "https://{canondata_backend}/1775059/3dc32f45e46d436c6117f9365f68660a04a7a9ef/resource.tar.gz#test.test_pg-order_by_distinct_same_expr_agg-default.txt-Results_/results.txt" + } + ], "test.test[pg-order_by_shadow_input_columns-default.txt-Debug]": [ { "checksum": "386c850c4ca0a581b365d1d9a47796af", @@ -1813,9 +1834,9 @@ ], "test.test[pg-tpcds-q42-default.txt-Debug]": [ { - "checksum": "787ce8c8d9198dc9b2f9dae6f0e0b97b", - "size": 1252, - "uri": "https://{canondata_backend}/1903885/d7489869939f5eed8aa352f8fd72d52a3c467e5a/resource.tar.gz#test.test_pg-tpcds-q42-default.txt-Debug_/opt.yql" + "checksum": "33556fcd6e2a28fce5eb71a039d015d7", + "size": 711, + "uri": "https://{canondata_backend}/1871182/e14ddf4aced6b94cffd65d03b6a9f851e5b5f624/resource.tar.gz#test.test_pg-tpcds-q42-default.txt-Debug_/opt.yql" } ], "test.test[pg-tpcds-q42-default.txt-Plan]": [ diff --git a/ydb/public/api/protos/ydb_table.proto b/ydb/public/api/protos/ydb_table.proto index e878cb5bdf17..295b67c12777 100644 --- a/ydb/public/api/protos/ydb_table.proto +++ b/ydb/public/api/protos/ydb_table.proto @@ -333,6 +333,16 @@ message TableProfile { CachingPolicy caching_policy = 7; } +message SequenceDescription { + optional string name = 1; // mandatorys + optional sint64 min_value = 2; // minimum value, defaults to 1 or Min + optional sint64 max_value = 3; // maximum value, defaults to Max or -1 + optional sint64 start_value = 4; // start value, defaults to min_value + optional uint64 cache = 5; // number of items to cache, defaults to 1 + optional sint64 increment = 6; // increment at each call, defaults to 1 + optional bool cycle = 7; // true when cycle on overflow is allowed +} + message ColumnMeta { // Name of column string name = 1; @@ -345,6 +355,7 @@ message ColumnMeta { // Column default value option oneof default_value { TypedValue from_literal = 5; + SequenceDescription from_sequence = 6; } } diff --git a/ydb/services/datastreams/datastreams_ut.cpp b/ydb/services/datastreams/datastreams_ut.cpp index 64df2d23a0d5..a5bfd8aca19c 100644 --- a/ydb/services/datastreams/datastreams_ut.cpp +++ b/ydb/services/datastreams/datastreams_ut.cpp @@ -1513,25 +1513,25 @@ Y_UNIT_TEST_SUITE(DataStreams) { } UNIT_ASSERT_VALUES_EQUAL(result.GetResult().failed_record_count(), 0); Cerr << result.GetResult().DebugString() << Endl; - Cerr << "Second put records\n"; + Cerr << "Second put records (async)\n"; + auto secondWriteAsync = client.PutRecords(streamPath, records); + + Cerr << Now().Seconds() << "Third put records\n"; result = client.PutRecords(streamPath, records).ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); if (result.GetStatus() != EStatus::SUCCESS) { result.GetIssues().PrintTo(Cerr); } - Cerr << result.GetResult().DebugString() << Endl; UNIT_ASSERT_VALUES_EQUAL(result.GetResult().failed_record_count(), 4); UNIT_ASSERT_VALUES_EQUAL(result.GetResult().records(0).error_code(), "ProvisionedThroughputExceededException"); - Sleep(TDuration::Seconds(4)); - - Cerr << "Third put records\n"; - result = client.PutRecords(streamPath, records).ExtractValueSync(); + result = secondWriteAsync.ExtractValueSync(); UNIT_ASSERT_VALUES_EQUAL(result.IsTransportError(), false); if (result.GetStatus() != EStatus::SUCCESS) { result.GetIssues().PrintTo(Cerr); } + Cerr << result.GetResult().DebugString() << Endl; UNIT_ASSERT_VALUES_EQUAL(result.GetResult().failed_record_count(), 0); Cerr << result.GetResult().DebugString() << Endl; diff --git a/ydb/services/persqueue_v1/persqueue_ut.cpp b/ydb/services/persqueue_v1/persqueue_ut.cpp index a223ec34ff89..312604fe48c2 100644 --- a/ydb/services/persqueue_v1/persqueue_ut.cpp +++ b/ydb/services/persqueue_v1/persqueue_ut.cpp @@ -788,8 +788,8 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { public: std::unique_ptr> ControlStream; std::unique_ptr> ReadStream; - TString SessionId; - + TString SessionId; + TDirectReadTestSetup(TPersQueueV1TestServer& server) : ReadContext(MakeHolder()) { @@ -830,7 +830,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { Cerr << "Got init response: " << resp.ShortDebugString() << Endl; UNIT_ASSERT(resp.server_message_case() == Ydb::Topic::StreamReadMessage::FromServer::kInitResponse); SessionId = resp.init_response().session_id(); - + req.Clear(); req.mutable_read_request()->set_bytes_size(40_MB); if (!ControlStream->Write(req)) { @@ -862,7 +862,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { } void DoWrite(NYdb::TDriver* driver, const TString& topic, ui64 size, ui32 count, - const TString& srcId = "srcID", const std::optional& partGroup = {}) + const TString& srcId = "srcID", const std::optional& partGroup = {}) { auto writer = CreateSimpleWriter(*driver, topic, srcId, partGroup, {"raw"}); @@ -880,7 +880,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { while (currTotalMessages < messageLimit) { Cerr << "Wait for direct read id: " << nextReadId << ", currently have " << currTotalMessages << " messages" << Endl; Ydb::Topic::StreamDirectReadMessage::FromServer resp; - UNIT_ASSERT(ReadStream->Read(&resp)); + UNIT_ASSERT(ReadStream->Read(&resp)); Cerr << "Got direct read response: " << resp.direct_read_response().direct_read_id() << Endl; UNIT_ASSERT_C(resp.status() == Ydb::StatusIds::SUCCESS, resp.DebugString()); UNIT_ASSERT(resp.server_message_case() == Ydb::Topic::StreamDirectReadMessage::FromServer::kDirectReadResponse); @@ -950,14 +950,14 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { auto resp = runtime->GrabEdgeEvent(); UNIT_ASSERT(resp); return resp; - } + } Y_UNIT_TEST(DirectReadPreCached) { TPersQueueV1TestServer server{true}; SET_LOCALS; TDirectReadTestSetup setup{server}; setup.DoWrite(pqClient->GetDriver(), "acc/topic1", 1_MB, 30); - + setup.InitControlSession("acc/topic1"); auto pair = setup.GetNextAssign("acc/topic1"); UNIT_ASSERT_VALUES_EQUAL(pair.first, 0); @@ -971,7 +971,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { ui32 totalMsg = 0; ui64 nextReadId = 1; setup.DoRead(assignId, nextReadId, totalMsg, 30); - + Sleep(TDuration::Seconds(1)); cachedData = RequestCacheData(runtime, new TEvPQ::TEvGetFullDirectReadData()); UNIT_ASSERT_VALUES_EQUAL(cachedData->Data.size(), 1); @@ -984,7 +984,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { TPersQueueV1TestServer server{true}; SET_LOCALS; TDirectReadTestSetup setup{server}; - + setup.InitControlSession("acc/topic1"); auto pair = setup.GetNextAssign("acc/topic1"); UNIT_ASSERT_VALUES_EQUAL(pair.first, 0); @@ -1050,7 +1050,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { TDirectReadTestSetup setup{server}; setup.DoWrite(pqClient->GetDriver(), "acc/topic1", 100_KB, 1, "src1", 0); setup.DoWrite(pqClient->GetDriver(), "acc/topic1", 100_KB, 1, "src2", 1); - + setup.InitControlSession("acc/topic1"); auto pair1 = setup.GetNextAssign("acc/topic1"); auto pair2 = setup.GetNextAssign("acc/topic1"); @@ -1067,7 +1067,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { for (auto i = 0u; i != 2; ++i) { Cerr << "Wait for direct read" << Endl; Ydb::Topic::StreamDirectReadMessage::FromServer resp; - UNIT_ASSERT(setup.ReadStream->Read(&resp)); + UNIT_ASSERT(setup.ReadStream->Read(&resp)); Cerr << "Got direct read response: " << resp.direct_read_response().direct_read_id() << Endl; UNIT_ASSERT_C(resp.status() == Ydb::StatusIds::SUCCESS, resp.DebugString()); UNIT_ASSERT(resp.server_message_case() == Ydb::Topic::StreamDirectReadMessage::FromServer::kDirectReadResponse); @@ -1143,7 +1143,7 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { TDirectReadTestSetup setup{server}; - + setup.InitControlSession("acc/topic2"); setup.InitReadSession("acc/topic2"); auto pair = setup.GetNextAssign("acc/topic2"); @@ -1154,8 +1154,8 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { // UNIT_ASSERT_VALUES_EQUAL(cachedData->Data.size(), 1); setup.DoWrite(pqClient->GetDriver(), "acc/topic2", 10_MB, 1); Ydb::Topic::StreamDirectReadMessage::FromServer resp; - Cerr << "Request initial read data\n"; - UNIT_ASSERT(setup.ReadStream->Read(&resp)); + Cerr << "Request initial read data\n"; + UNIT_ASSERT(setup.ReadStream->Read(&resp)); Cerr << "Request cache data\n"; auto cachedData = RequestCacheData(runtime, new TEvPQ::TEvGetFullDirectReadData()); @@ -2384,7 +2384,9 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { } Y_UNIT_TEST(WriteExistingBigValue) { - NPersQueue::TTestServer server(PQSettings(0).SetDomainName("Root").SetNodeCount(2)); + auto settings = PQSettings(0).SetDomainName("Root").SetNodeCount(2); + settings.PQConfig.MutableQuotingConfig()->SetEnableQuoting(true); + NPersQueue::TTestServer server{settings}; server.EnableLogs({ NKikimrServices::FLAT_TX_SCHEMESHARD, NKikimrServices::PERSQUEUE }); server.AnnoyingClient->CreateTopic(DEFAULT_TOPIC_NAME, 2, 8_MB, 86400, 100000); @@ -3614,7 +3616,10 @@ Y_UNIT_TEST_SUITE(TPersQueueTest) { UNIT_ASSERT(equal); }; - NPersQueue::TTestServer server(PQSettings(0, 1, "10"), false); + auto settings = PQSettings(0, 1, "10"); + settings.PQConfig.MutableQuotingConfig()->SetEnableQuoting(true); + + NPersQueue::TTestServer server{settings, false}; auto netDataUpdated = server.PrepareNetDataFile(FormNetData()); UNIT_ASSERT(netDataUpdated); server.StartServer(); diff --git a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp index 62d58022b989..4577b1b3676e 100644 --- a/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp +++ b/ydb/services/persqueue_v1/ut/rate_limiter_test_setup.cpp @@ -130,7 +130,7 @@ void TRateLimiterTestSetup::InitServer(bool enableReadQuoting) { NKikimrServices::QUOTER_SERVICE, NKikimrServices::QUOTER_PROXY, NKikimrServices::KESUS_TABLET, - NKikimrServices::PQ_READ_SPEED_LIMITER + NKikimrServices::PQ_RATE_LIMITER }, NActors::NLog::PRI_TRACE ); diff --git a/ydb/services/ydb/ydb_stats_ut.cpp b/ydb/services/ydb/ydb_stats_ut.cpp index d5c57962404e..67b68961b4fc 100644 --- a/ydb/services/ydb/ydb_stats_ut.cpp +++ b/ydb/services/ydb/ydb_stats_ut.cpp @@ -222,6 +222,56 @@ static NYdb::TStatus SimpleSelect(TSession session, const TString& query) { } Y_UNIT_TEST_SUITE(ClientStatsCollector) { + Y_UNIT_TEST(PrepareQuery) { + NYdb::TKikimrWithGrpcAndRootSchema server; + auto endpoint = TStringBuilder() << "localhost:" << server.GetPort(); + NYdb::TDriver driver(NYdb::TDriverConfig().SetEndpoint(endpoint)); + TCountersExtractor extractor; + driver.AddExtension(TCountersExtractExtension::TParams().SetExtractor(&extractor)); + auto clSettings = NYdb::NTable::TClientSettings().UseQueryCache(true); + NYdb::NTable::TTableClient client(driver, clSettings); + + auto createSessionResult = client.GetSession(TCreateSessionSettings().ClientTimeout(OPERATION_TIMEOUT)).GetValueSync(); + UNIT_ASSERT(createSessionResult.IsSuccess()); + + auto session = createSessionResult.GetSession(); + + { + auto prepareResult = session.PrepareDataQuery("SELECT 1").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(prepareResult.IsQueryFromCache(), false); + + TStatCounters counters = extractor.Extract(); + UNIT_ASSERT_VALUES_EQUAL(counters.CacheMiss, 1); + + UNIT_ASSERT(prepareResult.IsSuccess()); + + { + auto executeResult = prepareResult.GetQuery().Execute( + TTxControl::BeginTx(TTxSettings::SerializableRW())).GetValueSync(); + + UNIT_ASSERT(executeResult.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(executeResult.IsQueryFromCache(), false); // <- explicit prepared query + counters = extractor.Extract(); + UNIT_ASSERT_VALUES_EQUAL(counters.CacheMiss, 1); + } + + auto executeResult = session.ExecuteDataQuery("SELECT 1", + TTxControl::BeginTx(TTxSettings::SerializableRW()).CommitTx()).GetValueSync(); + + UNIT_ASSERT(executeResult.IsSuccess()); + UNIT_ASSERT_VALUES_EQUAL(executeResult.IsQueryFromCache(), true); + counters = extractor.Extract(); + UNIT_ASSERT_VALUES_EQUAL(counters.CacheMiss, 1); + } + + { + auto prepareResult = session.PrepareDataQuery("SELECT 1").GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(prepareResult.IsQueryFromCache(), true); + auto counters = extractor.Extract(); + UNIT_ASSERT_VALUES_EQUAL(counters.CacheMiss, 1); + } + } + Y_UNIT_TEST(CounterCacheMiss) { NYdb::TKikimrWithGrpcAndRootSchema server; auto endpoint = TStringBuilder() << "localhost:" << server.GetPort(); diff --git a/ydb/tests/fq/s3/test_s3.py b/ydb/tests/fq/s3/test_s3.py index 96f27358bd3b..50a0033e76bf 100644 --- a/ydb/tests/fq/s3/test_s3.py +++ b/ydb/tests/fq/s3/test_s3.py @@ -83,6 +83,56 @@ def test_csv(self, kikimr, s3, client, runtime_listing, yq_version): assert result_set.rows[2].items[2].int32_value == 33 assert sum(kikimr.control_plane.get_metering()) == 10 + @yq_all + @pytest.mark.parametrize("client", [{"folder_id": "my_folder"}], indirect=True) + def test_csv_with_hopping(self, kikimr, s3, client): + resource = boto3.resource( + "s3", + endpoint_url=s3.s3_url, + aws_access_key_id="key", + aws_secret_access_key="secret_key" + ) + + bucket = resource.Bucket("fbucket") + bucket.create(ACL='public-read') + bucket.objects.all().delete() + + s3_client = boto3.client( + "s3", + endpoint_url=s3.s3_url, + aws_access_key_id="key", + aws_secret_access_key="secret_key" + ) + + fruits = R'''Time,Fruit,Price +0,Banana,3 +1,Apple,2 +2,Pear,15''' + s3_client.put_object(Body=fruits, Bucket='fbucket', Key='fruits.csv', ContentType='text/plain') + kikimr.control_plane.wait_bootstrap(1) + client.create_storage_connection("fruitbucket", "fbucket") + + sql = R''' + SELECT COUNT(*) as count, + FROM fruitbucket.`fruits.csv` + WITH (format=csv_with_names, SCHEMA ( + Time UInt64 NOT NULL, + Fruit String NOT NULL, + Price Int NOT NULL + )) + GROUP BY HOP(CAST(Time AS Timestamp?), "PT1M", "PT1M", "PT1M") + ''' + + query_id = client.create_query("simple", sql, type=fq.QueryContent.QueryType.ANALYTICS).result.query_id + client.wait_query_status(query_id, fq.QueryMeta.COMPLETED) + + data = client.get_result_data(query_id) + result_set = data.result.result_set + logging.debug(str(result_set)) + assert len(result_set.columns) == 1 + assert len(result_set.rows) == 1 + assert result_set.rows[0].items[0].uint64_value == 3 + @yq_all @pytest.mark.parametrize("client", [{"folder_id": "my_folder"}], indirect=True) @pytest.mark.parametrize("runtime_listing", [False, True]) diff --git a/ydb/tests/functional/kqp/kqp_indexes/epilogue.cmake b/ydb/tests/functional/kqp/kqp_indexes/epilogue.cmake deleted file mode 100644 index 57089c5356b2..000000000000 --- a/ydb/tests/functional/kqp/kqp_indexes/epilogue.cmake +++ /dev/null @@ -1 +0,0 @@ -add_dependencies(ydb-tests-functional-kqp-kqp_indexes ydb_py_protos) diff --git a/ydb/tests/functional/kqp/kqp_query_session/epilogue.cmake b/ydb/tests/functional/kqp/kqp_query_session/epilogue.cmake deleted file mode 100644 index 7cdb02f44429..000000000000 --- a/ydb/tests/functional/kqp/kqp_query_session/epilogue.cmake +++ /dev/null @@ -1 +0,0 @@ -add_dependencies(ydb-tests-functional-kqp-kqp_query_session ydb_py_protos) diff --git a/ydb/tools/cfg/static.py b/ydb/tools/cfg/static.py index 2dbb9edeacc4..e6eaa5257e91 100644 --- a/ydb/tools/cfg/static.py +++ b/ydb/tools/cfg/static.py @@ -1208,7 +1208,11 @@ def get_uploader(uploader): span_export_timeout_seconds = uploader.get("span_export_timeout_seconds") if span_export_timeout_seconds is not None: - uploader_pb.SpanTtlSeconds = span_export_timeout_seconds + uploader_pb.SpanExportTimeoutSeconds = span_export_timeout_seconds + + max_export_requests_inflight = uploader.get("max_export_requests_inflight") + if max_export_requests_inflight is not None: + uploader_pb.MaxExportRequestsInflight = max_export_requests_inflight return uploader_pb diff --git a/ydb/tools/cfg/validation.py b/ydb/tools/cfg/validation.py index 3205c1dad69b..552b157c5b6b 100644 --- a/ydb/tools/cfg/validation.py +++ b/ydb/tools/cfg/validation.py @@ -176,9 +176,11 @@ max_spans_per_second=dict(type="integer", minimum=1), max_spans_in_batch=dict(type="integer", minimum=1), max_bytes_in_batch=dict(type="integer"), - max_bytes_accumulation_milliseconds=dict(type="integer"), + max_batch_accumulation_milliseconds=dict(type="integer"), span_export_timeout_seconds=dict(type="integer", minimum=1), + max_export_requests_inflight=dict(type="integer", minimum=1), ), + additionalProperties=False, ), sampling=dict( type="array",