Merge 968900b into 59fa9fb

ydb-platform · Nov 21, 2024 · 07ee231 · 07ee231
2 parents 59fa9fb + 968900b
commit 07ee231
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 1 deletion.
diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto
@@ -1698,6 +1698,7 @@ message TColumnShardConfig {
     optional uint32 SmallPortionDetectSizeLimit = 24 [default = 1048576];  // 1 << 20
     optional bool ColumnChunksV0Usage = 25 [default = true];
     optional bool ColumnChunksV1Usage = 26 [default = true];
+    optional uint64 MemoryLimitScanPortion = 27 [default = 100000000];
 }
 
 message TSchemeShardConfig {

diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.h b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetched_data.h
@@ -36,6 +36,7 @@ class TFetchedData {
             return;
         }
         AFL_VERIFY(!DataAdded);
+        UseFilter = value;
     }
 
     bool HasPortionAccessor() const {

diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/fetching.cpp
@@ -289,7 +289,8 @@ TConclusion<bool> TPortionAccessorFetchingStep::DoExecuteInplace(
 
 TConclusion<bool> TDetectInMem::DoExecuteInplace(const std::shared_ptr<IDataSource>& source, const TFetchingScriptCursor& /*step*/) const {
     if (Columns.GetColumnsCount()) {
-        source->SetSourceInMemory(source->GetColumnRawBytes(Columns.GetColumnIds()) < 1e+8);
+        source->SetSourceInMemory(
+            source->GetColumnRawBytes(Columns.GetColumnIds()) < NYDBTest::TControllers::GetColumnShardController()->GetMemoryLimitScanPortion());
     } else {
         source->SetSourceInMemory(true);
     }

diff --git a/ydb/core/tx/columnshard/hooks/abstract/abstract.h b/ydb/core/tx/columnshard/hooks/abstract/abstract.h
@@ -130,6 +130,9 @@ class ICSController {
     virtual TDuration DoGetLagForCompactionBeforeTierings(const TDuration defaultValue) const {
         return defaultValue;
     }
+    virtual ui64 DoGetMemoryLimitScanPortion(const ui64 defaultValue) const {
+        return defaultValue;
+    }
 
 private:
     inline static const NKikimrConfig::TColumnShardConfig DefaultConfig = {};
@@ -146,6 +149,9 @@ class ICSController {
         const std::set<NOlap::TSnapshot>& /*snapshotsToSave*/, const std::set<NOlap::TSnapshot>& /*snapshotsToRemove*/) {
     }
 
+    ui64 GetMemoryLimitScanPortion() const {
+        return DoGetMemoryLimitScanPortion(GetConfig().GetMemoryLimitScanPortion());
+    }
     virtual bool CheckPortionForEvict(const NOlap::TPortionInfo& portion) const;
 
     TDuration GetPingCheckPeriod() const {

diff --git a/ydb/core/tx/columnshard/hooks/testing/controller.h b/ydb/core/tx/columnshard/hooks/testing/controller.h
@@ -22,6 +22,7 @@ class TController: public TReadOnlyController {
     YDB_ACCESSOR_DEF(std::optional<TDuration>, OverrideCompactionActualizationLag);
     YDB_ACCESSOR_DEF(std::optional<TDuration>, OverrideTasksActualizationLag);
     YDB_ACCESSOR_DEF(std::optional<TDuration>, OverrideReadTimeoutClean);
+    YDB_ACCESSOR(std::optional<ui64>, OverrideMemoryLimitForPortionReading, 100);
     EOptimizerCompactionWeightControl CompactionControl = EOptimizerCompactionWeightControl::Force;
 
     YDB_ACCESSOR(std::optional<ui64>, OverrideReduceMemoryIntervalLimit, 1024);
@@ -130,6 +131,11 @@ class TController: public TReadOnlyController {
     THashSet<TString> SharingIds;
 protected:
     virtual ::NKikimr::NColumnShard::TBlobPutResult::TPtr OverrideBlobPutResultOnCompaction(const ::NKikimr::NColumnShard::TBlobPutResult::TPtr original, const NOlap::TWriteActionsCollection& actions) const override;
+
+    virtual ui64 DoGetMemoryLimitScanPortion(const ui64 defaultValue) const override {
+        return OverrideMemoryLimitForPortionReading.value_or(defaultValue);
+    }
+
     virtual TDuration DoGetLagForCompactionBeforeTierings(const TDuration def) const override {
         return OverrideLagForCompactionBeforeTierings.value_or(def);
     }
-Original file line number
+Diff line change
@@ Expand Up / @@ -36,6 +36,7 @@ class TFetchedData { @@
                 return;
             }
             AFL_VERIFY(!DataAdded);
+            UseFilter = value;
         }
         bool HasPortionAccessor() const {
@@ Expand Down @@