switchable slices filter (#12791)

ydb-platform · Dec 19, 2024 · a39dc2a · a39dc2a
1 parent 737b0d5
commit a39dc2a
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 3 deletions.
diff --git a/ydb/core/protos/config.proto b/ydb/core/protos/config.proto
@@ -1737,6 +1737,7 @@ message TColumnShardConfig {
     optional string ReaderClassName = 28;
     optional bool AllowNullableColumnsInPK = 29 [default = false];
     optional uint32 RestoreDataOnWriteTimeoutSeconds = 30;
+    optional bool UseSlicesFilter = 31 [default = true];
 }
 
 message TSchemeShardConfig {

diff --git a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetched_data.h b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetched_data.h
@@ -1,7 +1,9 @@
 #pragma once
+#include <ydb/core/base/appdata.h>
 #include <ydb/core/formats/arrow/arrow_filter.h>
 #include <ydb/core/formats/arrow/common/container.h>
 #include <ydb/core/formats/arrow/size_calcer.h>
+#include <ydb/core/protos/config.pb.h>
 #include <ydb/core/tx/columnshard/blob.h>
 #include <ydb/core/tx/columnshard/blobs_reader/task.h>
 #include <ydb/core/tx/columnshard/engines/portions/data_accessor.h>
@@ -146,7 +148,8 @@ class TFetchedData {
 
     void AddFilter(const NArrow::TColumnFilter& filter) {
         if (UseFilter && Table) {
-            AFL_VERIFY(filter.Apply(Table, NArrow::TColumnFilter::TApplyContext().SetTrySlices(true)));
+            AFL_VERIFY(filter.Apply(Table, 
+                NArrow::TColumnFilter::TApplyContext().SetTrySlices(!HasAppData() || AppDataVerified().ColumnShardConfig.GetUseSlicesFilter())));
         }
         if (!Filter) {
             Filter = std::make_shared<NArrow::TColumnFilter>(filter);
@@ -176,7 +179,8 @@ class TFetchedData {
         DataAdded = true;
         auto tableLocal = table;
         if (Filter && UseFilter) {
-            AFL_VERIFY(Filter->Apply(tableLocal, NArrow::TColumnFilter::TApplyContext().SetTrySlices(true)));
+            AFL_VERIFY(Filter->Apply(tableLocal, 
+                NArrow::TColumnFilter::TApplyContext().SetTrySlices(!HasAppData() || AppDataVerified().ColumnShardConfig.GetUseSlicesFilter())));
         }
         if (!Table) {
             Table = std::make_shared<NArrow::TGeneralContainer>(tableLocal);
@@ -246,4 +250,4 @@ class TFetchedResult {
     }
 };
 
-}   // namespace NKikimr::NOlap
+}   // namespace NKikimr::NOlap::NReader::NCommon