From ccfc4978c1a0d1dccbf083506101cd8bf72d42ff Mon Sep 17 00:00:00 2001 From: Chengpeng Yan <41809508+Reminiscent@users.noreply.github.com> Date: Tue, 30 May 2023 21:38:16 +0800 Subject: [PATCH] [feature](nereids) support the rewrite rule for push-down filter through sort (#20161) Support the rewrite rule for push-down filter through sort. We can directly push-down the filter through sort without any conditions check. Before this PR: ``` mysql> explain select * from (select * from t1 order by a) t2 where t2.b > 2; +-------------------------------------------------------------+ | Explain String | +-------------------------------------------------------------+ | PLAN FRAGMENT 0 | | OUTPUT EXPRS: | | a[#2] | | b[#3] | | PARTITION: UNPARTITIONED | | | | VRESULT SINK | | | | 3:VSELECT | | | predicates: b[#3] > 2 | | | | | 2:VMERGING-EXCHANGE | | offset: 0 | | | | PLAN FRAGMENT 1 | | | | PARTITION: HASH_PARTITIONED: a[#0] | | | | STREAM DATA SINK | | EXCHANGE ID: 02 | | UNPARTITIONED | | | | 1:VTOP-N | | | order by: a[#2] ASC | | | offset: 0 | | | | | 0:VOlapScanNode | | TABLE: default_cluster:test.t1(t1), PREAGGREGATION: ON | | partitions=0/1, tablets=0/0, tabletList= | | cardinality=1, avgRowSize=0.0, numNodes=1 | +-------------------------------------------------------------+ 30 rows in set (0.06 sec) ``` After this PR: ``` mysql> explain select * from (select * from t1 order by a) t2 where t2.b > 2; +-------------------------------------------------------------+ | Explain String | +-------------------------------------------------------------+ | PLAN FRAGMENT 0 | | OUTPUT EXPRS: | | a[#2] | | b[#3] | | PARTITION: UNPARTITIONED | | | | VRESULT SINK | | | | 2:VMERGING-EXCHANGE | | offset: 0 | | | | PLAN FRAGMENT 1 | | | | PARTITION: HASH_PARTITIONED: a[#0] | | | | STREAM DATA SINK | | EXCHANGE ID: 02 | | UNPARTITIONED | | | | 1:VTOP-N | | | order by: a[#2] ASC | | | offset: 0 | | | | | 0:VOlapScanNode | | TABLE: default_cluster:test.t1(t1), PREAGGREGATION: ON | | PREDICATES: b[#1] > 2 | | partitions=0/1, tablets=0/0, tabletList= | | cardinality=1, avgRowSize=0.0, numNodes=1 | +-------------------------------------------------------------+ 28 rows in set (0.40 sec) ``` --- .../apache/doris/nereids/rules/RuleSet.java | 2 + .../apache/doris/nereids/rules/RuleType.java | 1 + .../logical/PushdownFilterThroughSort.java | 43 +++++++++++++ .../PushdownFilterThroughSortTest.java | 64 +++++++++++++++++++ 4 files changed, 110 insertions(+) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSort.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSortTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index 169f556225a3cd..049680ee3f1102 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -80,6 +80,7 @@ import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughProject; import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughRepeat; import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughSetOperation; +import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughSort; import org.apache.doris.nereids.rules.rewrite.logical.PushdownFilterThroughWindow; import org.apache.doris.nereids.rules.rewrite.logical.PushdownJoinOtherCondition; import org.apache.doris.nereids.rules.rewrite.logical.PushdownProjectThroughLimit; @@ -113,6 +114,7 @@ public class RuleSet { public static final List PUSH_DOWN_FILTERS = ImmutableList.of( new PushdownFilterThroughProject(), + new PushdownFilterThroughSort(), new PushdownJoinOtherCondition(), new PushdownFilterThroughJoin(), new PushdownExpressionsInHashCondition(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 3db4863e07427b..b3006f799fe43f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -138,6 +138,7 @@ public enum RuleType { PUSHDOWN_PROJECT_THROUGH_LIMIT(RuleTypeClass.REWRITE), PUSHDOWN_ALIAS_THROUGH_JOIN(RuleTypeClass.REWRITE), PUSHDOWN_FILTER_THROUGH_SET_OPERATION(RuleTypeClass.REWRITE), + PUSHDOWN_FILTER_THROUGH_SORT(RuleTypeClass.REWRITE), PUSHDOWN_FILTER_THROUGH_CTE_ANCHOR(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSort.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSort.java new file mode 100644 index 00000000000000..c3bccfde5f14fb --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSort.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.logical; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; + +/** + * Push down filter through sort. + * input: + * filter -> sort + * output: + * sort -> filter + * The filter can be directly push down to the sort. + */ +public class PushdownFilterThroughSort extends OneRewriteRuleFactory { + @Override + public Rule build() { + return logicalFilter(logicalSort()).then(filter -> { + LogicalSort sort = filter.child(); + return sort.withChildren(new LogicalFilter<>(filter.getConjuncts(), sort.child())); + }).toRule(RuleType.PUSHDOWN_FILTER_THROUGH_SORT); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSortTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSortTest.java new file mode 100644 index 00000000000000..0ce850262ef401 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/PushdownFilterThroughSortTest.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.logical; + +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.util.LogicalPlanBuilder; +import org.apache.doris.nereids.util.MemoPatternMatchSupported; +import org.apache.doris.nereids.util.MemoTestUtils; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.nereids.util.PlanConstructor; +import org.apache.doris.nereids.util.RelationUtil; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Test; + +import java.util.stream.Collectors; + +public class PushdownFilterThroughSortTest implements MemoPatternMatchSupported { + private final LogicalOlapScan scan = new LogicalOlapScan(RelationUtil.newRelationId(), PlanConstructor.student, + ImmutableList.of("")); + + @Test + void testPushdownFilterThroughSortTest() { + Slot gender = scan.getOutput().get(1); + + Expression filterPredicate = new GreaterThan(gender, Literal.of(1)); + + LogicalPlan plan = new LogicalPlanBuilder(scan) + .sort(scan.getOutput().stream().map(c -> new OrderKey(c, true, true)).collect(Collectors.toList())) + .filter(filterPredicate) + .build(); + + PlanChecker.from(MemoTestUtils.createConnectContext(), plan) + .applyTopDown(new PushdownFilterThroughSort()) + .matches( + logicalSort( + logicalFilter( + logicalOlapScan() + ) + ) + ); + } +}