From 963759a011e3401630b637ffe1fdd1dad2354861 Mon Sep 17 00:00:00 2001 From: An Qi Date: Sun, 5 Jan 2025 21:49:20 +0800 Subject: [PATCH] feat(core): add option to remove `RemoveNullColumn` operator (#536) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 一个折衷 --- conf/config.properties | 2 + .../optimizer/rules/AllowNullColumnRule.java | 51 +++++++ test/README.md | 2 +- .../func/optimizer/OptimizerIT.java | 142 ++++++++++-------- test/src/test/resources/testConfig.properties | 2 +- 5 files changed, 131 insertions(+), 68 deletions(-) create mode 100644 optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/AllowNullColumnRule.java diff --git a/conf/config.properties b/conf/config.properties index ad5cd7d85e..4061aba5f5 100644 --- a/conf/config.properties +++ b/conf/config.properties @@ -98,6 +98,8 @@ ruleBasedOptimizer=NotFilterRemoveRule=on,ColumnPruningRule=on,ConstantPropagati ConstantFoldingRule=on,FilterPushDownRule=on,JoinFactorizationRule=on,SetTransformPushDownPathUnionJoinRule=on,InFilterTransformRule=on,\ OuterJoinEliminateRule=on +#ruleBasedOptimizer=AllowNullColumnRule=on + ########################## ### 执行层配置 ########################## diff --git a/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/AllowNullColumnRule.java b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/AllowNullColumnRule.java new file mode 100644 index 0000000000..3363fe0482 --- /dev/null +++ b/optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/AllowNullColumnRule.java @@ -0,0 +1,51 @@ +/* + * IGinX - the polystore system with high performance + * Copyright (C) Tsinghua University + * TSIGinX@gmail.com + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 3 of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + */ +package cn.edu.tsinghua.iginx.logical.optimizer.rules; + +import cn.edu.tsinghua.iginx.engine.shared.operator.RemoveNullColumn; +import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource; +import cn.edu.tsinghua.iginx.engine.shared.source.SourceType; +import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall; +import com.google.auto.service.AutoService; + +@AutoService(Rule.class) +public class AllowNullColumnRule extends Rule { + + public AllowNullColumnRule() { + super( + AllowNullColumnRule.class.getSimpleName(), + operand(RemoveNullColumn.class, any()), + Long.MAX_VALUE, + RuleStrategy.FIXED_POINT); + } + + @Override + public boolean matches(RuleCall call) { + RemoveNullColumn removeNullColumn = (RemoveNullColumn) call.getMatchedRoot(); + return removeNullColumn.getSource().getType() == SourceType.Operator; + } + + @Override + public void onMatch(RuleCall call) { + RemoveNullColumn removeNullColumn = (RemoveNullColumn) call.getMatchedRoot(); + OperatorSource source = (OperatorSource) removeNullColumn.getSource(); + call.transformTo(source.getOperator()); + } +} diff --git a/test/README.md b/test/README.md index c72442f1eb..0715a6bbd9 100644 --- a/test/README.md +++ b/test/README.md @@ -12,7 +12,7 @@ IGinX代码测试框架于2023年完成重构，测试主要分为以下部分 - **`standalone-test`**：单机数据库功能测试 - 文件位于`integration/controller/Controller.java` - `testUnion()`方法负责在一组存储引擎上运行测试。首先加载测试配置文件，并提取要测试的存储引擎的相关特性。并将其支持的测试文件写入待测试列表中，然后调用脚本对数据库一一执行测试列表中的测试。 - - 目前测试的包括`SQLSessionIT,SQLSessionPoolIT,TagIT,RestAnnotationIT,RestIT,TransformIT,UDFIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT` + - 目前测试的包括`SQLSessionIT,OptimizerIT,SQLSessionPoolIT,TagIT,RestAnnotationIT,RestIT,TransformIT,UDFIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT` - **`unit-mds`**：元数据功能测试 - 文件位于`integration/mds/`下 - 分别针对ETCD和ZOOKEEPER进行测试 diff --git a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/optimizer/OptimizerIT.java b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/optimizer/OptimizerIT.java index af3710c053..2683de12d1 100644 --- a/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/optimizer/OptimizerIT.java +++ b/test/src/test/java/cn/edu/tsinghua/iginx/integration/func/optimizer/OptimizerIT.java @@ -22,7 +22,6 @@ import static cn.edu.tsinghua.iginx.integration.controller.Controller.SUPPORT_KEY; import static cn.edu.tsinghua.iginx.integration.controller.Controller.clearAllData; import static org.junit.Assert.*; -import static org.junit.Assert.assertTrue; import cn.edu.tsinghua.iginx.exception.SessionException; import cn.edu.tsinghua.iginx.integration.controller.Controller; @@ -33,7 +32,9 @@ import cn.edu.tsinghua.iginx.session.Session; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Pair; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -104,9 +105,8 @@ public OptimizerIT() { String rules = executor.execute("SHOW RULES;"); this.isOptimizerOpen = rules.contains("FilterPushDownRule| ON|"); - if (isOptimizerOpen) { - ruleList = getRuleList(); - } + Assume.assumeTrue(isOptimizerOpen); + ruleList = getRuleList(); } @BeforeClass @@ -178,6 +178,7 @@ public void closeAllRules() { } String statement = "set rules" + sb.substring(0, sb.length() - 1) + ";"; executor.execute(statement); + executor.execute("SET RULES AllowNullColumnRule=on;"); } @After @@ -188,6 +189,8 @@ public void openAllRules() { } String statement = "set rules" + sb.substring(0, sb.length() - 1) + ";"; executor.execute(statement); + executor.execute("SET RULES FragmentPruningByFilterRule=off;"); + executor.execute("SET RULES AllowNullColumnRule=off;"); } private void generateData(long start, long end) { @@ -244,11 +247,6 @@ public void clearData() { @Test public void testModifyRules() { - if (!isOptimizerOpen) { - LOGGER.info("Skip SQLSessionIT.testModifyRules because optimizer is not open"); - return; - } - String statement, expected; statement = "show rules;"; @@ -284,12 +282,6 @@ public void testModifyRules() { @Test public void testFilterPushDownExplain() { - // 临时修改 - if (!isOptimizerOpen) { - LOGGER.info("Skip SQLSessionIT.testFilterPushDownExplain because optimizer is not open"); - return; - } - executor.execute("SET RULES FilterPushDownRule=on;"); String insert = @@ -683,29 +675,16 @@ public void testFilterPushDownExplain() { expectRes = expectRes.replaceAll("&mark\\d+", "&mark").replaceAll(" ", ""); } - assertEquals(res, expectRes); + assertEquals(expectRes, res); } } @Test public void testFilterFragmentOptimizer() { - String policy = executor.execute("SHOW CONFIG \"policyClassName\";"); - if (!policy.contains("KeyRangeTestPolicy")) { - LOGGER.info( - "Skip SQLSessionIT.testFilterFragmentOptimizer because policy is not KeyRangeTestPolicy"); - return; - } - - if (!isOptimizerOpen) { - LOGGER.info( - "Skip SQLSessionIT.testFilterFragmentOptimizer because optimizer is not remove_not,filter_fragment"); - return; - } + Assume.assumeFalse(isScaling); - if (isScaling) { - LOGGER.info("Skip SQLSessionIT.testFilterFragmentOptimizer because it is scaling test"); - return; - } + String policy = executor.execute("SHOW CONFIG \"policyClassName\";"); + Assume.assumeTrue(policy.contains("KeyRangeTestPolicy")); String insert = "INSERT INTO us.d2(key, c) VALUES (1, \"asdas\"), (2, \"sadaa\"), (3, \"sadada\"), (4, \"asdad\"), (5, \"deadsa\"), (6, \"dasda\"), (7, \"asdsad\"), (8, \"frgsa\"), (9, \"asdad\");"; @@ -902,11 +881,7 @@ public void testFilterFragmentOptimizer() { @Test public void testColumnPruningAndFragmentPruning() { - if (!isOptimizerOpen || isScaling) { - LOGGER.info( - "Skip SQLSessionIT.testColumnPruningAndFragmentPruning because scaling test or filter push down test"); - return; - } + Assume.assumeFalse(isScaling); StringBuilder insert = new StringBuilder( @@ -1167,11 +1142,6 @@ public void testColumnPruningAndFragmentPruning() { @Test public void testConstantPropagation() { - if (!isOptimizerOpen) { - LOGGER.info("Skip SQLSessionIT.testConstantPropagation because filter push down test"); - return; - } - String openRule = "SET RULES ConstantPropagationRule=on;"; String closeRule = "SET RULES ConstantPropagationRule=off;"; @@ -1226,10 +1196,6 @@ public void testConstantPropagation() { /** 对常量折叠进行测试，因为RowTransform常量折叠和Filter常量折叠使用的代码都是公共的，所以这里只测试更好对比结果的RowTransform常量折叠 */ @Test public void testConstantFolding() { - if (!isOptimizerOpen) { - LOGGER.info("Skip SQLSessionIT.testConstantFolding because optimizer is closed"); - return; - } String openRule = "SET RULES ConstantFoldingRule=on;"; String closeRule = "SET RULES ConstantFoldingRule=off;"; @@ -1347,10 +1313,6 @@ public void testConstantFolding() { @Test public void testDistinctEliminate() { - if (!isOptimizerOpen) { - LOGGER.info("Skip SQLSessionIT.testDistinctEliminate because optimizer is closed"); - return; - } // 插入数据 StringBuilder insert = new StringBuilder(); insert.append("INSERT INTO us.d2 (key, s1, s2) VALUES "); @@ -1435,11 +1397,8 @@ public void testDistinctEliminate() { @Test public void testJoinFactorizationRule() { - if (!isOptimizerOpen || isScaling) { - LOGGER.info( - "Skip SQLSessionIT.testJoinFactorizationRule because optimizer is closed or scaling test"); - return; - } + Assume.assumeFalse(isScaling); + String openRule = "SET RULES JoinFactorizationRule=on;"; String closeRule = "SET RULES JoinFactorizationRule=off;"; @@ -1649,11 +1608,6 @@ public void testJoinFactorizationRule() { @Test public void testOuterJoinEliminate() { - if (!isOptimizerOpen) { - LOGGER.info("Skip SQLSessionIT.testOuterJoinEliminate because optimizer is closed"); - return; - } - StringBuilder insert = new StringBuilder(); insert.append("INSERT INTO us (key, d2.s1, d2.s2, d3.s1, d3.s2) VALUES "); int rows = 15000; @@ -1707,11 +1661,6 @@ public void testOuterJoinEliminate() { @Test public void testInFilterTransformRule() { - if (!isOptimizerOpen) { - LOGGER.info("Skip SQLSessionIT.testInFilterTransformRule because optimizer is closed"); - return; - } - // 插入数据 StringBuilder insert = new StringBuilder(); insert.append("INSERT INTO us.d2 (key, s1, s2) VALUES "); @@ -1780,4 +1729,65 @@ public void testInFilterTransformRule() { assertTrue( closeExplain.contains("us.*.s1 &!= 1 && us.*.s1 &!= 2 && us.*.s1 != 3 && us.*.s1 != 4")); } + + @Test + public void testAllowNullColumnRule() { + String openRule = "SET RULES AllowNullColumnRule=on;"; + String closeRule = "SET RULES AllowNullColumnRule=off;"; + String statement = "SELECT * FROM us.d1 WHERE s1 = 1;"; + executor.execute("insert into test(key, a) values (0, 1), (1, 2), (2, 3);"); + executor.execute("insert into test(key, b) values (3, 1), (4, 2), (5, 3);"); + + executor.execute(openRule); + String openExplain = executor.execute("EXPLAIN " + statement); + LOGGER.info("openExplain: \n" + openExplain); + Assert.assertFalse(openExplain.contains("RemoveNullColumn")); + + executor.executeAndCompare( + "select avg(*), sum(*), count(*) from test where key < 3;", + "ResultSets:\n" + + "+-----------+-----------+-----------+-----------+-------------+-------------+\n" + + "|avg(test.a)|avg(test.b)|sum(test.a)|sum(test.b)|count(test.a)|count(test.b)|\n" + + "+-----------+-----------+-----------+-----------+-------------+-------------+\n" + + "| 2.0| null| 6| null| 3| 0|\n" + + "+-----------+-----------+-----------+-----------+-------------+-------------+\n" + + "Total line number = 1\n"); + + executor.executeAndCompare( + "select avg(*), sum(*), count(*) from test where key > 2;", + "ResultSets:\n" + + "+-----------+-----------+-----------+-----------+-------------+-------------+\n" + + "|avg(test.a)|avg(test.b)|sum(test.a)|sum(test.b)|count(test.a)|count(test.b)|\n" + + "+-----------+-----------+-----------+-----------+-------------+-------------+\n" + + "| null| 2.0| null| 6| 0| 3|\n" + + "+-----------+-----------+-----------+-----------+-------------+-------------+\n" + + "Total line number = 1\n"); + + executor.execute(closeRule); + String closeExplain = executor.execute("EXPLAIN " + statement); + LOGGER.info("closeExplain: \n" + closeExplain); + Assert.assertTrue(closeExplain.contains("RemoveNullColumn")); + + executor.executeAndCompare( + "select avg(*), sum(*), count(*) from test where key < 3;", + "ResultSets:\n" + + "+-----------+-----------+-------------+-------------+\n" + + "|avg(test.a)|sum(test.a)|count(test.a)|count(test.b)|\n" + + "+-----------+-----------+-------------+-------------+\n" + + "| 2.0| 6| 3| 0|\n" + + "+-----------+-----------+-------------+-------------+\n" + + "Total line number = 1\n"); + + executor.executeAndCompare( + "select avg(*), sum(*), count(*) from test where key > 2;", + "ResultSets:\n" + + "+-----------+-----------+-------------+-------------+\n" + + "|avg(test.b)|sum(test.b)|count(test.a)|count(test.b)|\n" + + "+-----------+-----------+-------------+-------------+\n" + + "| 2.0| 6| 0| 3|\n" + + "+-----------+-----------+-------------+-------------+\n" + + "Total line number = 1\n"); + + executor.execute("delete columns test.a,test.b;"); + } } diff --git a/test/src/test/resources/testConfig.properties b/test/src/test/resources/testConfig.properties index 1b8b6763b8..2b86c9dd39 100644 --- a/test/src/test/resources/testConfig.properties +++ b/test/src/test/resources/testConfig.properties @@ -4,7 +4,7 @@ storageEngineList=IoTDB12,InfluxDB,FileSystem,Relational,MongoDB,Redis relationalStorageEngineList=PostgreSQL,MySQL # the test for every engine -test-list=UserPermissionIT,DataSourceIT,SQLSessionIT,SQLSessionPoolIT,SQLCompareIT,NewSessionIT,TagIT,RestAnnotationIT,RestIT,TransformIT,UDFIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT,PySessionIT,HostUtilsTest +test-list=UserPermissionIT,DataSourceIT,SQLSessionIT,OptimizerIT,SQLSessionPoolIT,SQLCompareIT,NewSessionIT,TagIT,RestAnnotationIT,RestIT,TransformIT,UDFIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT,PySessionIT,HostUtilsTest # the DB config # isSupportDiffTypeHistoryData: 跟dummy有关，是否历史数据写进去和查出来不一样，主要是key不一样和value的类型不一样