feat(core): add option to remove RemoveNullColumn operator (#536)

一个折衷

conf/config.properties

@@ -98,6 +98,8 @@ ruleBasedOptimizer=NotFilterRemoveRule=on,ColumnPruningRule=on,ConstantPropagati
   ConstantFoldingRule=on,FilterPushDownRule=on,JoinFactorizationRule=on,SetTransformPushDownPathUnionJoinRule=on,InFilterTransformRule=on,\
   OuterJoinEliminateRule=on
 
+#ruleBasedOptimizer=AllowNullColumnRule=on
+
 ##########################
 ### 执行层配置
 ##########################

optimizer/src/main/java/cn/edu/tsinghua/iginx/logical/optimizer/rules/AllowNullColumnRule.java

@@ -0,0 +1,51 @@
+/*
+ * IGinX - the polystore system with high performance
+ * Copyright (C) Tsinghua University
+ * TSIGinX@gmail.com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package cn.edu.tsinghua.iginx.logical.optimizer.rules;
+
+import cn.edu.tsinghua.iginx.engine.shared.operator.RemoveNullColumn;
+import cn.edu.tsinghua.iginx.engine.shared.source.OperatorSource;
+import cn.edu.tsinghua.iginx.engine.shared.source.SourceType;
+import cn.edu.tsinghua.iginx.logical.optimizer.core.RuleCall;
+import com.google.auto.service.AutoService;
+
+@AutoService(Rule.class)
+public class AllowNullColumnRule extends Rule {
+
+  public AllowNullColumnRule() {
+    super(
+        AllowNullColumnRule.class.getSimpleName(),
+        operand(RemoveNullColumn.class, any()),
+        Long.MAX_VALUE,
+        RuleStrategy.FIXED_POINT);
+  }
+
+  @Override
+  public boolean matches(RuleCall call) {
+    RemoveNullColumn removeNullColumn = (RemoveNullColumn) call.getMatchedRoot();
+    return removeNullColumn.getSource().getType() == SourceType.Operator;
+  }
+
+  @Override
+  public void onMatch(RuleCall call) {
+    RemoveNullColumn removeNullColumn = (RemoveNullColumn) call.getMatchedRoot();
+    OperatorSource source = (OperatorSource) removeNullColumn.getSource();
+    call.transformTo(source.getOperator());
+  }
+}

test/README.md

@@ -12,7 +12,7 @@ IGinX代码测试框架于2023年完成重构，测试主要分为以下部分
 - **`standalone-test`**：单机数据库功能测试
   - 文件位于`integration/controller/Controller.java`
   - `testUnion()`方法负责在一组存储引擎上运行测试。首先加载测试配置文件，并提取要测试的存储引擎的相关特性。并将其支持的测试文件写入待测试列表中，然后调用脚本对数据库一一执行测试列表中的测试。
-  - 目前测试的包括`SQLSessionIT,SQLSessionPoolIT,TagIT,RestAnnotationIT,RestIT,TransformIT,UDFIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT`
+  - 目前测试的包括`SQLSessionIT,OptimizerIT,SQLSessionPoolIT,TagIT,RestAnnotationIT,RestIT,TransformIT,UDFIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT`
 - **`unit-mds`**：元数据功能测试
   - 文件位于`integration/mds/`下
   - 分别针对ETCD和ZOOKEEPER进行测试

test/src/test/java/cn/edu/tsinghua/iginx/integration/func/optimizer/OptimizerIT.java

@@ -22,7 +22,6 @@
 import static cn.edu.tsinghua.iginx.integration.controller.Controller.SUPPORT_KEY;
 import static cn.edu.tsinghua.iginx.integration.controller.Controller.clearAllData;
 import static org.junit.Assert.*;
-import static org.junit.Assert.assertTrue;
 
 import cn.edu.tsinghua.iginx.exception.SessionException;
 import cn.edu.tsinghua.iginx.integration.controller.Controller;
@@ -33,7 +32,9 @@
 import cn.edu.tsinghua.iginx.session.Session;
 import cn.edu.tsinghua.iginx.thrift.DataType;
 import cn.edu.tsinghua.iginx.utils.Pair;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -104,9 +105,8 @@ public OptimizerIT() {
 
     String rules = executor.execute("SHOW RULES;");
     this.isOptimizerOpen = rules.contains("FilterPushDownRule|    ON|");
-    if (isOptimizerOpen) {
-      ruleList = getRuleList();
-    }
+    Assume.assumeTrue(isOptimizerOpen);
+    ruleList = getRuleList();
   }
 
   @BeforeClass
@@ -178,6 +178,7 @@ public void closeAllRules() {
     }
     String statement = "set rules" + sb.substring(0, sb.length() - 1) + ";";
     executor.execute(statement);
+    executor.execute("SET RULES AllowNullColumnRule=on;");
   }
 
   @After
@@ -188,6 +189,8 @@ public void openAllRules() {
     }
     String statement = "set rules" + sb.substring(0, sb.length() - 1) + ";";
     executor.execute(statement);
+    executor.execute("SET RULES FragmentPruningByFilterRule=off;");
+    executor.execute("SET RULES AllowNullColumnRule=off;");
   }
 
   private void generateData(long start, long end) {
@@ -244,11 +247,6 @@ public void clearData() {
 
   @Test
   public void testModifyRules() {
-    if (!isOptimizerOpen) {
-      LOGGER.info("Skip SQLSessionIT.testModifyRules because optimizer is not open");
-      return;
-    }
-
     String statement, expected;
     statement = "show rules;";
 
@@ -284,12 +282,6 @@ public void testModifyRules() {
 
   @Test
   public void testFilterPushDownExplain() {
-    // 临时修改
-    if (!isOptimizerOpen) {
-      LOGGER.info("Skip SQLSessionIT.testFilterPushDownExplain because optimizer is not open");
-      return;
-    }
-
     executor.execute("SET RULES FilterPushDownRule=on;");
 
     String insert =
@@ -683,29 +675,16 @@ public void testFilterPushDownExplain() {
         expectRes = expectRes.replaceAll("&mark\\d+", "&mark").replaceAll(" ", "");
       }
 
-      assertEquals(res, expectRes);
+      assertEquals(expectRes, res);
     }
   }
 
   @Test
   public void testFilterFragmentOptimizer() {
-    String policy = executor.execute("SHOW CONFIG \"policyClassName\";");
-    if (!policy.contains("KeyRangeTestPolicy")) {
-      LOGGER.info(
-          "Skip SQLSessionIT.testFilterFragmentOptimizer because policy is not KeyRangeTestPolicy");
-      return;
-    }
-
-    if (!isOptimizerOpen) {
-      LOGGER.info(
-          "Skip SQLSessionIT.testFilterFragmentOptimizer because optimizer is not remove_not,filter_fragment");
-      return;
-    }
+    Assume.assumeFalse(isScaling);
 
-    if (isScaling) {
-      LOGGER.info("Skip SQLSessionIT.testFilterFragmentOptimizer because it is scaling test");
-      return;
-    }
+    String policy = executor.execute("SHOW CONFIG \"policyClassName\";");
+    Assume.assumeTrue(policy.contains("KeyRangeTestPolicy"));
 
     String insert =
         "INSERT INTO us.d2(key, c) VALUES (1, \"asdas\"), (2, \"sadaa\"), (3, \"sadada\"), (4, \"asdad\"), (5, \"deadsa\"), (6, \"dasda\"), (7, \"asdsad\"), (8, \"frgsa\"), (9, \"asdad\");";
@@ -902,11 +881,7 @@ public void testFilterFragmentOptimizer() {
 
   @Test
   public void testColumnPruningAndFragmentPruning() {
-    if (!isOptimizerOpen || isScaling) {
-      LOGGER.info(
-          "Skip SQLSessionIT.testColumnPruningAndFragmentPruning because scaling test or filter push down test");
-      return;
-    }
+    Assume.assumeFalse(isScaling);
 
     StringBuilder insert =
         new StringBuilder(
@@ -1167,11 +1142,6 @@ public void testColumnPruningAndFragmentPruning() {
 
   @Test
   public void testConstantPropagation() {
-    if (!isOptimizerOpen) {
-      LOGGER.info("Skip SQLSessionIT.testConstantPropagation because filter push down test");
-      return;
-    }
-
     String openRule = "SET RULES ConstantPropagationRule=on;";
     String closeRule = "SET RULES ConstantPropagationRule=off;";
 
@@ -1226,10 +1196,6 @@ public void testConstantPropagation() {
   /** 对常量折叠进行测试，因为RowTransform常量折叠和Filter常量折叠使用的代码都是公共的，所以这里只测试更好对比结果的RowTransform常量折叠 */
   @Test
   public void testConstantFolding() {
-    if (!isOptimizerOpen) {
-      LOGGER.info("Skip SQLSessionIT.testConstantFolding because optimizer is closed");
-      return;
-    }
     String openRule = "SET RULES ConstantFoldingRule=on;";
     String closeRule = "SET RULES ConstantFoldingRule=off;";
 
@@ -1347,10 +1313,6 @@ public void testConstantFolding() {
 
   @Test
   public void testDistinctEliminate() {
-    if (!isOptimizerOpen) {
-      LOGGER.info("Skip SQLSessionIT.testDistinctEliminate because optimizer is closed");
-      return;
-    }
     // 插入数据
     StringBuilder insert = new StringBuilder();
     insert.append("INSERT INTO us.d2 (key, s1, s2) VALUES ");
@@ -1435,11 +1397,8 @@ public void testDistinctEliminate() {
 
   @Test
   public void testJoinFactorizationRule() {
-    if (!isOptimizerOpen || isScaling) {
-      LOGGER.info(
-          "Skip SQLSessionIT.testJoinFactorizationRule because optimizer is closed or scaling test");
-      return;
-    }
+    Assume.assumeFalse(isScaling);
+
     String openRule = "SET RULES JoinFactorizationRule=on;";
     String closeRule = "SET RULES JoinFactorizationRule=off;";
 
@@ -1649,11 +1608,6 @@ public void testJoinFactorizationRule() {
 
   @Test
   public void testOuterJoinEliminate() {
-    if (!isOptimizerOpen) {
-      LOGGER.info("Skip SQLSessionIT.testOuterJoinEliminate because optimizer is closed");
-      return;
-    }
-
     StringBuilder insert = new StringBuilder();
     insert.append("INSERT INTO us (key, d2.s1, d2.s2, d3.s1, d3.s2) VALUES ");
     int rows = 15000;
@@ -1707,11 +1661,6 @@ public void testOuterJoinEliminate() {
 
   @Test
   public void testInFilterTransformRule() {
-    if (!isOptimizerOpen) {
-      LOGGER.info("Skip SQLSessionIT.testInFilterTransformRule because optimizer is closed");
-      return;
-    }
-
     // 插入数据
     StringBuilder insert = new StringBuilder();
     insert.append("INSERT INTO us.d2 (key, s1, s2) VALUES ");
@@ -1780,4 +1729,65 @@ public void testInFilterTransformRule() {
     assertTrue(
         closeExplain.contains("us.*.s1 &!= 1 && us.*.s1 &!= 2 && us.*.s1 != 3 && us.*.s1 != 4"));
   }
+
+  @Test
+  public void testAllowNullColumnRule() {
+    String openRule = "SET RULES AllowNullColumnRule=on;";
+    String closeRule = "SET RULES AllowNullColumnRule=off;";
+    String statement = "SELECT * FROM us.d1 WHERE s1 = 1;";
+    executor.execute("insert into test(key, a) values (0, 1), (1, 2), (2, 3);");
+    executor.execute("insert into test(key, b) values (3, 1), (4, 2), (5, 3);");
+
+    executor.execute(openRule);
+    String openExplain = executor.execute("EXPLAIN " + statement);
+    LOGGER.info("openExplain: \n" + openExplain);
+    Assert.assertFalse(openExplain.contains("RemoveNullColumn"));
+
+    executor.executeAndCompare(
+        "select avg(*), sum(*), count(*) from test where key < 3;",
+        "ResultSets:\n"
+            + "+-----------+-----------+-----------+-----------+-------------+-------------+\n"
+            + "|avg(test.a)|avg(test.b)|sum(test.a)|sum(test.b)|count(test.a)|count(test.b)|\n"
+            + "+-----------+-----------+-----------+-----------+-------------+-------------+\n"
+            + "|        2.0|       null|          6|       null|            3|            0|\n"
+            + "+-----------+-----------+-----------+-----------+-------------+-------------+\n"
+            + "Total line number = 1\n");
+
+    executor.executeAndCompare(
+        "select avg(*), sum(*), count(*) from test where key > 2;",
+        "ResultSets:\n"
+            + "+-----------+-----------+-----------+-----------+-------------+-------------+\n"
+            + "|avg(test.a)|avg(test.b)|sum(test.a)|sum(test.b)|count(test.a)|count(test.b)|\n"
+            + "+-----------+-----------+-----------+-----------+-------------+-------------+\n"
+            + "|       null|        2.0|       null|          6|            0|            3|\n"
+            + "+-----------+-----------+-----------+-----------+-------------+-------------+\n"
+            + "Total line number = 1\n");
+
+    executor.execute(closeRule);
+    String closeExplain = executor.execute("EXPLAIN " + statement);
+    LOGGER.info("closeExplain: \n" + closeExplain);
+    Assert.assertTrue(closeExplain.contains("RemoveNullColumn"));
+
+    executor.executeAndCompare(
+        "select avg(*), sum(*), count(*) from test where key < 3;",
+        "ResultSets:\n"
+            + "+-----------+-----------+-------------+-------------+\n"
+            + "|avg(test.a)|sum(test.a)|count(test.a)|count(test.b)|\n"
+            + "+-----------+-----------+-------------+-------------+\n"
+            + "|        2.0|          6|            3|            0|\n"
+            + "+-----------+-----------+-------------+-------------+\n"
+            + "Total line number = 1\n");
+
+    executor.executeAndCompare(
+        "select avg(*), sum(*), count(*) from test where key > 2;",
+        "ResultSets:\n"
+            + "+-----------+-----------+-------------+-------------+\n"
+            + "|avg(test.b)|sum(test.b)|count(test.a)|count(test.b)|\n"
+            + "+-----------+-----------+-------------+-------------+\n"
+            + "|        2.0|          6|            0|            3|\n"
+            + "+-----------+-----------+-------------+-------------+\n"
+            + "Total line number = 1\n");
+
+    executor.execute("delete columns test.a,test.b;");
+  }
 }

test/src/test/resources/testConfig.properties

@@ -4,7 +4,7 @@ storageEngineList=IoTDB12,InfluxDB,FileSystem,Relational,MongoDB,Redis
 relationalStorageEngineList=PostgreSQL,MySQL
 
 # the test for every engine
-test-list=UserPermissionIT,DataSourceIT,SQLSessionIT,SQLSessionPoolIT,SQLCompareIT,NewSessionIT,TagIT,RestAnnotationIT,RestIT,TransformIT,UDFIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT,PySessionIT,HostUtilsTest
+test-list=UserPermissionIT,DataSourceIT,SQLSessionIT,OptimizerIT,SQLSessionPoolIT,SQLCompareIT,NewSessionIT,TagIT,RestAnnotationIT,RestIT,TransformIT,UDFIT,SessionV2IT,SessionIT,SessionPoolIT,CompactionIT,TimePrecisionIT,PySessionIT,HostUtilsTest
 
 # the DB config
 # isSupportDiffTypeHistoryData: 跟dummy有关，是否历史数据写进去和查出来不一样，主要是key不一样和value的类型不一样