Property pruner (vesoft-inc#3750)

* property pruner * postprocess works. * add Travese::pruneProperties * make vertexprops of travese be null when node alias is not used * fix extractPropsFromExps for tagPropExpr and edgePropexpr, vFilter and eFilter * add PropertyTracker::update for project node * add DeduceMatchPropsVisitor and FLAGS_enable_opt_collapse_project_rule * add Filter::pruneProperties and only 3 tck cases not passed * Do not do propsUsed.colsSet.erase(it) * revert labeltagpropexpr and pass all tck * add hasAlias, add deduce types for id, src, dst func * rename visitor * ingore func id, src, dst, type, typeid, rank, hash in property tracker visitor * add Aggregate::pruneProperties and store the alias of id/src/dst... to propsused * format tck * remove some unusedless headers * add tck for plan * rename has1, has2, has3 * add flag_enable_optimizer_property_pruner_rule and support prune edge props... * add kUnknownEdgeType * add PrunePropertiesVisitor * remove PlanNode::pruneProperties, and move PropertyTracker to PropertyTrackerVisitor * add PrunePropertiesRule.feature * add markDeleted interface * remove unused code * fix header macro * update tck Co-authored-by: kyle.cao <kyle.cao@vesoft.com>
liwenhui-soul · Feb 15, 2022 · 29dfd29 · 29dfd29
1 parent ccc5bce
commit 29dfd29
Show file tree

Hide file tree

Showing 19 changed files with 1,189 additions and 8 deletions.
diff --git a/src/common/expression/PropertyExpression.h b/src/common/expression/PropertyExpression.h
@@ -160,10 +160,10 @@ class LabelTagPropertyExpression final : public PropertyExpression {
   }
 
  private:
-  LabelTagPropertyExpression(ObjectPool* pool,
-                             Expression* label = nullptr,
-                             const std::string& tag = "",
-                             const std::string& prop = "")
+  explicit LabelTagPropertyExpression(ObjectPool* pool,
+                                      Expression* label = nullptr,
+                                      const std::string& tag = "",
+                                      const std::string& prop = "")
       : PropertyExpression(pool, Kind::kLabelTagProperty, "", tag, prop), label_(label) {}
 
   void writeTo(Encoder& encoder) const override;

diff --git a/src/daemons/CMakeLists.txt b/src/daemons/CMakeLists.txt
@@ -128,6 +128,7 @@ nebula_add_executable(
         $<TARGET_OBJECTS:validator_obj>
         $<TARGET_OBJECTS:expr_visitor_obj>
         $<TARGET_OBJECTS:optimizer_obj>
+        $<TARGET_OBJECTS:plan_node_visitor_obj>
         $<TARGET_OBJECTS:planner_obj>
         $<TARGET_OBJECTS:plan_obj>
         $<TARGET_OBJECTS:executor_obj>
@@ -211,6 +212,7 @@ nebula_add_executable(
         $<TARGET_OBJECTS:validator_obj>
         $<TARGET_OBJECTS:expr_visitor_obj>
         $<TARGET_OBJECTS:optimizer_obj>
+        $<TARGET_OBJECTS:plan_node_visitor_obj>
         $<TARGET_OBJECTS:planner_obj>
         $<TARGET_OBJECTS:plan_obj>
         $<TARGET_OBJECTS:executor_obj>

diff --git a/src/graph/optimizer/Optimizer.cpp b/src/graph/optimizer/Optimizer.cpp
@@ -12,6 +12,7 @@
 #include "graph/planner/plan/ExecutionPlan.h"
 #include "graph/planner/plan/Logic.h"
 #include "graph/planner/plan/PlanNode.h"
+#include "graph/visitor/PrunePropertiesVisitor.h"
 
 using nebula::graph::BinaryInputNode;
 using nebula::graph::Loop;
@@ -20,6 +21,8 @@ using nebula::graph::QueryContext;
 using nebula::graph::Select;
 using nebula::graph::SingleDependencyNode;
 
+DEFINE_bool(enable_optimizer_property_pruner_rule, true, "");
+
 namespace nebula {
 namespace opt {
 
@@ -30,12 +33,32 @@ StatusOr<const PlanNode *> Optimizer::findBestPlan(QueryContext *qctx) {
   auto optCtx = std::make_unique<OptContext>(qctx);
 
   auto root = qctx->plan()->root();
-  auto status = prepare(optCtx.get(), root);
-  NG_RETURN_IF_ERROR(status);
-  auto rootGroup = std::move(status).value();
+  auto spaceID = qctx->rctx()->session()->space().id;
+
+  auto ret = prepare(optCtx.get(), root);
+  NG_RETURN_IF_ERROR(ret);
+  auto rootGroup = std::move(ret).value();
 
   NG_RETURN_IF_ERROR(doExploration(optCtx.get(), rootGroup));
-  return rootGroup->getPlan();
+  auto *newRoot = rootGroup->getPlan();
+
+  auto status2 = postprocess(const_cast<PlanNode *>(newRoot), qctx, spaceID);
+  if (!status2.ok()) {
+    LOG(ERROR) << "Failed to postprocess plan: " << status2;
+  }
+  return newRoot;
+}
+
+Status Optimizer::postprocess(PlanNode *root, graph::QueryContext *qctx, GraphSpaceID spaceID) {
+  if (FLAGS_enable_optimizer_property_pruner_rule) {
+    graph::PropertyTracker propsUsed;
+    graph::PrunePropertiesVisitor visitor(propsUsed, qctx, spaceID);
+    root->accept(&visitor);
+    if (!visitor.ok()) {
+      return visitor.status();
+    }
+  }
+  return Status::OK();
 }
 
 StatusOr<OptGroup *> Optimizer::prepare(OptContext *ctx, PlanNode *root) {

diff --git a/src/graph/optimizer/Optimizer.h b/src/graph/optimizer/Optimizer.h
@@ -8,6 +8,9 @@
 
 #include "common/base/Base.h"
 #include "common/base/StatusOr.h"
+#include "common/thrift/ThriftTypes.h"
+
+DECLARE_bool(enable_optimizer_property_pruner_rule);
 
 namespace nebula {
 namespace graph {
@@ -30,7 +33,10 @@ class Optimizer final {
   StatusOr<const graph::PlanNode *> findBestPlan(graph::QueryContext *qctx);
 
  private:
+  Status postprocess(graph::PlanNode *root, graph::QueryContext *qctx, GraphSpaceID spaceID);
+
   StatusOr<OptGroup *> prepare(OptContext *ctx, graph::PlanNode *root);
+
   Status doExploration(OptContext *octx, OptGroup *rootGroup);
 
   OptGroup *convertToGroup(OptContext *ctx,

diff --git a/src/graph/optimizer/rule/CollapseProjectRule.cpp b/src/graph/optimizer/rule/CollapseProjectRule.cpp
@@ -11,6 +11,8 @@
 #include "graph/planner/plan/Query.h"
 #include "graph/util/ExpressionUtils.h"
 
+DEFINE_bool(enable_optimizer_collapse_project_rule, true, "");
+
 using nebula::graph::PlanNode;
 using nebula::graph::QueryContext;
 
@@ -110,6 +112,9 @@ StatusOr<OptRule::TransformResult> CollapseProjectRule::transform(
 }
 
 bool CollapseProjectRule::match(OptContext* octx, const MatchedResult& matched) const {
+  if (!FLAGS_enable_optimizer_collapse_project_rule) {
+    return false;
+  }
   return OptRule::match(octx, matched);
 }
 

diff --git a/src/graph/optimizer/rule/CollapseProjectRule.h b/src/graph/optimizer/rule/CollapseProjectRule.h
@@ -8,6 +8,8 @@
 
 #include "graph/optimizer/OptRule.h"
 
+DECLARE_bool(enable_optimizer_collapse_project_rule);
+
 namespace nebula {
 namespace opt {
 

diff --git a/src/graph/optimizer/test/CMakeLists.txt b/src/graph/optimizer/test/CMakeLists.txt
@@ -43,6 +43,7 @@ set(OPTIMIZER_TEST_LIB
     $<TARGET_OBJECTS:graph_context_obj>
     $<TARGET_OBJECTS:validator_obj>
     $<TARGET_OBJECTS:optimizer_obj>
+    $<TARGET_OBJECTS:plan_node_visitor_obj>
     $<TARGET_OBJECTS:ssl_obj>
     $<TARGET_OBJECTS:memory_obj>
     $<TARGET_OBJECTS:geo_index_obj>

diff --git a/src/graph/planner/plan/PlanNode.cpp b/src/graph/planner/plan/PlanNode.cpp
@@ -13,6 +13,7 @@
 
 #include "common/graph/Response.h"
 #include "graph/context/QueryContext.h"
+#include "graph/planner/plan/PlanNodeVisitor.h"
 #include "graph/util/ToJson.h"
 
 namespace nebula {
@@ -360,6 +361,10 @@ std::unique_ptr<PlanNodeDescription> PlanNode::explain() const {
   return desc;
 }
 
+void PlanNode::accept(PlanNodeVisitor* visitor) {
+  visitor->visit(this);
+}
+
 void PlanNode::releaseSymbols() {
   auto symTbl = qctx_->symTable();
   for (auto in : inputVars_) {

diff --git a/src/graph/planner/plan/PlanNode.h b/src/graph/planner/plan/PlanNode.h
@@ -14,6 +14,8 @@
 namespace nebula {
 namespace graph {
 
+class PlanNodeVisitor;
+
 /**
  * PlanNode is an abstraction of nodes in an execution plan which
  * is a kind of directed cyclic graph.
@@ -189,6 +191,12 @@ class PlanNode {
   // Describe plan node
   virtual std::unique_ptr<PlanNodeDescription> explain() const;
 
+  virtual void accept(PlanNodeVisitor* visitor);
+
+  void markDeleted() {
+    deleted_ = true;
+  }
+
   virtual PlanNode* clone() const = 0;
 
   virtual void calcCost();
@@ -311,6 +319,7 @@ class PlanNode {
   std::vector<const PlanNode*> dependencies_;
   std::vector<Variable*> inputVars_;
   std::vector<Variable*> outputVars_;
+  bool deleted_{false};
 };
 
 std::ostream& operator<<(std::ostream& os, PlanNode::Kind kind);

diff --git a/src/graph/planner/plan/PlanNodeVisitor.h b/src/graph/planner/plan/PlanNodeVisitor.h
@@ -0,0 +1,31 @@
+/* Copyright (c) 2020 vesoft inc. All rights reserved.
+ *
+ * This source code is licensed under Apache 2.0 License.
+ */
+
+#ifndef PLAN_PLANNODEVISITOR_H_
+#define PLAN_PLANNODEVISITOR_H_
+
+#include "graph/planner/plan/PlanNode.h"
+#include "graph/planner/plan/Query.h"
+
+namespace nebula {
+namespace graph {
+
+class PlanNodeVisitor {
+ public:
+  virtual ~PlanNodeVisitor() = default;
+
+  virtual void visit(PlanNode *node) = 0;
+  virtual void visit(Filter *node) = 0;
+  virtual void visit(Project *node) = 0;
+  virtual void visit(Aggregate *node) = 0;
+  virtual void visit(Traverse *node) = 0;
+  virtual void visit(AppendVertices *node) = 0;
+  virtual void visit(BiJoin *node) = 0;
+};
+
+}  // namespace graph
+}  // namespace nebula
+
+#endif  // PLAN_PLANNODEVISITOR_H_
diff --git a/src/graph/planner/plan/Query.cpp b/src/graph/planner/plan/Query.cpp
@@ -10,6 +10,7 @@
 #include <folly/json.h>
 #include <thrift/lib/cpp/util/EnumUtils.h>
 
+#include "graph/planner/plan/PlanNodeVisitor.h"
 #include "graph/util/ExpressionUtils.h"
 #include "graph/util/ToJson.h"
 
@@ -269,6 +270,10 @@ std::unique_ptr<PlanNodeDescription> Filter::explain() const {
   return desc;
 }
 
+void Filter::accept(PlanNodeVisitor* visitor) {
+  visitor->visit(this);
+}
+
 PlanNode* Filter::clone() const {
   auto* newFilter = Filter::make(qctx_, nullptr);
   newFilter->cloneMembers(*this);
@@ -336,6 +341,10 @@ std::unique_ptr<PlanNodeDescription> Project::explain() const {
   return desc;
 }
 
+void Project::accept(PlanNodeVisitor* visitor) {
+  visitor->visit(this);
+}
+
 PlanNode* Project::clone() const {
   auto* newProj = Project::make(qctx_, nullptr);
   newProj->cloneMembers(*this);
@@ -490,6 +499,10 @@ std::unique_ptr<PlanNodeDescription> Aggregate::explain() const {
   return desc;
 }
 
+void Aggregate::accept(PlanNodeVisitor* visitor) {
+  visitor->visit(this);
+}
+
 PlanNode* Aggregate::clone() const {
   auto* newAggregate = Aggregate::make(qctx_, nullptr);
   newAggregate->cloneMembers(*this);
@@ -734,6 +747,10 @@ std::unique_ptr<PlanNodeDescription> Traverse::explain() const {
   return desc;
 }
 
+void Traverse::accept(PlanNodeVisitor* visitor) {
+  visitor->visit(this);
+}
+
 AppendVertices* AppendVertices::clone() const {
   auto newAV = AppendVertices::make(qctx_, nullptr, space_);
   newAV->cloneMembers(*this);
@@ -760,13 +777,21 @@ std::unique_ptr<PlanNodeDescription> AppendVertices::explain() const {
   return desc;
 }
 
+void AppendVertices::accept(PlanNodeVisitor* visitor) {
+  visitor->visit(this);
+}
+
 std::unique_ptr<PlanNodeDescription> BiJoin::explain() const {
   auto desc = BinaryInputNode::explain();
   addDescription("hashKeys", folly::toJson(util::toJson(hashKeys_)), desc.get());
   addDescription("probeKeys", folly::toJson(util::toJson(probeKeys_)), desc.get());
   return desc;
 }
 
+void BiJoin::accept(PlanNodeVisitor* visitor) {
+  visitor->visit(this);
+}
+
 void BiJoin::cloneMembers(const BiJoin& j) {
   BinaryInputNode::cloneMembers(j);
 

diff --git a/src/graph/planner/plan/Query.h b/src/graph/planner/plan/Query.h
@@ -730,6 +730,8 @@ class Filter final : public SingleInputNode {
   PlanNode* clone() const override;
   std::unique_ptr<PlanNodeDescription> explain() const override;
 
+  void accept(PlanNodeVisitor* visitor) override;
+
  private:
   Filter(QueryContext* qctx, PlanNode* input, Expression* condition, bool needStableFilter);
   void cloneMembers(const Filter&);
@@ -826,6 +828,8 @@ class Project final : public SingleInputNode {
   PlanNode* clone() const override;
   std::unique_ptr<PlanNodeDescription> explain() const override;
 
+  void accept(PlanNodeVisitor* visitor) override;
+
  private:
   Project(QueryContext* qctx, PlanNode* input, YieldColumns* cols);
 
@@ -1114,6 +1118,8 @@ class Aggregate final : public SingleInputNode {
   PlanNode* clone() const override;
   std::unique_ptr<PlanNodeDescription> explain() const override;
 
+  void accept(PlanNodeVisitor* visitor) override;
+
  private:
   Aggregate(QueryContext* qctx,
             PlanNode* input,
@@ -1469,6 +1475,8 @@ class Traverse final : public GetNeighbors {
 
   std::unique_ptr<PlanNodeDescription> explain() const override;
 
+  void accept(PlanNodeVisitor* visitor) override;
+
   Traverse* clone() const override;
 
   MatchStepRange* stepRange() const {
@@ -1526,6 +1534,8 @@ class AppendVertices final : public GetVertices {
 
   std::unique_ptr<PlanNodeDescription> explain() const override;
 
+  void accept(PlanNodeVisitor* visitor) override;
+
   AppendVertices* clone() const override;
 
   Expression* vFilter() const {
@@ -1585,6 +1595,8 @@ class BiJoin : public BinaryInputNode {
 
   std::unique_ptr<PlanNodeDescription> explain() const override;
 
+  void accept(PlanNodeVisitor* visitor) override;
+
  protected:
   BiJoin(QueryContext* qctx,
          Kind kind,

diff --git a/src/graph/util/ExpressionUtils.cpp b/src/graph/util/ExpressionUtils.cpp
@@ -17,6 +17,7 @@
 #include "graph/context/QueryContext.h"
 #include "graph/context/QueryExpressionContext.h"
 #include "graph/visitor/FoldConstantExprVisitor.h"
+#include "graph/visitor/PropertyTrackerVisitor.h"
 
 DEFINE_int32(max_expression_depth, 512, "Max depth of expression tree.");
 

diff --git a/src/graph/visitor/CMakeLists.txt b/src/graph/visitor/CMakeLists.txt
@@ -6,6 +6,7 @@ nebula_add_library(
     expr_visitor_obj OBJECT
     ExprVisitorImpl.cpp
     DeducePropsVisitor.cpp
+    PropertyTrackerVisitor.cpp
     DeduceTypeVisitor.cpp
     ExtractPropExprVisitor.cpp
     ExtractFilterExprVisitor.cpp
@@ -17,4 +18,9 @@ nebula_add_library(
     EvaluableExprVisitor.cpp
 )
 
+nebula_add_library(
+    plan_node_visitor_obj OBJECT
+    PrunePropertiesVisitor.cpp
+)
+
 nebula_add_subdirectory(test)