[Feature](Nereids) Add hint to enable pre-aggregation when scan OLAP …

…table. (#15614) This pr added support for the pre-aggregation hint. Users could use /*+PREAGGOPEN*/ to enable pre-preaggregation for OLAP table. For example: Let's say we have an aggregate-keys table t (k1 int, k2 int, v1 int sum, v2 int sum). Pre-aggregation could be enabled by query with a hint: select k1, v1 from t /*+PREAGGOPEN*/.
apache · Feb 7, 2023 · bed1ab7 · bed1ab7
1 parent 27216dc
commit bed1ab7
Show file tree

Hide file tree

Showing 9 changed files with 183 additions and 39 deletions.
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -127,8 +127,13 @@ joinRelation
 
 // Just like `opt_plan_hints` in legacy CUP parser.
 joinHint
-    : LEFT_BRACKET identifier RIGHT_BRACKET                           #bracketStyleHint
-    | HINT_START identifier HINT_END                                  #commentStyleHint
+    : LEFT_BRACKET identifier RIGHT_BRACKET                           #bracketJoinHint
+    | HINT_START identifier HINT_END                                  #commentJoinHint
+    ;
+
+relationHint
+    : LEFT_BRACKET identifier (COMMA identifier)* RIGHT_BRACKET       #bracketRelationHint
+    | HINT_START identifier (COMMA identifier)* HINT_END              #commentRelationHint
     ;
 
 aggClause
@@ -208,11 +213,11 @@ identifierSeq
     ;
 
 relationPrimary
-    : multipartIdentifier specifiedPartition? tableAlias lateralView*           #tableName
-    | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView*                      #aliasedQuery
+    : multipartIdentifier specifiedPartition? tableAlias relationHint? lateralView*           #tableName
+    | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView*                                    #aliasedQuery
     | tvfName=identifier LEFT_PAREN
       (properties+=tvfProperty (COMMA properties+=tvfProperty)*)?
-      RIGHT_PAREN tableAlias                                                    #tableValuedFunction
+      RIGHT_PAREN tableAlias                                                                  #tableValuedFunction
     ;
 
 tvfProperty

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java
@@ -32,6 +32,8 @@
 import org.apache.doris.nereids.util.Utils;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 
 import java.util.List;
@@ -46,21 +48,29 @@ public class UnboundRelation extends LogicalRelation implements Unbound {
     private final List<String> nameParts;
     private final List<String> partNames;
     private final boolean isTempPart;
+    private final List<String> hints;
 
     public UnboundRelation(RelationId id, List<String> nameParts) {
-        this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false);
+        this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false, ImmutableList.of());
     }
 
     public UnboundRelation(RelationId id, List<String> nameParts, List<String> partNames, boolean isTempPart) {
-        this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart);
+        this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, ImmutableList.of());
+    }
+
+    public UnboundRelation(RelationId id, List<String> nameParts, List<String> partNames, boolean isTempPart,
+            List<String> hints) {
+        this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, hints);
     }
 
     public UnboundRelation(RelationId id, List<String> nameParts, Optional<GroupExpression> groupExpression,
-            Optional<LogicalProperties> logicalProperties, List<String> partNames, boolean isTempPart) {
+            Optional<LogicalProperties> logicalProperties, List<String> partNames, boolean isTempPart,
+            List<String> hints) {
         super(id, PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties);
         this.nameParts = ImmutableList.copyOf(Objects.requireNonNull(nameParts, "nameParts should not null"));
         this.partNames = ImmutableList.copyOf(Objects.requireNonNull(partNames, "partNames should not null"));
         this.isTempPart = isTempPart;
+        this.hints = ImmutableList.copyOf(Objects.requireNonNull(hints, "hints should not be null."));
     }
 
     @Override
@@ -84,13 +94,14 @@ public LogicalProperties computeLogicalProperties() {
 
     @Override
     public Plan withGroupExpression(Optional<GroupExpression> groupExpression) {
-        return new UnboundRelation(id, nameParts, groupExpression, Optional.of(getLogicalProperties()),
-                partNames, isTempPart);
+        return new UnboundRelation(id, nameParts, groupExpression, Optional.of(getLogicalProperties()), partNames,
+                isTempPart, hints);
     }
 
     @Override
     public Plan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
-        return new UnboundRelation(id, nameParts, Optional.empty(), logicalProperties, partNames, isTempPart);
+        return new UnboundRelation(id, nameParts, Optional.empty(), logicalProperties, partNames,
+                isTempPart, hints);
     }
 
     @Override
@@ -100,10 +111,15 @@ public List<Slot> computeOutput() {
 
     @Override
     public String toString() {
-        return Utils.toSqlString("UnboundRelation",
+        List<Object> args = Lists.newArrayList(
                 "id", id,
                 "nameParts", StringUtils.join(nameParts, ".")
         );
+        if (CollectionUtils.isNotEmpty(hints)) {
+            args.add("hints");
+            args.add(StringUtils.join(hints, ", "));
+        }
+        return Utils.toSqlString("UnboundRelation", args.toArray());
     }
 
     @Override
@@ -147,4 +163,8 @@ public List<String> getPartNames() {
     public boolean isTempPart() {
         return isTempPart;
     }
+
+    public List<String> getHints() {
+        return hints;
+    }
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -29,9 +29,11 @@
 import org.apache.doris.nereids.DorisParser.ArithmeticUnaryContext;
 import org.apache.doris.nereids.DorisParser.BitOperationContext;
 import org.apache.doris.nereids.DorisParser.BooleanLiteralContext;
-import org.apache.doris.nereids.DorisParser.BracketStyleHintContext;
+import org.apache.doris.nereids.DorisParser.BracketJoinHintContext;
+import org.apache.doris.nereids.DorisParser.BracketRelationHintContext;
 import org.apache.doris.nereids.DorisParser.ColumnReferenceContext;
-import org.apache.doris.nereids.DorisParser.CommentStyleHintContext;
+import org.apache.doris.nereids.DorisParser.CommentJoinHintContext;
+import org.apache.doris.nereids.DorisParser.CommentRelationHintContext;
 import org.apache.doris.nereids.DorisParser.ComparisonContext;
 import org.apache.doris.nereids.DorisParser.CreateRowPolicyContext;
 import org.apache.doris.nereids.DorisParser.CteContext;
@@ -460,8 +462,16 @@ public LogicalPlan visitTableName(TableNameContext ctx) {
                 partitionNames.addAll(visitIdentifierList(ctx.specifiedPartition().identifierList()));
             }
         }
+
+        final List<String> relationHints;
+        if (ctx.relationHint() != null) {
+            relationHints = typedVisit(ctx.relationHint());
+        } else {
+            relationHints = ImmutableList.of();
+        }
+
         LogicalPlan checkedRelation = withCheckPolicy(
-                new UnboundRelation(RelationUtil.newRelationId(), tableId, partitionNames, isTempPart));
+                new UnboundRelation(RelationUtil.newRelationId(), tableId, partitionNames, isTempPart, relationHints));
         LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias());
         for (LateralViewContext lateralViewContext : ctx.lateralView()) {
             plan = withGenerate(plan, lateralViewContext);
@@ -1412,15 +1422,29 @@ private LogicalPlan withSelectHint(LogicalPlan logicalPlan, SelectHintContext hi
     }
 
     @Override
-    public String visitBracketStyleHint(BracketStyleHintContext ctx) {
+    public String visitBracketJoinHint(BracketJoinHintContext ctx) {
         return ctx.identifier().getText();
     }
 
     @Override
-    public Object visitCommentStyleHint(CommentStyleHintContext ctx) {
+    public String visitCommentJoinHint(CommentJoinHintContext ctx) {
         return ctx.identifier().getText();
     }
 
+    @Override
+    public List<String> visitBracketRelationHint(BracketRelationHintContext ctx) {
+        return ctx.identifier().stream()
+                .map(RuleContext::getText)
+                .collect(ImmutableList.toImmutableList());
+    }
+
+    @Override
+    public Object visitCommentRelationHint(CommentRelationHintContext ctx) {
+        return ctx.identifier().stream()
+                .map(RuleContext::getText)
+                .collect(ImmutableList.toImmutableList());
+    }
+
     private LogicalPlan withProjection(LogicalPlan input, SelectColumnClauseContext selectCtx,
                                        Optional<AggClauseContext> aggCtx, boolean isDistinct) {
         return ParserUtils.withOrigin(selectCtx, () -> {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -161,10 +161,10 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation,
         List<Long> partIds = getPartitionIds(table, unboundRelation);
         if (!CollectionUtils.isEmpty(partIds)) {
             scan = new LogicalOlapScan(RelationUtil.newRelationId(),
-                    (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), partIds);
+                    (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), partIds, unboundRelation.getHints());
         } else {
             scan = new LogicalOlapScan(RelationUtil.newRelationId(),
-                    (OlapTable) table, ImmutableList.of(tableQualifier.get(1)));
+                    (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), unboundRelation.getHints());
         }
         if (!Util.showHiddenColumns() && scan.getTable().hasDeleteSign()
                 && !ConnectContext.get().getSessionVariable()

diff --git a/.../src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java b/.../src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java
@@ -282,4 +282,8 @@ private int indexKeyPrefixMatchCount(
         }
         return matchCount;
     }
+
+    protected boolean preAggEnabledByHint(LogicalOlapScan olapScan) {
+        return olapScan.getHints().stream().anyMatch("PREAGGOPEN"::equalsIgnoreCase);
+    }
 }
diff --git a/...src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java b/...src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java
@@ -493,11 +493,17 @@ private SelectResult select(
         switch (table.getKeysType()) {
             case AGG_KEYS:
             case UNIQUE_KEYS: {
-                // Only checking pre-aggregation status by base index is enough for aggregate-keys and
-                // unique-keys OLAP table.
-                // Because the schemas in non-base materialized index are subsets of the schema of base index.
-                PreAggStatus preAggStatus = checkPreAggStatus(scan, table.getBaseIndexId(), predicates,
-                        aggregateFunctions, groupingExprs);
+                final PreAggStatus preAggStatus;
+                if (preAggEnabledByHint(scan)) {
+                    // PreAggStatus could be enabled by pre-aggregation hint for agg-keys and unique-keys.
+                    preAggStatus = PreAggStatus.on();
+                } else {
+                    // Only checking pre-aggregation status by base index is enough for aggregate-keys and
+                    // unique-keys OLAP table.
+                    // Because the schemas in non-base materialized index are subsets of the schema of base index.
+                    preAggStatus = checkPreAggStatus(scan, table.getBaseIndexId(), predicates,
+                            aggregateFunctions, groupingExprs);
+                }
                 if (preAggStatus.isOff()) {
                     // return early if pre agg status if off.
                     return new SelectResult(preAggStatus, scan.getTable().getBaseIndexId(), new ExprRewriteMap());

diff --git a/.../main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java b/.../main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java
@@ -128,12 +128,18 @@ private LogicalOlapScan select(
                         .filter(index -> containAllRequiredColumns(index, scan, requiredScanOutputSupplier.get()))
                         .collect(Collectors.toList());
 
-                PreAggStatus preAgg = PreAggStatus.off("No aggregate on scan.");
+                final PreAggStatus preAggStatus;
+                if (preAggEnabledByHint(scan)) {
+                    // PreAggStatus could be enabled by pre-aggregation hint for agg-keys and unique-keys.
+                    preAggStatus = PreAggStatus.on();
+                } else {
+                    preAggStatus = PreAggStatus.off("No aggregate on scan.");
+                }
                 if (candidates.size() == 1) {
                     // `candidates` only have base index.
-                    return scan.withMaterializedIndexSelected(preAgg, baseIndexId);
+                    return scan.withMaterializedIndexSelected(preAggStatus, baseIndexId);
                 } else {
-                    return scan.withMaterializedIndexSelected(preAgg,
+                    return scan.withMaterializedIndexSelected(preAggStatus,
                             selectBestIndex(candidates, scan, predicatesSupplier.get()));
                 }
             case DUP_KEYS:

diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
@@ -99,6 +99,11 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation,
 
     private final List<Long> selectedPartitionIds;
 
+    ///////////////////////////////////////////////////////////////////////////
+    // Members for hints.
+    ///////////////////////////////////////////////////////////////////////////
+    private final List<String> hints;
+
     public LogicalOlapScan(RelationId id, OlapTable table) {
         this(id, table, ImmutableList.of());
     }
@@ -107,19 +112,27 @@ public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier) {
         this(id, table, qualifier, Optional.empty(), Optional.empty(),
                 table.getPartitionIds(), false,
                 ImmutableList.of(), false,
-                -1, false, PreAggStatus.on(), ImmutableList.of());
+                -1, false, PreAggStatus.on(), ImmutableList.of(), ImmutableList.of());
     }
 
-    public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> specifiedPartitions) {
+    public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<String> hints) {
+        this(id, table, qualifier, Optional.empty(), Optional.empty(),
+                table.getPartitionIds(), false,
+                ImmutableList.of(), false,
+                -1, false, PreAggStatus.on(), ImmutableList.of(), hints);
+    }
+
+    public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> specifiedPartitions,
+            List<String> hints) {
         this(id, table, qualifier, Optional.empty(), Optional.empty(),
                 specifiedPartitions, false, ImmutableList.of(), false,
-                -1, false, PreAggStatus.on(), specifiedPartitions);
+                -1, false, PreAggStatus.on(), specifiedPartitions, hints);
     }
 
     public LogicalOlapScan(RelationId id, Table table, List<String> qualifier) {
         this(id, table, qualifier, Optional.empty(), Optional.empty(),
                 ((OlapTable) table).getPartitionIds(), false, ImmutableList.of(), false,
-                -1, false, PreAggStatus.on(), ImmutableList.of());
+                -1, false, PreAggStatus.on(), ImmutableList.of(), ImmutableList.of());
     }
 
     /**
@@ -129,7 +142,8 @@ public LogicalOlapScan(RelationId id, Table table, List<String> qualifier,
             Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties,
             List<Long> selectedPartitionIds, boolean partitionPruned,
             List<Long> selectedTabletIds, boolean tabletPruned,
-            long selectedIndexId, boolean indexSelected, PreAggStatus preAggStatus, List<Long> partitions) {
+            long selectedIndexId, boolean indexSelected, PreAggStatus preAggStatus, List<Long> partitions,
+            List<String> hints) {
 
         super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier,
                 groupExpression, logicalProperties);
@@ -142,6 +156,7 @@ public LogicalOlapScan(RelationId id, Table table, List<String> qualifier,
         this.manuallySpecifiedPartitions = ImmutableList.copyOf(partitions);
         this.selectedPartitionIds = ImmutableList.copyOf(
                 Objects.requireNonNull(selectedPartitionIds, "selectedPartitionIds can not be null"));
+        this.hints = Objects.requireNonNull(hints, "hints can not be null");
     }
 
     public List<Long> getSelectedPartitionIds() {
@@ -186,47 +201,49 @@ public boolean equals(Object o) {
                 && Objects.equals(selectedIndexId, ((LogicalOlapScan) o).selectedIndexId)
                 && Objects.equals(indexSelected, ((LogicalOlapScan) o).indexSelected)
                 && Objects.equals(selectedTabletIds, ((LogicalOlapScan) o).selectedTabletIds)
-                && Objects.equals(tabletPruned, ((LogicalOlapScan) o).tabletPruned);
+                && Objects.equals(tabletPruned, ((LogicalOlapScan) o).tabletPruned)
+                && Objects.equals(hints, ((LogicalOlapScan) o).hints);
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(id,
                 selectedPartitionIds, partitionPruned,
                 selectedIndexId, indexSelected,
-                selectedTabletIds, tabletPruned);
+                selectedTabletIds, tabletPruned,
+                hints);
     }
 
     @Override
     public LogicalOlapScan withGroupExpression(Optional<GroupExpression> groupExpression) {
         return new LogicalOlapScan(id, (Table) table, qualifier, groupExpression, Optional.of(getLogicalProperties()),
                 selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned,
-                selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions);
+                selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints);
     }
 
     @Override
     public LogicalOlapScan withLogicalProperties(Optional<LogicalProperties> logicalProperties) {
         return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), logicalProperties,
                 selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned,
-                selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions);
+                selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints);
     }
 
     public LogicalOlapScan withSelectedPartitionIds(List<Long> selectedPartitionIds) {
         return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()),
                 selectedPartitionIds, true, selectedTabletIds, tabletPruned,
-                selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions);
+                selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints);
     }
 
     public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, long indexId) {
         return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()),
                 selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned,
-                indexId, true, preAgg, manuallySpecifiedPartitions);
+                indexId, true, preAgg, manuallySpecifiedPartitions, hints);
     }
 
     public LogicalOlapScan withSelectedTabletIds(List<Long> selectedTabletIds) {
         return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()),
                 selectedPartitionIds, partitionPruned, selectedTabletIds, true,
-                selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions);
+                selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints);
     }
 
     @Override
@@ -301,4 +318,7 @@ public List<Long> getManuallySpecifiedPartitions() {
         return manuallySpecifiedPartitions;
     }
 
+    public List<String> getHints() {
+        return hints;
+    }
 }