diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 89b22431afba7c..fdde0446961935 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -127,8 +127,13 @@ joinRelation // Just like `opt_plan_hints` in legacy CUP parser. joinHint - : LEFT_BRACKET identifier RIGHT_BRACKET #bracketStyleHint - | HINT_START identifier HINT_END #commentStyleHint + : LEFT_BRACKET identifier RIGHT_BRACKET #bracketJoinHint + | HINT_START identifier HINT_END #commentJoinHint + ; + +relationHint + : LEFT_BRACKET identifier (COMMA identifier)* RIGHT_BRACKET #bracketRelationHint + | HINT_START identifier (COMMA identifier)* HINT_END #commentRelationHint ; aggClause @@ -208,11 +213,11 @@ identifierSeq ; relationPrimary - : multipartIdentifier specifiedPartition? tableAlias lateralView* #tableName - | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery + : multipartIdentifier specifiedPartition? tableAlias relationHint? lateralView* #tableName + | LEFT_PAREN query RIGHT_PAREN tableAlias lateralView* #aliasedQuery | tvfName=identifier LEFT_PAREN (properties+=tvfProperty (COMMA properties+=tvfProperty)*)? - RIGHT_PAREN tableAlias #tableValuedFunction + RIGHT_PAREN tableAlias #tableValuedFunction ; tvfProperty diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index acef46f170a20b..431d75611fa8e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -32,6 +32,8 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import java.util.List; @@ -46,21 +48,29 @@ public class UnboundRelation extends LogicalRelation implements Unbound { private final List nameParts; private final List partNames; private final boolean isTempPart; + private final List hints; public UnboundRelation(RelationId id, List nameParts) { - this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false); + this(id, nameParts, Optional.empty(), Optional.empty(), ImmutableList.of(), false, ImmutableList.of()); } public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart) { - this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart); + this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, ImmutableList.of()); + } + + public UnboundRelation(RelationId id, List nameParts, List partNames, boolean isTempPart, + List hints) { + this(id, nameParts, Optional.empty(), Optional.empty(), partNames, isTempPart, hints); } public UnboundRelation(RelationId id, List nameParts, Optional groupExpression, - Optional logicalProperties, List partNames, boolean isTempPart) { + Optional logicalProperties, List partNames, boolean isTempPart, + List hints) { super(id, PlanType.LOGICAL_UNBOUND_RELATION, groupExpression, logicalProperties); this.nameParts = ImmutableList.copyOf(Objects.requireNonNull(nameParts, "nameParts should not null")); this.partNames = ImmutableList.copyOf(Objects.requireNonNull(partNames, "partNames should not null")); this.isTempPart = isTempPart; + this.hints = ImmutableList.copyOf(Objects.requireNonNull(hints, "hints should not be null.")); } @Override @@ -84,13 +94,14 @@ public LogicalProperties computeLogicalProperties() { @Override public Plan withGroupExpression(Optional groupExpression) { - return new UnboundRelation(id, nameParts, groupExpression, Optional.of(getLogicalProperties()), - partNames, isTempPart); + return new UnboundRelation(id, nameParts, groupExpression, Optional.of(getLogicalProperties()), partNames, + isTempPart, hints); } @Override public Plan withLogicalProperties(Optional logicalProperties) { - return new UnboundRelation(id, nameParts, Optional.empty(), logicalProperties, partNames, isTempPart); + return new UnboundRelation(id, nameParts, Optional.empty(), logicalProperties, partNames, + isTempPart, hints); } @Override @@ -100,10 +111,15 @@ public List computeOutput() { @Override public String toString() { - return Utils.toSqlString("UnboundRelation", + List args = Lists.newArrayList( "id", id, "nameParts", StringUtils.join(nameParts, ".") ); + if (CollectionUtils.isNotEmpty(hints)) { + args.add("hints"); + args.add(StringUtils.join(hints, ", ")); + } + return Utils.toSqlString("UnboundRelation", args.toArray()); } @Override @@ -147,4 +163,8 @@ public List getPartNames() { public boolean isTempPart() { return isTempPart; } + + public List getHints() { + return hints; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 0a8e313bbdda92..1fcc5b743fe4ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -29,9 +29,11 @@ import org.apache.doris.nereids.DorisParser.ArithmeticUnaryContext; import org.apache.doris.nereids.DorisParser.BitOperationContext; import org.apache.doris.nereids.DorisParser.BooleanLiteralContext; -import org.apache.doris.nereids.DorisParser.BracketStyleHintContext; +import org.apache.doris.nereids.DorisParser.BracketJoinHintContext; +import org.apache.doris.nereids.DorisParser.BracketRelationHintContext; import org.apache.doris.nereids.DorisParser.ColumnReferenceContext; -import org.apache.doris.nereids.DorisParser.CommentStyleHintContext; +import org.apache.doris.nereids.DorisParser.CommentJoinHintContext; +import org.apache.doris.nereids.DorisParser.CommentRelationHintContext; import org.apache.doris.nereids.DorisParser.ComparisonContext; import org.apache.doris.nereids.DorisParser.CreateRowPolicyContext; import org.apache.doris.nereids.DorisParser.CteContext; @@ -460,8 +462,16 @@ public LogicalPlan visitTableName(TableNameContext ctx) { partitionNames.addAll(visitIdentifierList(ctx.specifiedPartition().identifierList())); } } + + final List relationHints; + if (ctx.relationHint() != null) { + relationHints = typedVisit(ctx.relationHint()); + } else { + relationHints = ImmutableList.of(); + } + LogicalPlan checkedRelation = withCheckPolicy( - new UnboundRelation(RelationUtil.newRelationId(), tableId, partitionNames, isTempPart)); + new UnboundRelation(RelationUtil.newRelationId(), tableId, partitionNames, isTempPart, relationHints)); LogicalPlan plan = withTableAlias(checkedRelation, ctx.tableAlias()); for (LateralViewContext lateralViewContext : ctx.lateralView()) { plan = withGenerate(plan, lateralViewContext); @@ -1412,15 +1422,29 @@ private LogicalPlan withSelectHint(LogicalPlan logicalPlan, SelectHintContext hi } @Override - public String visitBracketStyleHint(BracketStyleHintContext ctx) { + public String visitBracketJoinHint(BracketJoinHintContext ctx) { return ctx.identifier().getText(); } @Override - public Object visitCommentStyleHint(CommentStyleHintContext ctx) { + public String visitCommentJoinHint(CommentJoinHintContext ctx) { return ctx.identifier().getText(); } + @Override + public List visitBracketRelationHint(BracketRelationHintContext ctx) { + return ctx.identifier().stream() + .map(RuleContext::getText) + .collect(ImmutableList.toImmutableList()); + } + + @Override + public Object visitCommentRelationHint(CommentRelationHintContext ctx) { + return ctx.identifier().stream() + .map(RuleContext::getText) + .collect(ImmutableList.toImmutableList()); + } + private LogicalPlan withProjection(LogicalPlan input, SelectColumnClauseContext selectCtx, Optional aggCtx, boolean isDistinct) { return ParserUtils.withOrigin(selectCtx, () -> { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 7d21e2895f9699..9011b80f0885eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -161,10 +161,10 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, List partIds = getPartitionIds(table, unboundRelation); if (!CollectionUtils.isEmpty(partIds)) { scan = new LogicalOlapScan(RelationUtil.newRelationId(), - (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), partIds); + (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), partIds, unboundRelation.getHints()); } else { scan = new LogicalOlapScan(RelationUtil.newRelationId(), - (OlapTable) table, ImmutableList.of(tableQualifier.get(1))); + (OlapTable) table, ImmutableList.of(tableQualifier.get(1)), unboundRelation.getHints()); } if (!Util.showHiddenColumns() && scan.getTable().hasDeleteSign() && !ConnectContext.get().getSessionVariable() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java index 48215db57dbfa4..230a3a71c66bf2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/AbstractSelectMaterializedIndexRule.java @@ -282,4 +282,8 @@ private int indexKeyPrefixMatchCount( } return matchCount; } + + protected boolean preAggEnabledByHint(LogicalOlapScan olapScan) { + return olapScan.getHints().stream().anyMatch("PREAGGOPEN"::equalsIgnoreCase); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java index 07a67fb63143d3..27143b9864a35a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithAggregate.java @@ -493,11 +493,17 @@ private SelectResult select( switch (table.getKeysType()) { case AGG_KEYS: case UNIQUE_KEYS: { - // Only checking pre-aggregation status by base index is enough for aggregate-keys and - // unique-keys OLAP table. - // Because the schemas in non-base materialized index are subsets of the schema of base index. - PreAggStatus preAggStatus = checkPreAggStatus(scan, table.getBaseIndexId(), predicates, - aggregateFunctions, groupingExprs); + final PreAggStatus preAggStatus; + if (preAggEnabledByHint(scan)) { + // PreAggStatus could be enabled by pre-aggregation hint for agg-keys and unique-keys. + preAggStatus = PreAggStatus.on(); + } else { + // Only checking pre-aggregation status by base index is enough for aggregate-keys and + // unique-keys OLAP table. + // Because the schemas in non-base materialized index are subsets of the schema of base index. + preAggStatus = checkPreAggStatus(scan, table.getBaseIndexId(), predicates, + aggregateFunctions, groupingExprs); + } if (preAggStatus.isOff()) { // return early if pre agg status if off. return new SelectResult(preAggStatus, scan.getTable().getBaseIndexId(), new ExprRewriteMap()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java index 4d55385c07fa32..3c30590a343f1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/mv/SelectMaterializedIndexWithoutAggregate.java @@ -128,12 +128,18 @@ private LogicalOlapScan select( .filter(index -> containAllRequiredColumns(index, scan, requiredScanOutputSupplier.get())) .collect(Collectors.toList()); - PreAggStatus preAgg = PreAggStatus.off("No aggregate on scan."); + final PreAggStatus preAggStatus; + if (preAggEnabledByHint(scan)) { + // PreAggStatus could be enabled by pre-aggregation hint for agg-keys and unique-keys. + preAggStatus = PreAggStatus.on(); + } else { + preAggStatus = PreAggStatus.off("No aggregate on scan."); + } if (candidates.size() == 1) { // `candidates` only have base index. - return scan.withMaterializedIndexSelected(preAgg, baseIndexId); + return scan.withMaterializedIndexSelected(preAggStatus, baseIndexId); } else { - return scan.withMaterializedIndexSelected(preAgg, + return scan.withMaterializedIndexSelected(preAggStatus, selectBestIndex(candidates, scan, predicatesSupplier.get())); } case DUP_KEYS: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index 577ff807cceb3d..db4b9e2f0eb71f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -99,6 +99,11 @@ public class LogicalOlapScan extends LogicalRelation implements CatalogRelation, private final List selectedPartitionIds; + /////////////////////////////////////////////////////////////////////////// + // Members for hints. + /////////////////////////////////////////////////////////////////////////// + private final List hints; + public LogicalOlapScan(RelationId id, OlapTable table) { this(id, table, ImmutableList.of()); } @@ -107,19 +112,27 @@ public LogicalOlapScan(RelationId id, OlapTable table, List qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, ImmutableList.of(), false, - -1, false, PreAggStatus.on(), ImmutableList.of()); + -1, false, PreAggStatus.on(), ImmutableList.of(), ImmutableList.of()); } - public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, List specifiedPartitions) { + public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, List hints) { + this(id, table, qualifier, Optional.empty(), Optional.empty(), + table.getPartitionIds(), false, + ImmutableList.of(), false, + -1, false, PreAggStatus.on(), ImmutableList.of(), hints); + } + + public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, List specifiedPartitions, + List hints) { this(id, table, qualifier, Optional.empty(), Optional.empty(), specifiedPartitions, false, ImmutableList.of(), false, - -1, false, PreAggStatus.on(), specifiedPartitions); + -1, false, PreAggStatus.on(), specifiedPartitions, hints); } public LogicalOlapScan(RelationId id, Table table, List qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty(), ((OlapTable) table).getPartitionIds(), false, ImmutableList.of(), false, - -1, false, PreAggStatus.on(), ImmutableList.of()); + -1, false, PreAggStatus.on(), ImmutableList.of(), ImmutableList.of()); } /** @@ -129,7 +142,8 @@ public LogicalOlapScan(RelationId id, Table table, List qualifier, Optional groupExpression, Optional logicalProperties, List selectedPartitionIds, boolean partitionPruned, List selectedTabletIds, boolean tabletPruned, - long selectedIndexId, boolean indexSelected, PreAggStatus preAggStatus, List partitions) { + long selectedIndexId, boolean indexSelected, PreAggStatus preAggStatus, List partitions, + List hints) { super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier, groupExpression, logicalProperties); @@ -142,6 +156,7 @@ public LogicalOlapScan(RelationId id, Table table, List qualifier, this.manuallySpecifiedPartitions = ImmutableList.copyOf(partitions); this.selectedPartitionIds = ImmutableList.copyOf( Objects.requireNonNull(selectedPartitionIds, "selectedPartitionIds can not be null")); + this.hints = Objects.requireNonNull(hints, "hints can not be null"); } public List getSelectedPartitionIds() { @@ -186,7 +201,8 @@ public boolean equals(Object o) { && Objects.equals(selectedIndexId, ((LogicalOlapScan) o).selectedIndexId) && Objects.equals(indexSelected, ((LogicalOlapScan) o).indexSelected) && Objects.equals(selectedTabletIds, ((LogicalOlapScan) o).selectedTabletIds) - && Objects.equals(tabletPruned, ((LogicalOlapScan) o).tabletPruned); + && Objects.equals(tabletPruned, ((LogicalOlapScan) o).tabletPruned) + && Objects.equals(hints, ((LogicalOlapScan) o).hints); } @Override @@ -194,39 +210,40 @@ public int hashCode() { return Objects.hash(id, selectedPartitionIds, partitionPruned, selectedIndexId, indexSelected, - selectedTabletIds, tabletPruned); + selectedTabletIds, tabletPruned, + hints); } @Override public LogicalOlapScan withGroupExpression(Optional groupExpression) { return new LogicalOlapScan(id, (Table) table, qualifier, groupExpression, Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned, - selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions); + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints); } @Override public LogicalOlapScan withLogicalProperties(Optional logicalProperties) { return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), logicalProperties, selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned, - selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions); + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints); } public LogicalOlapScan withSelectedPartitionIds(List selectedPartitionIds) { return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, true, selectedTabletIds, tabletPruned, - selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions); + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints); } public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, long indexId) { return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, tabletPruned, - indexId, true, preAgg, manuallySpecifiedPartitions); + indexId, true, preAgg, manuallySpecifiedPartitions, hints); } public LogicalOlapScan withSelectedTabletIds(List selectedTabletIds) { return new LogicalOlapScan(id, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, true, - selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions); + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints); } @Override @@ -301,4 +318,7 @@ public List getManuallySpecifiedPartitions() { return manuallySpecifiedPartitions; } + public List getHints() { + return hints; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java index ec6d6bf913d9d6..eded31d0ecc1b0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/mv/SelectRollupIndexTest.java @@ -379,4 +379,63 @@ public void testOnlyValueColumn1() throws Exception { public void testOnlyValueColumn2() throws Exception { singleTableTest("select v1 from t", "t", false); } + + @Test + public void testPreAggHint() throws Exception { + createTable(" CREATE TABLE `test_preagg_hint` (\n" + + " `k1` int(11) NULL,\n" + + " `k2` int(11) NULL,\n" + + " `v1` int(11) SUM NULL,\n" + + " `v2` int(11) SUM NULL\n" + + ") ENGINE=OLAP\n" + + "AGGREGATE KEY(`k1`, `k2`)\n" + + "COMMENT 'OLAP'\n" + + "DISTRIBUTED BY HASH(`k1`) BUCKETS 3\n" + + "PROPERTIES (\n" + + "\"replication_allocation\" = \"tag.location.default: 1\",\n" + + "\"in_memory\" = \"false\",\n" + + "\"storage_format\" = \"V2\",\n" + + "\"disable_auto_compaction\" = \"false\"\n" + + ");"); + + addRollup("alter table test_preagg_hint add rollup r1(k1, k2, v1)"); + + // no pre-agg hint + String queryWithoutHint = "select k1, v1 from test_preagg_hint"; + // legacy planner + Assertions.assertTrue(getSQLPlanOrErrorMsg(queryWithoutHint).contains( + "TABLE: default_cluster:test.test_preagg_hint(r1), PREAGGREGATION: OFF. Reason: No AggregateInfo")); + // nereids planner + PlanChecker.from(connectContext) + .analyze(queryWithoutHint) + .rewrite() + .matches(logicalOlapScan().when(scan -> { + Assertions.assertTrue(scan.getHints().isEmpty()); + Assertions.assertEquals("r1", scan.getSelectedMaterializedIndexName().get()); + PreAggStatus preAggStatus = scan.getPreAggStatus(); + Assertions.assertTrue(preAggStatus.isOff()); + Assertions.assertEquals("No aggregate on scan.", preAggStatus.getOffReason()); + return true; + })); + + // has pre-agg hint + String queryWithHint = "select k1, v1 from test_preagg_hint /*+ PREAGGOPEN*/"; + // legacy planner + Assertions.assertTrue(getSQLPlanOrErrorMsg(queryWithHint).contains( + "TABLE: default_cluster:test.test_preagg_hint(r1), PREAGGREGATION: ON")); + // nereids planner + PlanChecker.from(connectContext) + .analyze(queryWithHint) + .rewrite() + .matches(logicalOlapScan().when(scan -> { + Assertions.assertEquals(1, scan.getHints().size()); + Assertions.assertEquals("PREAGGOPEN", scan.getHints().get(0)); + Assertions.assertEquals("r1", scan.getSelectedMaterializedIndexName().get()); + PreAggStatus preAggStatus = scan.getPreAggStatus(); + Assertions.assertTrue(preAggStatus.isOn()); + return true; + })); + + dropTable("test_preagg_hint", true); + } }