Skip to content

Commit

Permalink
Compute and propagate logical properties
Browse files Browse the repository at this point in the history 
Logical properties are initially derived from constraints defined for
base tables and from properties of values nodes.These logical
properties hold for the result table produced by a plan node. These
base logical properties are then propagated through various query
operations including filters, projects, joins, and aggregations.
Logical properties are only computed by iterative planners that pass
a logical property provider as input. See the design doc linked from
issue 16413 for futher details. Such optimizers will be introduced by
next commit; however, there are test cases in this commit that trigger
logical property propgation. Note that if the session variable
exploit_constraints=false (the default now) no attempt is made to
compute logical properties and hence optimization rules that seek them
out will simply fail to fire.
  • Loading branch information
@simmend
simmend committed Jun 6, 2022
1 parent 43bfd15 commit 13c2211
Show file tree
Hide file tree
Showing 31 changed files with 4,466 additions and 25 deletions.
4 changes: 4 additions & 0 deletions presto-main/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.facebook.presto.sql.parser.SqlParser;
import com.facebook.presto.sql.planner.iterative.IterativeOptimizer;
import com.facebook.presto.sql.planner.iterative.Rule;
import com.facebook.presto.sql.planner.iterative.properties.LogicalPropertiesProviderImpl;
import com.facebook.presto.sql.planner.iterative.rule.AddIntermediateAggregations;
import com.facebook.presto.sql.planner.iterative.rule.CanonicalizeExpressions;
import com.facebook.presto.sql.planner.iterative.rule.CreatePartialTopN;
Expand Down Expand Up @@ -140,6 +141,7 @@
import com.facebook.presto.sql.planner.optimizations.TransformQuantifiedComparisonApplyToLateralJoin;
import com.facebook.presto.sql.planner.optimizations.UnaliasSymbolReferences;
import com.facebook.presto.sql.planner.optimizations.WindowFilterPushDown;
import com.facebook.presto.sql.relational.FunctionResolution;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.weakref.jmx.MBeanExporter;
Expand All @@ -149,6 +151,7 @@
import javax.inject.Inject;

import java.util.List;
import java.util.Optional;
import java.util.Set;

import static com.facebook.presto.sql.planner.ConnectorPlanOptimizerManager.PlanPhase.LOGICAL;
Expand Down Expand Up @@ -430,6 +433,7 @@ public PlanOptimizers(
ruleStats,
statsCalculator,
estimatedExchangesCostCalculator,
Optional.of(new LogicalPropertiesProviderImpl(new FunctionResolution(metadata.getFunctionAndTypeManager()))),
ImmutableSet.of(
new RemoveRedundantIdentityProjections(),
new PushAggregationThroughOuterJoin(metadata.getFunctionAndTypeManager()))),
Expand Down
10 changes: 9 additions & 1 deletion presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/GroupReference.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
package com.facebook.presto.sql.planner.iterative;

import com.facebook.presto.spi.SourceLocation;
import com.facebook.presto.spi.plan.LogicalProperties;
import com.facebook.presto.spi.plan.PlanNode;
import com.facebook.presto.spi.plan.PlanNodeId;
import com.facebook.presto.spi.relation.VariableReferenceExpression;
Expand All @@ -29,12 +30,14 @@ public class GroupReference
{
private final int groupId;
private final List<VariableReferenceExpression> outputs;
private final Optional<LogicalProperties> logicalProperties;

public GroupReference(Optional<SourceLocation> sourceLocation, PlanNodeId id, int groupId, List<VariableReferenceExpression> outputs)
public GroupReference(Optional<SourceLocation> sourceLocation, PlanNodeId id, int groupId, List<VariableReferenceExpression> outputs, Optional<LogicalProperties> logicalProperties)
{
super(sourceLocation, id);
this.groupId = groupId;
this.outputs = ImmutableList.copyOf(outputs);
this.logicalProperties = logicalProperties;
}

public int getGroupId()
Expand Down Expand Up @@ -65,4 +68,9 @@ public PlanNode replaceChildren(List<PlanNode> newChildren)
{
throw new UnsupportedOperationException();
}

public Optional<LogicalProperties> getLogicalProperties()
{
return logicalProperties;
}
}
24 changes: 22 additions & 2 deletions presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/IterativeOptimizer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import com.facebook.presto.matching.Matcher;
import com.facebook.presto.spi.PrestoException;
import com.facebook.presto.spi.WarningCollector;
import com.facebook.presto.spi.plan.LogicalPropertiesProvider;
import com.facebook.presto.spi.plan.PlanNode;
import com.facebook.presto.spi.plan.PlanNodeIdAllocator;
import com.facebook.presto.sql.planner.PlanVariableAllocator;
Expand Down Expand Up @@ -56,13 +57,24 @@ public class IterativeOptimizer
private final CostCalculator costCalculator;
private final List<PlanOptimizer> legacyRules;
private final RuleIndex ruleIndex;
private final Optional<LogicalPropertiesProvider> logicalPropertiesProvider;

public IterativeOptimizer(RuleStatsRecorder stats, StatsCalculator statsCalculator, CostCalculator costCalculator, Set<Rule<?>> rules)
{
this(stats, statsCalculator, costCalculator, ImmutableList.of(), rules);
this(stats, statsCalculator, costCalculator, ImmutableList.of(), Optional.empty(), rules);
}

public IterativeOptimizer(RuleStatsRecorder stats, StatsCalculator statsCalculator, CostCalculator costCalculator, Optional<LogicalPropertiesProvider> logicalPropertiesProvider, Set<Rule<?>> rules)
{
this(stats, statsCalculator, costCalculator, ImmutableList.of(), logicalPropertiesProvider, rules);
}

public IterativeOptimizer(RuleStatsRecorder stats, StatsCalculator statsCalculator, CostCalculator costCalculator, List<PlanOptimizer> legacyRules, Set<Rule<?>> newRules)
{
this(stats, statsCalculator, costCalculator, legacyRules, Optional.empty(), newRules);
}

public IterativeOptimizer(RuleStatsRecorder stats, StatsCalculator statsCalculator, CostCalculator costCalculator, List<PlanOptimizer> legacyRules, Optional<LogicalPropertiesProvider> logicalPropertiesProvider, Set<Rule<?>> newRules)
{
this.stats = requireNonNull(stats, "stats is null");
this.statsCalculator = requireNonNull(statsCalculator, "statsCalculator is null");
Expand All @@ -71,6 +83,7 @@ public IterativeOptimizer(RuleStatsRecorder stats, StatsCalculator statsCalculat
this.ruleIndex = RuleIndex.builder()
.register(newRules)
.build();
this.logicalPropertiesProvider = requireNonNull(logicalPropertiesProvider, "logicalPropertiesProvider is null");

stats.registerAll(newRules);
}
Expand All @@ -87,7 +100,14 @@ public PlanNode optimize(PlanNode plan, Session session, TypeProvider types, Pla
return plan;
}

Memo memo = new Memo(idAllocator, plan);
Memo memo;
if (SystemSessionProperties.isExploitConstraints(session)) {
memo = new Memo(idAllocator, plan, logicalPropertiesProvider);
}
else {
memo = new Memo(idAllocator, plan, Optional.empty());
}

Lookup lookup = Lookup.from(planNode -> Stream.of(memo.resolve(planNode)));
Matcher matcher = new PlanNodeMatcher(lookup);

Expand Down
50 changes: 37 additions & 13 deletions presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/Memo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@

import com.facebook.presto.cost.PlanCostEstimate;
import com.facebook.presto.cost.PlanNodeStatsEstimate;
import com.facebook.presto.spi.plan.LogicalProperties;
import com.facebook.presto.spi.plan.LogicalPropertiesProvider;
import com.facebook.presto.spi.plan.PlanNode;
import com.facebook.presto.spi.plan.PlanNodeIdAllocator;
import com.google.common.collect.HashMultiset;
Expand Down Expand Up @@ -69,12 +71,19 @@ public class Memo
private final int rootGroup;

private final Map<Integer, Group> groups = new HashMap<>();
private final Optional<LogicalPropertiesProvider> logicalPropertiesProvider;

private int nextGroupId = ROOT_GROUP_REF + 1;

public Memo(PlanNodeIdAllocator idAllocator, PlanNode plan)
{
this(idAllocator, plan, Optional.empty());
}

public Memo(PlanNodeIdAllocator idAllocator, PlanNode plan, Optional<LogicalPropertiesProvider> logicalPropertiesProvider)
{
this.idAllocator = idAllocator;
this.logicalPropertiesProvider = logicalPropertiesProvider;
rootGroup = insertRecursive(plan);
groups.get(rootGroup).incomingReferences.add(ROOT_GROUP_REF);
}
Expand All @@ -90,6 +99,12 @@ private Group getGroup(int group)
return groups.get(group);
}

public Optional<LogicalProperties> getLogicalProperties(int group)
{
checkArgument(groups.containsKey(group), "Invalid group: %s", group);
return groups.get(group).logicalProperties;
}

public PlanNode getNode(int group)
{
return getGroup(group).membership;
Expand Down Expand Up @@ -129,6 +144,14 @@ public PlanNode replace(int group, PlanNode node, String reason)

incrementReferenceCounts(node, group);
getGroup(group).membership = node;

if (logicalPropertiesProvider.isPresent()) {
// for now, we replace existing group logical properties with those computed for the new node
// as we cannot ensure equivalence for all plans in a group until we support functional dependencies
// once we can ensure equivalence we can simply reuse the previously computed properties for all plans in the group
LogicalProperties newLogicalProperties = node.computeLogicalProperties(logicalPropertiesProvider.get());
getGroup(group).logicalProperties = Optional.of(newLogicalProperties);
}
decrementReferenceCounts(old, group);
evictStatisticsAndCost(group);

Expand Down Expand Up @@ -212,11 +235,15 @@ private PlanNode insertChildrenAndRewrite(PlanNode node)
{
return node.replaceChildren(
node.getSources().stream()
.map(child -> new GroupReference(
node.getSourceLocation(),
idAllocator.getNextId(),
insertRecursive(child),
child.getOutputVariables()))
.map(child -> {
int childId = insertRecursive(child);
return new GroupReference(
node.getSourceLocation(),
idAllocator.getNextId(),
childId,
child.getOutputVariables(),
groups.get(childId).logicalProperties);
})
.collect(Collectors.toList()));
}

Expand All @@ -229,7 +256,7 @@ private int insertRecursive(PlanNode node)
int group = nextGroupId();
PlanNode rewritten = insertChildrenAndRewrite(node);

groups.put(group, Group.withMember(rewritten));
groups.put(group, new Group(rewritten, logicalPropertiesProvider.map(rewritten::computeLogicalProperties)));
incrementReferenceCounts(rewritten, group);

return group;
Expand All @@ -247,21 +274,18 @@ public int getGroupCount()

private static final class Group
{
static Group withMember(PlanNode member)
{
return new Group(member);
}

private PlanNode membership;
private final Multiset<Integer> incomingReferences = HashMultiset.create();
private PlanNode membership;
private Optional<LogicalProperties> logicalProperties;
@Nullable
private PlanNodeStatsEstimate stats;
@Nullable
private PlanCostEstimate cost;

private Group(PlanNode member)
private Group(PlanNode member, Optional<LogicalProperties> logicalProperties)
{
this.membership = requireNonNull(member, "member is null");
this.logicalProperties = logicalProperties;
}
}
}
Loading

0 comments on commit 13c2211

Please sign in to comment.