diff --git a/presto-common/src/main/java/com/facebook/presto/common/plan/PlanCanonicalizationStrategy.java b/presto-common/src/main/java/com/facebook/presto/common/plan/PlanCanonicalizationStrategy.java index 7cccb28310670..c6cbbfdfef808 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/plan/PlanCanonicalizationStrategy.java +++ b/presto-common/src/main/java/com/facebook/presto/common/plan/PlanCanonicalizationStrategy.java @@ -13,11 +13,6 @@ */ package com.facebook.presto.common.plan; -import java.util.List; - -import static java.util.Arrays.asList; -import static java.util.Collections.unmodifiableList; - public enum PlanCanonicalizationStrategy { /** @@ -31,7 +26,7 @@ public enum PlanCanonicalizationStrategy * * This is used in context of fragment result caching */ - DEFAULT, + DEFAULT(0), /** * CONNECTOR strategy will canonicalize plan according to DEFAULT strategy, and additionally * canoncialize `TableScanNode` by giving a connector specific implementation. Unlike DEFAULT strategy, @@ -46,9 +41,9 @@ public enum PlanCanonicalizationStrategy * * This is used in context of history based optimizations. */ - CONNECTOR, + CONNECTOR(1), /** - * REMOVE_SAFE_CONSTANTS strategy is used to canonicalize plan with + * IGNORE_SAFE_CONSTANTS strategy is used to canonicalize plan with * CONNECTOR strategy and will additionally remove constants from plan * which are not bound to have impact on plan statistics. * @@ -61,15 +56,37 @@ public enum PlanCanonicalizationStrategy * * This is used in context of history based optimizations. */ - REMOVE_SAFE_CONSTANTS; + IGNORE_SAFE_CONSTANTS(2), + + /** + * IGNORE_SCAN_CONSTANTS further relaxes over the IGNORE_SAFE_CONSTANTS strategy. + * In IGNORE_SAFE_CONSTANTS, only predicate on partitioned column in scan node is canonicalized, but + * in IGNORE_SCAN_CONSTANTS, predicates on non-partitioned columns in scan node are also canonicalized + * + * For example: + * `SELECT *, 1 FROM table` will be equivalent to `SELECT *, 2 FROM table` + * `SELECT * FROM table WHERE id = 1` will also be equivalent to `SELECT * FROM table WHERE id = 1000` even if id is not partitioned column + * + * This is used in context of history based optimizations. + */ + IGNORE_SCAN_CONSTANTS(3); /** * Creates a list of PlanCanonicalizationStrategy to be used for history based optimizations. * Output is ordered by decreasing accuracy of statistics, at benefit of more coverage. * TODO: Remove CONNECTOR strategy */ - public static List historyBasedPlanCanonicalizationStrategyList() + + // Smaller value means more accurate + private final int errorLevel; + + PlanCanonicalizationStrategy(int errorLevel) + { + this.errorLevel = errorLevel; + } + + public int getErrorLevel() { - return unmodifiableList(asList(REMOVE_SAFE_CONSTANTS)); + return errorLevel; } } diff --git a/presto-docs/src/main/sphinx/admin/properties.rst b/presto-docs/src/main/sphinx/admin/properties.rst index 1d8652cd598a6..4c4de93937d00 100644 --- a/presto-docs/src/main/sphinx/admin/properties.rst +++ b/presto-docs/src/main/sphinx/admin/properties.rst @@ -795,6 +795,14 @@ Optimizer Properties Extract expressions which have constant value from filter and assignment expressions, and replace the expressions with constant value. +``optimizer.history-based-optimizer-plan-canonicalization-strategies`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + * **Type:** ``string`` + * **Default value:** ``IGNORE_SAFE_CONSTANTS`` + + Plan canonicalization strategies used to canonicalize a query plan for history based optimization. + Planner Properties -------------------------------------- diff --git a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableLayoutHandle.java b/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableLayoutHandle.java index 1d7f19202e5eb..8eb05363e8211 100644 --- a/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableLayoutHandle.java +++ b/presto-hive/src/main/java/com/facebook/presto/hive/HiveTableLayoutHandle.java @@ -290,7 +290,7 @@ private TupleDomain getConstraint(PlanCanonicalizationStrategy can .transform(ColumnHandle.class::cast) .intersect(constraint); - constraint = constraint.canonicalize(HiveTableLayoutHandle::isPartitionKey); + constraint = canonicalizationStrategy.equals(PlanCanonicalizationStrategy.IGNORE_SCAN_CONSTANTS) ? constraint.canonicalize(x -> true) : constraint.canonicalize(HiveTableLayoutHandle::isPartitionKey); return constraint; } @@ -305,7 +305,7 @@ public static TupleDomain canonicalizeDomainPredicate(TupleDomain false); } diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveCanonicalPlanGenerator.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveCanonicalPlanGenerator.java index 60bd3fe10967c..cec7f7d9d0008 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveCanonicalPlanGenerator.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveCanonicalPlanGenerator.java @@ -39,7 +39,8 @@ import static com.facebook.presto.SystemSessionProperties.REWRITE_EXPRESSION_WITH_CONSTANT_EXPRESSION; import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.CONNECTOR; -import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.REMOVE_SAFE_CONSTANTS; +import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.IGNORE_SAFE_CONSTANTS; +import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.IGNORE_SCAN_CONSTANTS; import static com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG; import static com.facebook.presto.hive.HiveSessionProperties.PUSHDOWN_FILTER_ENABLED; import static com.facebook.presto.sql.planner.CanonicalPlanGenerator.generateCanonicalPlan; @@ -112,13 +113,37 @@ public void testCanonicalizationStrategies() pushdownFilterEnabled(), "SELECT orderkey from test_orders where ds = '2020-09-01' AND orderkey < 10", "SELECT orderkey from test_orders where ds = '2020-09-02' AND orderkey < 20", - REMOVE_SAFE_CONSTANTS); + IGNORE_SAFE_CONSTANTS); assertSameCanonicalLeafPlan( pushdownFilterEnabled(), "SELECT orderkey, CAST('1' AS VARCHAR) from test_orders where ds = '2020-09-01' AND orderkey < 10 AND ts >= '00:01'", "SELECT orderkey, CAST('11' AS VARCHAR) from test_orders where ds = '2020-09-02' AND orderkey < 10 AND ts >= '00:02'", - REMOVE_SAFE_CONSTANTS); + IGNORE_SAFE_CONSTANTS); + + assertDifferentCanonicalLeafPlan( + pushdownFilterEnabled(), + "SELECT orderkey, CAST('1' AS VARCHAR) from test_orders where ds = '2020-09-01' AND orderkey = 10", + "SELECT orderkey, CAST('11' AS VARCHAR) from test_orders where ds = '2020-09-02' AND orderkey = 20", + IGNORE_SAFE_CONSTANTS); + + assertDifferentCanonicalLeafPlan( + pushdownFilterEnabled(), + "SELECT orderkey from test_orders where ds = '2020-09-01' AND orderkey < 10", + "SELECT orderkey from test_orders where ds = '2020-09-02' AND orderkey < 20", + IGNORE_SCAN_CONSTANTS); + + assertSameCanonicalLeafPlan( + pushdownFilterEnabled(), + "SELECT orderkey, CAST('1' AS VARCHAR) from test_orders where ds = '2020-09-01' AND orderkey < 10 AND ts >= '00:01'", + "SELECT orderkey, CAST('11' AS VARCHAR) from test_orders where ds = '2020-09-02' AND orderkey < 10 AND ts >= '00:02'", + IGNORE_SCAN_CONSTANTS); + + assertSameCanonicalLeafPlan( + pushdownFilterEnabled(), + "SELECT orderkey, CAST('1' AS VARCHAR) from test_orders where ds = '2020-09-01' AND orderkey = 10", + "SELECT orderkey, CAST('11' AS VARCHAR) from test_orders where ds = '2020-09-02' AND orderkey = 20", + IGNORE_SCAN_CONSTANTS); } finally { queryRunner.execute("DROP TABLE IF EXISTS test_orders"); diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveCanonicalPlanHashes.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveCanonicalPlanHashes.java index 4ba706770b5be..19c5a6efbaf3e 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveCanonicalPlanHashes.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveCanonicalPlanHashes.java @@ -43,7 +43,8 @@ import static com.facebook.presto.SystemSessionProperties.USE_HISTORY_BASED_PLAN_STATISTICS; import static com.facebook.presto.SystemSessionProperties.USE_PERFECTLY_CONSISTENT_HISTORIES; import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.CONNECTOR; -import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.REMOVE_SAFE_CONSTANTS; +import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.IGNORE_SAFE_CONSTANTS; +import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.IGNORE_SCAN_CONSTANTS; import static com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG; import static com.facebook.presto.hive.HiveSessionProperties.PUSHDOWN_FILTER_ENABLED; import static com.facebook.presto.sql.planner.CanonicalPlanGenerator.generateCanonicalPlan; @@ -120,11 +121,20 @@ public void testCanonicalizationStrategies() assertSamePlanHash( "SELECT orderkey, CAST(1 AS VARCHAR) from test_orders where ds = '2020-09-01' AND orderkey < 10", "SELECT orderkey, CAST(2 AS VARCHAR) from test_orders where ds = '2020-09-02' AND orderkey < 10", - REMOVE_SAFE_CONSTANTS); + IGNORE_SAFE_CONSTANTS); assertDifferentPlanHash( "SELECT orderkey, CAST(1 AS VARCHAR) from test_orders where ds = '2020-09-01' AND orderkey < 10", "SELECT orderkey, CAST(1 AS VARCHAR) from test_orders where ds = '2020-09-02' AND orderkey < 20", - REMOVE_SAFE_CONSTANTS); + IGNORE_SAFE_CONSTANTS); + + assertSamePlanHash( + "SELECT orderkey, CAST(1 AS VARCHAR) from test_orders where ds = '2020-09-01' AND orderkey < 10", + "SELECT orderkey, CAST(2 AS VARCHAR) from test_orders where ds = '2020-09-02' AND orderkey < 10", + IGNORE_SCAN_CONSTANTS); + assertDifferentPlanHash( + "SELECT orderkey, CAST(1 AS VARCHAR) from test_orders where ds = '2020-09-01' AND orderkey < 10", + "SELECT orderkey, CAST(1 AS VARCHAR) from test_orders where ds = '2020-09-02' AND orderkey < 20", + IGNORE_SCAN_CONSTANTS); assertSamePlanHash( "INSERT INTO test_orders select * from test_orders", diff --git a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveHistoryBasedStatsTracking.java b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveHistoryBasedStatsTracking.java index 5c25c2bca80c0..2d999d3ab9d63 100644 --- a/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveHistoryBasedStatsTracking.java +++ b/presto-hive/src/test/java/com/facebook/presto/hive/TestHiveHistoryBasedStatsTracking.java @@ -29,9 +29,13 @@ import com.facebook.presto.tests.AbstractTestQueryFramework; import com.facebook.presto.tests.DistributedQueryRunner; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.intellij.lang.annotations.Language; import org.testng.annotations.Test; +import java.util.Map; + +import static com.facebook.presto.SystemSessionProperties.HISTORY_BASED_OPTIMIZATION_PLAN_CANONICALIZATION_STRATEGY; import static com.facebook.presto.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE; import static com.facebook.presto.SystemSessionProperties.PARTIAL_AGGREGATION_STRATEGY; import static com.facebook.presto.SystemSessionProperties.RESTRICT_HISTORY_BASED_OPTIMIZATION_TO_COMPLEX_QUERY; @@ -93,6 +97,38 @@ public void testHistoryBasedStatsCalculator() } } + @Test + public void testHistoryBasedStatsCalculatorMultipleStrategies() + { + try { + getQueryRunner().execute("CREATE TABLE test_orders WITH (partitioned_by = ARRAY['ds', 'ts']) AS " + + "SELECT orderkey, orderpriority, comment, custkey, '2020-09-01' as ds, '00:01' as ts FROM orders WHERE orderkey < 1000 " + + "UNION ALL " + + "SELECT orderkey, orderpriority, comment, custkey, '2020-09-02' as ds, '00:02' as ts FROM orders WHERE orderkey >= 1000 AND orderkey < 2000"); + + // CBO Statistics + assertPlan( + "SELECT *, 1 FROM test_orders where ds = '2020-09-01' and orderpriority = '1-URGENT'", + anyTree(node(ProjectNode.class, any())).withOutputRowCount(51.0)); + + // HBO Statistics + executeAndTrackHistory("SELECT *, 1 FROM test_orders where ds = '2020-09-01' and orderpriority = '1-URGENT'", + createSession(ImmutableMap.of(HISTORY_BASED_OPTIMIZATION_PLAN_CANONICALIZATION_STRATEGY, "IGNORE_SAFE_CONSTANTS,IGNORE_SCAN_CONSTANTS"))); + assertPlan(createSession(ImmutableMap.of(HISTORY_BASED_OPTIMIZATION_PLAN_CANONICALIZATION_STRATEGY, "IGNORE_SAFE_CONSTANTS,IGNORE_SCAN_CONSTANTS")), + "SELECT *, 2 FROM test_orders where ds = '2020-09-02' and orderpriority = '1-URGENT'", + anyTree(node(ProjectNode.class, any()).withOutputRowCount(48))); + assertPlan(createSession(ImmutableMap.of(HISTORY_BASED_OPTIMIZATION_PLAN_CANONICALIZATION_STRATEGY, "IGNORE_SAFE_CONSTANTS,IGNORE_SCAN_CONSTANTS")), + "SELECT *, 2 FROM test_orders where ds = '2020-09-02' and orderpriority = '2-HIGH'", + anyTree(node(ProjectNode.class, any()).withOutputRowCount(48))); + assertPlan(createSession(ImmutableMap.of(HISTORY_BASED_OPTIMIZATION_PLAN_CANONICALIZATION_STRATEGY, "IGNORE_SAFE_CONSTANTS")), + "SELECT *, 2 FROM test_orders where ds = '2020-09-02' and orderpriority = '2-HIGH'", + anyTree(node(ProjectNode.class, any()).withOutputRowCount(49.6))); + } + finally { + getQueryRunner().execute("DROP TABLE IF EXISTS test_orders"); + } + } + @Test public void testInsertTable() { @@ -164,7 +200,7 @@ public void testPartialAggStatistics() "SELECT orderkey, orderpriority, comment, custkey, '2020-09-01' as ds, '00:01' as ts FROM orders where orderkey < 2000 "); String query = "SELECT count(*) FROM test_orders group by custkey"; - Session session = createSession("always"); + Session session = createSession(ImmutableMap.of(PARTIAL_AGGREGATION_STRATEGY, "always")); Plan plan = plan(query, session); assertTrue(PlanNodeSearcher.searchFrom(plan.getRoot()) @@ -173,9 +209,9 @@ public void testPartialAggStatistics() .isPresent()); // collect HBO Statistics - executeAndTrackHistory(query, createSession("always")); + executeAndTrackHistory(query, createSession(ImmutableMap.of(PARTIAL_AGGREGATION_STRATEGY, "always"))); - plan = plan(query, createSession("automatic")); + plan = plan(query, createSession(ImmutableMap.of(PARTIAL_AGGREGATION_STRATEGY, "automatic"))); assertTrue(PlanNodeSearcher.searchFrom(plan.getRoot()) .where(node -> node instanceof AggregationNode && ((AggregationNode) node).getStep() == AggregationNode.Step.PARTIAL).findAll().isEmpty()); @@ -196,11 +232,11 @@ public void testPartialAggStatisticsGroupByPartKey() // collect HBO Statistics String queryGBPartitionKey = "SELECT ds FROM test_orders group by ds"; - Plan plan = plan(queryGBPartitionKey, createSession("always")); + Plan plan = plan(queryGBPartitionKey, createSession(ImmutableMap.of(PARTIAL_AGGREGATION_STRATEGY, "always"))); assertTrue(PlanNodeSearcher.searchFrom(plan.getRoot()) .where(node -> node instanceof AggregationNode && ((AggregationNode) node).getStep() == AggregationNode.Step.PARTIAL).findFirst().isPresent()); - executeAndTrackHistory(queryGBPartitionKey, createSession("always")); + executeAndTrackHistory(queryGBPartitionKey, createSession(ImmutableMap.of(PARTIAL_AGGREGATION_STRATEGY, "always"))); } finally { getQueryRunner().execute("DROP TABLE IF EXISTS test_orders"); @@ -225,19 +261,19 @@ private void executeAndTrackHistory(String sql, Session session) private Session defaultSession() { - return createSession("automatic"); + return createSession(ImmutableMap.of(PARTIAL_AGGREGATION_STRATEGY, "automatic")); } - private Session createSession(String partialAggregationStrategy) + private Session createSession(Map properties) { - return Session.builder(getQueryRunner().getDefaultSession()) + Session.SessionBuilder builder = Session.builder(getQueryRunner().getDefaultSession()) .setSystemProperty(USE_HISTORY_BASED_PLAN_STATISTICS, "true") .setSystemProperty(TRACK_HISTORY_BASED_PLAN_STATISTICS, "true") .setSystemProperty(JOIN_DISTRIBUTION_TYPE, "automatic") - .setSystemProperty(PARTIAL_AGGREGATION_STRATEGY, partialAggregationStrategy) .setSystemProperty(USE_PARTIAL_AGGREGATION_HISTORY, "true") .setCatalogSessionProperty(HIVE_CATALOG, PUSHDOWN_FILTER_ENABLED, "true") - .setSystemProperty(RESTRICT_HISTORY_BASED_OPTIMIZATION_TO_COMPLEX_QUERY, "false") - .build(); + .setSystemProperty(RESTRICT_HISTORY_BASED_OPTIMIZATION_TO_COMPLEX_QUERY, "false"); + properties.forEach((property, value) -> builder.setSystemProperty(property, value)); + return builder.build(); } } diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index b59171f20d854..cc7dd40353971 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -14,6 +14,7 @@ package com.facebook.presto; import com.facebook.presto.common.WarningHandlingLevel; +import com.facebook.presto.common.plan.PlanCanonicalizationStrategy; import com.facebook.presto.execution.QueryManagerConfig; import com.facebook.presto.execution.QueryManagerConfig.ExchangeMaterializationStrategy; import com.facebook.presto.execution.TaskManagerConfig; @@ -43,12 +44,14 @@ import com.facebook.presto.sql.analyzer.FeaturesConfig.SingleStreamSpillerChoice; import com.facebook.presto.sql.planner.CompilerConfig; import com.facebook.presto.tracing.TracingConfig; +import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import io.airlift.units.DataSize; import io.airlift.units.Duration; import javax.inject.Inject; +import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.OptionalInt; @@ -73,6 +76,7 @@ import static com.facebook.presto.sql.analyzer.FeaturesConfig.PartialAggregationStrategy.ALWAYS; import static com.facebook.presto.sql.analyzer.FeaturesConfig.PartialAggregationStrategy.NEVER; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableList.toImmutableList; import static java.lang.Boolean.TRUE; import static java.lang.Math.min; import static java.lang.String.format; @@ -264,6 +268,7 @@ public final class SystemSessionProperties public static final String HISTORY_BASED_OPTIMIZER_TIMEOUT_LIMIT = "history_based_optimizer_timeout_limit"; public static final String RESTRICT_HISTORY_BASED_OPTIMIZATION_TO_COMPLEX_QUERY = "restrict_history_based_optimization_to_complex_query"; public static final String HISTORY_INPUT_TABLE_STATISTICS_MATCHING_THRESHOLD = "history_input_table_statistics_matching_threshold"; + public static final String HISTORY_BASED_OPTIMIZATION_PLAN_CANONICALIZATION_STRATEGY = "history_based_optimization_plan_canonicalization_strategy"; public static final String MAX_LEAF_NODES_IN_PLAN = "max_leaf_nodes_in_plan"; public static final String LEAF_NODE_LIMIT_ENABLED = "leaf_node_limit_enabled"; public static final String PUSH_REMOTE_EXCHANGE_THROUGH_GROUP_ID = "push_remote_exchange_through_group_id"; @@ -1528,6 +1533,14 @@ public SystemSessionProperties( "When the size difference between current table and history table exceed this threshold, do not match history statistics", 0.0, true), + stringProperty( + HISTORY_BASED_OPTIMIZATION_PLAN_CANONICALIZATION_STRATEGY, + format("The plan canonicalization strategies used for history based optimization, the strategies will be applied based on the accuracy of the strategies, from more accurate to less accurate. Options are %s", + Stream.of(PlanCanonicalizationStrategy.values()) + .map(PlanCanonicalizationStrategy::name) + .collect(joining(","))), + featuresConfig.getHistoryBasedOptimizerPlanCanonicalizationStrategies(), + false), new PropertyMetadata<>( MAX_LEAF_NODES_IN_PLAN, "Maximum number of leaf nodes in the logical plan of SQL statement", @@ -2938,6 +2951,20 @@ public static double getHistoryInputTableStatisticsMatchingThreshold(Session ses return session.getSystemProperty(HISTORY_INPUT_TABLE_STATISTICS_MATCHING_THRESHOLD, Double.class); } + public static List getHistoryOptimizationPlanCanonicalizationStrategies(Session session) + { + List strategyList; + try { + strategyList = Splitter.on(",").trimResults().splitToList(session.getSystemProperty(HISTORY_BASED_OPTIMIZATION_PLAN_CANONICALIZATION_STRATEGY, String.class)).stream() + .map(x -> PlanCanonicalizationStrategy.valueOf(x)).sorted(Comparator.comparingInt(PlanCanonicalizationStrategy::getErrorLevel)).collect(toImmutableList()); + } + catch (Exception e) { + strategyList = ImmutableList.of(); + } + + return strategyList; + } + public static boolean shouldPushRemoteExchangeThroughGroupId(Session session) { return session.getSystemProperty(PUSH_REMOTE_EXCHANGE_THROUGH_GROUP_ID, Boolean.class); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java index 5db802c2e8836..fcdcec906cda9 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsCalculator.java @@ -41,8 +41,8 @@ import static com.facebook.presto.common.RuntimeMetricName.HISTORY_OPTIMIZER_QUERY_REGISTRATION_GET_PLAN_NODE_HASHES; import static com.facebook.presto.common.RuntimeMetricName.HISTORY_OPTIMIZER_QUERY_REGISTRATION_GET_STATISTICS; import static com.facebook.presto.common.RuntimeUnit.NANO; -import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.historyBasedPlanCanonicalizationStrategyList; import static com.facebook.presto.cost.HistoricalPlanStatisticsUtil.getPredictedPlanStatistics; +import static com.facebook.presto.cost.HistoryBasedPlanStatisticsManager.historyBasedPlanCanonicalizationStrategyList; import static com.facebook.presto.sql.planner.iterative.Plans.resolveGroupReferences; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.graph.Traverser.forTree; @@ -159,7 +159,7 @@ private Map getPlanNodeHashes(Pl PlanNode statsEquivalentPlanNode = plan.getStatsEquivalentPlanNode().get(); ImmutableMap.Builder allHashesBuilder = ImmutableMap.builder(); - for (PlanCanonicalizationStrategy strategy : historyBasedPlanCanonicalizationStrategyList()) { + for (PlanCanonicalizationStrategy strategy : historyBasedPlanCanonicalizationStrategyList(session)) { Optional hash = planCanonicalInfoProvider.hash(session, statsEquivalentPlanNode, strategy, cacheOnly); if (hash.isPresent()) { allHashesBuilder.put(strategy, new PlanNodeWithHash(statsEquivalentPlanNode, hash)); @@ -189,7 +189,7 @@ private PlanNodeStatsEstimate getStatistics(PlanNode planNode, Session session, } double historyMatchingThreshold = getHistoryInputTableStatisticsMatchingThreshold(session) > 0 ? getHistoryInputTableStatisticsMatchingThreshold(session) : config.getHistoryMatchingThreshold(); // Return statistics corresponding to first strategy that we find, in order specified by `historyBasedPlanCanonicalizationStrategyList` - for (PlanCanonicalizationStrategy strategy : historyBasedPlanCanonicalizationStrategyList()) { + for (PlanCanonicalizationStrategy strategy : historyBasedPlanCanonicalizationStrategyList(session)) { for (Map.Entry entry : statistics.entrySet()) { if (allHashes.containsKey(strategy) && entry.getKey().getHash().isPresent() && allHashes.get(strategy).equals(entry.getKey())) { Optional> inputTableStatistics = getPlanNodeInputTableStatistics(plan, session, true); diff --git a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsManager.java b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsManager.java index 2a576cbe45229..8e68fdfadfe5b 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsManager.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsManager.java @@ -13,6 +13,8 @@ */ package com.facebook.presto.cost; +import com.facebook.presto.Session; +import com.facebook.presto.common.plan.PlanCanonicalizationStrategy; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.metadata.SessionPropertyManager; import com.facebook.presto.spi.statistics.EmptyPlanStatisticsProvider; @@ -23,6 +25,9 @@ import com.fasterxml.jackson.databind.SerializationFeature; import com.google.inject.Inject; +import java.util.List; + +import static com.facebook.presto.SystemSessionProperties.getHistoryOptimizationPlanCanonicalizationStrategies; import static java.util.Objects.requireNonNull; public class HistoryBasedPlanStatisticsManager @@ -69,4 +74,9 @@ public PlanCanonicalInfoProvider getPlanCanonicalInfoProvider() { return planCanonicalInfoProvider; } + + public static List historyBasedPlanCanonicalizationStrategyList(Session session) + { + return getHistoryOptimizationPlanCanonicalizationStrategies(session); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java index 1fd4622919191..e5f79678d5c8d 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java @@ -55,10 +55,10 @@ import static com.facebook.presto.SystemSessionProperties.getHistoryBasedOptimizerTimeoutLimit; import static com.facebook.presto.SystemSessionProperties.trackHistoryBasedPlanStatisticsEnabled; import static com.facebook.presto.SystemSessionProperties.trackPartialAggregationHistory; -import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.historyBasedPlanCanonicalizationStrategyList; import static com.facebook.presto.common.resourceGroups.QueryType.INSERT; import static com.facebook.presto.common.resourceGroups.QueryType.SELECT; import static com.facebook.presto.cost.HistoricalPlanStatisticsUtil.updatePlanStatistics; +import static com.facebook.presto.cost.HistoryBasedPlanStatisticsManager.historyBasedPlanCanonicalizationStrategyList; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SCALED_WRITER_DISTRIBUTION; import static com.facebook.presto.sql.planner.planPrinter.PlanNodeStatsSummarizer.aggregateStageStats; import static com.google.common.collect.ImmutableList.toImmutableList; @@ -181,7 +181,7 @@ public Map getQueryStats(QueryIn } PlanNode statsEquivalentPlanNode = planNode.getStatsEquivalentPlanNode().get(); - for (PlanCanonicalizationStrategy strategy : historyBasedPlanCanonicalizationStrategyList()) { + for (PlanCanonicalizationStrategy strategy : historyBasedPlanCanonicalizationStrategyList(session)) { Optional planNodeCanonicalInfo = Optional.ofNullable( canonicalInfoMap.get(new CanonicalPlan(statsEquivalentPlanNode, strategy))); if (planNodeCanonicalInfo.isPresent()) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index 0a9dbd3e02efd..1b01030773265 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -98,6 +98,7 @@ public class FeaturesConfig private boolean usePerfectlyConsistentHistories; private int historyCanonicalPlanNodeLimit = 1000; private Duration historyBasedOptimizerTimeout = new Duration(10, SECONDS); + private String historyBasedOptimizerPlanCanonicalizationStrategies = "IGNORE_SAFE_CONSTANTS"; private boolean redistributeWrites = true; private boolean scaleWriters; private DataSize writerMinSize = new DataSize(32, MEGABYTE); @@ -936,6 +937,19 @@ public FeaturesConfig setHistoryBasedOptimizerTimeout(Duration historyBasedOptim return this; } + @NotNull + public String getHistoryBasedOptimizerPlanCanonicalizationStrategies() + { + return historyBasedOptimizerPlanCanonicalizationStrategies; + } + + @Config("optimizer.history-based-optimizer-plan-canonicalization-strategies") + public FeaturesConfig setHistoryBasedOptimizerPlanCanonicalizationStrategies(String historyBasedOptimizerPlanCanonicalizationStrategies) + { + this.historyBasedOptimizerPlanCanonicalizationStrategies = historyBasedOptimizerPlanCanonicalizationStrategies; + return this; + } + public AggregationPartitioningMergingStrategy getAggregationPartitioningMergingStrategy() { return aggregationPartitioningMergingStrategy; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/CachingPlanCanonicalInfoProvider.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/CachingPlanCanonicalInfoProvider.java index 2e47ba101d842..f8d51610e93eb 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/CachingPlanCanonicalInfoProvider.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/CachingPlanCanonicalInfoProvider.java @@ -39,7 +39,7 @@ import static com.facebook.presto.SystemSessionProperties.getHistoryBasedOptimizerTimeoutLimit; import static com.facebook.presto.common.RuntimeUnit.NANO; -import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.historyBasedPlanCanonicalizationStrategyList; +import static com.facebook.presto.cost.HistoryBasedPlanStatisticsManager.historyBasedPlanCanonicalizationStrategyList; import static com.google.common.hash.Hashing.sha256; import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; @@ -69,7 +69,7 @@ public Optional hash(Session session, PlanNode planNode, PlanCanonicaliz @Override public Optional> getInputTableStatistics(Session session, PlanNode planNode, boolean cacheOnly) { - CacheKey key = new CacheKey(planNode, historyBasedPlanCanonicalizationStrategyList().get(0)); + CacheKey key = new CacheKey(planNode, historyBasedPlanCanonicalizationStrategyList(session).get(0)); return loadValue(session, key, cacheOnly).map(PlanNodeCanonicalInfo::getInputTableStatistics); } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java index 84578460246b4..348962aef6e74 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java @@ -84,7 +84,8 @@ import static com.facebook.presto.SystemSessionProperties.usePerfectlyConsistentHistories; import static com.facebook.presto.common.function.OperatorType.EQUAL; import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.DEFAULT; -import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.REMOVE_SAFE_CONSTANTS; +import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.IGNORE_SAFE_CONSTANTS; +import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.IGNORE_SCAN_CONSTANTS; import static com.facebook.presto.common.type.IntegerType.INTEGER; import static com.facebook.presto.expressions.CanonicalRowExpressionRewriter.canonicalizeRowExpression; import static com.facebook.presto.expressions.LogicalRowExpressions.extractConjuncts; @@ -737,7 +738,7 @@ public Optional visitOutput(OutputNode node, Context context) } List rowExpressionReferences = node.getOutputVariables().stream() - .map(variable -> new RowExpressionReference(inlineAndCanonicalize(context.getExpressions(), variable, strategy == REMOVE_SAFE_CONSTANTS), variable)) + .map(variable -> new RowExpressionReference(inlineAndCanonicalize(context.getExpressions(), variable, strategy == IGNORE_SAFE_CONSTANTS), variable)) .sorted(comparing(rowExpressionReference -> writeValueAsString(rowExpressionReference.getRowExpression()))) .collect(toImmutableList()); @@ -945,7 +946,7 @@ public Optional visitProject(ProjectNode node, Context context) } List rowExpressionReferences = node.getAssignments().entrySet().stream() - .map(entry -> new RowExpressionReference(inlineAndCanonicalize(context.getExpressions(), entry.getValue(), strategy == REMOVE_SAFE_CONSTANTS), entry.getKey())) + .map(entry -> new RowExpressionReference(inlineAndCanonicalize(context.getExpressions(), entry.getValue(), strategy == IGNORE_SAFE_CONSTANTS || strategy == IGNORE_SCAN_CONSTANTS), entry.getKey())) .sorted(comparing(rowExpressionReference -> writeValueAsString(rowExpressionReference.getRowExpression()))) .collect(toImmutableList()); ImmutableMap.Builder assignments = ImmutableMap.builder(); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanNodeCanonicalInfo.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanNodeCanonicalInfo.java index 8401b96b5d876..840118d2c6578 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanNodeCanonicalInfo.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanNodeCanonicalInfo.java @@ -25,7 +25,7 @@ import java.util.Objects; import java.util.Optional; -import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.historyBasedPlanCanonicalizationStrategyList; +import static com.facebook.presto.cost.HistoryBasedPlanStatisticsManager.historyBasedPlanCanonicalizationStrategyList; import static com.google.common.graph.Traverser.forTree; import static java.util.Objects.requireNonNull; @@ -78,7 +78,7 @@ public static List getCanonicalInfo( PlanCanonicalInfoProvider planCanonicalInfoProvider) { ImmutableList.Builder result = ImmutableList.builder(); - for (PlanCanonicalizationStrategy strategy : historyBasedPlanCanonicalizationStrategyList()) { + for (PlanCanonicalizationStrategy strategy : historyBasedPlanCanonicalizationStrategyList(session)) { for (PlanNode node : forTree(PlanNode::getSources).depthFirstPreOrder(root)) { if (!node.getStatsEquivalentPlanNode().isPresent()) { continue; diff --git a/presto-main/src/test/java/com/facebook/presto/cost/TestHistoryBasedOptimizationConfig.java b/presto-main/src/test/java/com/facebook/presto/cost/TestHistoryBasedOptimizationConfig.java index 11915d9b154ae..f8f8775a48e18 100644 --- a/presto-main/src/test/java/com/facebook/presto/cost/TestHistoryBasedOptimizationConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/cost/TestHistoryBasedOptimizationConfig.java @@ -13,14 +13,21 @@ */ package com.facebook.presto.cost; +import com.facebook.presto.Session; +import com.facebook.presto.common.plan.PlanCanonicalizationStrategy; import com.google.common.collect.ImmutableMap; import org.testng.annotations.Test; +import java.util.List; import java.util.Map; import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertFullMapping; import static com.facebook.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; import static com.facebook.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static com.facebook.presto.SystemSessionProperties.HISTORY_BASED_OPTIMIZATION_PLAN_CANONICALIZATION_STRATEGY; +import static com.facebook.presto.cost.HistoryBasedPlanStatisticsManager.historyBasedPlanCanonicalizationStrategyList; +import static com.facebook.presto.testing.TestingSession.testSessionBuilder; +import static org.testng.Assert.assertEquals; public class TestHistoryBasedOptimizationConfig { @@ -46,4 +53,34 @@ public void testExplicitPropertyMappings() assertFullMapping(properties, expected); } + + @Test + public void testPlanCanonicalizationStrategyOrder() + { + Session session = testSessionBuilder() + .setSystemProperty(HISTORY_BASED_OPTIMIZATION_PLAN_CANONICALIZATION_STRATEGY, "IGNORE_SAFE_CONSTANTS,DEFAULT,CONNECTOR,IGNORE_SCAN_CONSTANTS") + .build(); + List strategyList = historyBasedPlanCanonicalizationStrategyList(session); + assertEquals(strategyList.size(), 4); + assertEquals(strategyList.get(0), PlanCanonicalizationStrategy.DEFAULT); + assertEquals(strategyList.get(1), PlanCanonicalizationStrategy.CONNECTOR); + assertEquals(strategyList.get(2), PlanCanonicalizationStrategy.IGNORE_SAFE_CONSTANTS); + assertEquals(strategyList.get(3), PlanCanonicalizationStrategy.IGNORE_SCAN_CONSTANTS); + + session = testSessionBuilder() + .setSystemProperty(HISTORY_BASED_OPTIMIZATION_PLAN_CANONICALIZATION_STRATEGY, "IGNORE_SAFE_CONSTANTS,IGNORE_SCAN_CONSTANTS") + .build(); + strategyList = historyBasedPlanCanonicalizationStrategyList(session); + assertEquals(strategyList.size(), 2); + assertEquals(strategyList.get(0), PlanCanonicalizationStrategy.IGNORE_SAFE_CONSTANTS); + assertEquals(strategyList.get(1), PlanCanonicalizationStrategy.IGNORE_SCAN_CONSTANTS); + + session = testSessionBuilder() + .setSystemProperty(HISTORY_BASED_OPTIMIZATION_PLAN_CANONICALIZATION_STRATEGY, "IGNORE_SCAN_CONSTANTS,IGNORE_SAFE_CONSTANTS") + .build(); + strategyList = historyBasedPlanCanonicalizationStrategyList(session); + assertEquals(strategyList.size(), 2); + assertEquals(strategyList.get(0), PlanCanonicalizationStrategy.IGNORE_SAFE_CONSTANTS); + assertEquals(strategyList.get(1), PlanCanonicalizationStrategy.IGNORE_SCAN_CONSTANTS); + } } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index 487cc5f8591c2..2fd2ff751c538 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -86,6 +86,7 @@ public void testDefaults() .setUsePerfectlyConsistentHistories(false) .setHistoryCanonicalPlanNodeLimit(1000) .setHistoryBasedOptimizerTimeout(new Duration(10, SECONDS)) + .setHistoryBasedOptimizerPlanCanonicalizationStrategies("IGNORE_SAFE_CONSTANTS") .setRedistributeWrites(true) .setScaleWriters(false) .setWriterMinSize(new DataSize(32, MEGABYTE)) @@ -313,6 +314,7 @@ public void testExplicitPropertyMappings() .put("optimizer.track-partial-aggregation-history", "false") .put("optimizer.use-perfectly-consistent-histories", "true") .put("optimizer.history-canonical-plan-node-limit", "2") + .put("optimizer.history-based-optimizer-plan-canonicalization-strategies", "IGNORE_SAFE_CONSTANTS,IGNORE_SCAN_CONSTANTS") .put("optimizer.history-based-optimizer-timeout", "1s") .put("redistribute-writes", "false") .put("scale-writers", "true") @@ -509,6 +511,7 @@ public void testExplicitPropertyMappings() .setUsePerfectlyConsistentHistories(true) .setHistoryCanonicalPlanNodeLimit(2) .setHistoryBasedOptimizerTimeout(new Duration(1, SECONDS)) + .setHistoryBasedOptimizerPlanCanonicalizationStrategies("IGNORE_SAFE_CONSTANTS,IGNORE_SCAN_CONSTANTS") .setRedistributeWrites(false) .setScaleWriters(true) .setWriterMinSize(new DataSize(42, GIGABYTE)) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCachingPlanCanonicalInfoProvider.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCachingPlanCanonicalInfoProvider.java index 200c964eb854d..0859fcaa5dc55 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCachingPlanCanonicalInfoProvider.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCachingPlanCanonicalInfoProvider.java @@ -29,7 +29,7 @@ import static com.facebook.presto.SystemSessionProperties.RESTRICT_HISTORY_BASED_OPTIMIZATION_TO_COMPLEX_QUERY; import static com.facebook.presto.SystemSessionProperties.USE_HISTORY_BASED_PLAN_STATISTICS; -import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.historyBasedPlanCanonicalizationStrategyList; +import static com.facebook.presto.cost.HistoryBasedPlanStatisticsManager.historyBasedPlanCanonicalizationStrategyList; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.google.common.graph.Traverser.forTree; import static org.testng.Assert.assertEquals; @@ -67,17 +67,17 @@ public void testCache() assertTrue(root.getStatsEquivalentPlanNode().isPresent()); CachingPlanCanonicalInfoProvider planCanonicalInfoProvider = (CachingPlanCanonicalInfoProvider) ((HistoryBasedPlanStatisticsCalculator) getQueryRunner().getStatsCalculator()).getPlanCanonicalInfoProvider(); - assertEquals(planCanonicalInfoProvider.getCacheSize(), 5L * historyBasedPlanCanonicalizationStrategyList().size()); + assertEquals(planCanonicalInfoProvider.getCacheSize(), 5L * historyBasedPlanCanonicalizationStrategyList(session).size()); forTree(PlanNode::getSources).depthFirstPreOrder(root).forEach(child -> { if (!child.getStatsEquivalentPlanNode().isPresent()) { return; } - for (PlanCanonicalizationStrategy strategy : historyBasedPlanCanonicalizationStrategyList()) { + for (PlanCanonicalizationStrategy strategy : historyBasedPlanCanonicalizationStrategyList(session)) { planCanonicalInfoProvider.hash(session, child.getStatsEquivalentPlanNode().get(), strategy, false).get(); } }); // Assert that size of cache remains same, meaning all needed hashes were already cached. - assertEquals(planCanonicalInfoProvider.getCacheSize(), 5L * historyBasedPlanCanonicalizationStrategyList().size()); + assertEquals(planCanonicalInfoProvider.getCacheSize(), 5L * historyBasedPlanCanonicalizationStrategyList(session).size()); planCanonicalInfoProvider.getHistoryBasedStatisticsCacheManager().invalidate(session.getQueryId()); assertEquals(planCanonicalInfoProvider.getCacheSize(), 0); @@ -85,7 +85,7 @@ public void testCache() if (!child.getStatsEquivalentPlanNode().isPresent()) { return; } - for (PlanCanonicalizationStrategy strategy : historyBasedPlanCanonicalizationStrategyList()) { + for (PlanCanonicalizationStrategy strategy : historyBasedPlanCanonicalizationStrategyList(session)) { // Only read from cache, hence will return Optional.empty() as the cache is already invalidated assertFalse(planCanonicalInfoProvider.hash(session, child.getStatsEquivalentPlanNode().get(), strategy, true).isPresent()); } diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanGenerator.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanGenerator.java index 7dd3c99186ec6..c48a24e56f8dc 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanGenerator.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanGenerator.java @@ -38,7 +38,8 @@ import java.util.stream.Collectors; import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.CONNECTOR; -import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.REMOVE_SAFE_CONSTANTS; +import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.IGNORE_SAFE_CONSTANTS; +import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.IGNORE_SCAN_CONSTANTS; import static com.facebook.presto.sql.Optimizer.PlanStage.OPTIMIZED_AND_VALIDATED; import static com.facebook.presto.sql.planner.CanonicalPlanGenerator.generateCanonicalPlan; import static com.facebook.presto.sql.planner.CanonicalPlanGenerator.generateCanonicalPlanFragment; @@ -175,14 +176,17 @@ public void testTableScan() assertDifferentCanonicalLeafSubPlan("SELECT totalprice FROM orders", "SELECT orderkey, totalprice FROM orders"); assertDifferentCanonicalLeafSubPlan("SELECT * FROM orders", "SELECT orderkey, totalprice FROM orders"); } + @Test public void testTableScanAndProjectWithStrategy() throws Exception { assertSameCanonicalLeafPlan("SELECT 1 from orders", "SELECT 1 from orders", CONNECTOR); assertDifferentCanonicalLeafPlan("SELECT 1 from orders", "SELECT 2 from orders", CONNECTOR); - assertSameCanonicalLeafPlan("SELECT 1 from orders", "SELECT 2 from orders", REMOVE_SAFE_CONSTANTS); - assertSameCanonicalLeafPlan("SELECT CAST(1 AS VARCHAR) from orders", "SELECT CAST(2 AS VARCHAR) from orders", REMOVE_SAFE_CONSTANTS); + assertSameCanonicalLeafPlan("SELECT 1 from orders", "SELECT 2 from orders", IGNORE_SAFE_CONSTANTS); + assertSameCanonicalLeafPlan("SELECT CAST(1 AS VARCHAR) from orders", "SELECT CAST(2 AS VARCHAR) from orders", IGNORE_SAFE_CONSTANTS); + assertSameCanonicalLeafPlan("SELECT 1 from orders", "SELECT 2 from orders", IGNORE_SCAN_CONSTANTS); + assertSameCanonicalLeafPlan("SELECT CAST(1 AS VARCHAR) from orders", "SELECT CAST(2 AS VARCHAR) from orders", IGNORE_SCAN_CONSTANTS); assertSameCanonicalLeafPlan( "SELECT totalprice, custkey + (totalprice / 10) from orders", @@ -195,7 +199,11 @@ public void testTableScanAndProjectWithStrategy() assertSameCanonicalLeafPlan( "SELECT totalprice, custkey + (totalprice / 10) from orders", "SELECT custkey + (totalprice / 5), totalprice from orders", - REMOVE_SAFE_CONSTANTS); + IGNORE_SAFE_CONSTANTS); + assertSameCanonicalLeafPlan( + "SELECT totalprice, custkey + (totalprice / 10) from orders", + "SELECT custkey + (totalprice / 5), totalprice from orders", + IGNORE_SCAN_CONSTANTS); } @Test @@ -213,7 +221,11 @@ public void testFilterWithStrategy() assertDifferentCanonicalLeafPlan( "SELECT totalprice from orders WHERE custkey > 100 AND custkey < 120", "SELECT totalprice from orders WHERE custkey > 100 AND custkey < 110", - REMOVE_SAFE_CONSTANTS); + IGNORE_SAFE_CONSTANTS); + assertDifferentCanonicalLeafPlan( + "SELECT totalprice from orders WHERE custkey > 100 AND custkey < 120", + "SELECT totalprice from orders WHERE custkey > 100 AND custkey < 110", + IGNORE_SCAN_CONSTANTS); assertSameCanonicalLeafPlan( "SELECT totalprice from orders WHERE custkey IN (10,20,30)", @@ -222,11 +234,19 @@ public void testFilterWithStrategy() assertDifferentCanonicalLeafPlan( "SELECT totalprice from orders WHERE custkey IN (10,20,30)", "SELECT totalprice from orders WHERE custkey IN (10,30,40)", - REMOVE_SAFE_CONSTANTS); + IGNORE_SAFE_CONSTANTS); + assertSameCanonicalLeafPlan( + "SELECT totalprice, CAST(3 AS VARCHAR) from orders WHERE custkey > 100 AND custkey < 120", + "SELECT totalprice, CAST(2 AS VARCHAR) as x from orders WHERE custkey > 100 AND custkey < 120", + IGNORE_SAFE_CONSTANTS); + assertDifferentCanonicalLeafPlan( + "SELECT totalprice from orders WHERE custkey IN (10,20,30)", + "SELECT totalprice from orders WHERE custkey IN (10,30,40)", + IGNORE_SCAN_CONSTANTS); assertSameCanonicalLeafPlan( "SELECT totalprice, CAST(3 AS VARCHAR) from orders WHERE custkey > 100 AND custkey < 120", "SELECT totalprice, CAST(2 AS VARCHAR) as x from orders WHERE custkey > 100 AND custkey < 120", - REMOVE_SAFE_CONSTANTS); + IGNORE_SCAN_CONSTANTS); } private static List getLeafSubPlans(SubPlan subPlan) diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanHashes.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanHashes.java index bbfc8692363ad..7679740dd408c 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanHashes.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestCanonicalPlanHashes.java @@ -36,7 +36,8 @@ import static com.facebook.presto.SystemSessionProperties.USE_HISTORY_BASED_PLAN_STATISTICS; import static com.facebook.presto.SystemSessionProperties.USE_PERFECTLY_CONSISTENT_HISTORIES; import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.CONNECTOR; -import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.REMOVE_SAFE_CONSTANTS; +import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.IGNORE_SAFE_CONSTANTS; +import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.IGNORE_SCAN_CONSTANTS; import static com.facebook.presto.sql.planner.CanonicalPlanGenerator.generateCanonicalPlan; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static com.google.common.graph.Traverser.forTree; @@ -79,7 +80,11 @@ public void testScanFilterProject() assertSamePlanHash( "SELECT totalprice, orderkey / 2 from orders WHERE custkey > 100 AND custkey < 120", "SELECT totalprice, orderkey / 4 from orders WHERE custkey > 100 AND custkey < 120", - REMOVE_SAFE_CONSTANTS); + IGNORE_SAFE_CONSTANTS); + assertSamePlanHash( + "SELECT totalprice, orderkey / 2 from orders WHERE custkey > 100 AND custkey < 120", + "SELECT totalprice, orderkey / 4 from orders WHERE custkey > 100 AND custkey < 120", + IGNORE_SCAN_CONSTANTS); assertDifferentPlanHash( "SELECT totalprice from orders WHERE custkey > 100 AND custkey < 110", @@ -88,11 +93,19 @@ public void testScanFilterProject() assertDifferentPlanHash( "SELECT totalprice from orders WHERE custkey > 100 AND custkey < 110", "SELECT totalprice from orders WHERE custkey > 100 AND custkey < 120", - REMOVE_SAFE_CONSTANTS); + IGNORE_SAFE_CONSTANTS); + assertSamePlanHash( + "SELECT cast(totalprice as varchar), orderkey / 2.0 from orders WHERE custkey > 100 AND custkey < 120", + "SELECT cast(totalprice as varchar), orderkey / 4.0 from orders WHERE custkey > 100 AND custkey < 120", + IGNORE_SAFE_CONSTANTS); + assertDifferentPlanHash( + "SELECT totalprice from orders WHERE custkey > 100 AND custkey < 110", + "SELECT totalprice from orders WHERE custkey > 100 AND custkey < 120", + IGNORE_SCAN_CONSTANTS); assertSamePlanHash( "SELECT cast(totalprice as varchar), orderkey / 2.0 from orders WHERE custkey > 100 AND custkey < 120", "SELECT cast(totalprice as varchar), orderkey / 4.0 from orders WHERE custkey > 100 AND custkey < 120", - REMOVE_SAFE_CONSTANTS); + IGNORE_SCAN_CONSTANTS); } @Test @@ -110,11 +123,19 @@ public void testGroupBy() assertDifferentPlanHash( "SELECT COUNT_IF(totalprice > 0) from orders WHERE custkey > 100 AND custkey < 200 GROUP BY orderkey, orderstatus", "SELECT COUNT_IF(totalprice > 5) from orders WHERE custkey > 100 AND custkey < 250 GROUP BY orderkey, orderstatus", - REMOVE_SAFE_CONSTANTS); + IGNORE_SAFE_CONSTANTS); + assertSamePlanHash( + "SELECT COUNT_IF(totalprice > 0), 1 from (select *, shippriority/2 as pri from orders) WHERE custkey > 100 AND custkey < 200 GROUP BY GROUPING SETS ((pri), (shippriority, custkey))", + "SELECT COUNT_IF(totalprice > 0), 2 from (select *, shippriority/4 as pri from orders) WHERE custkey > 100 AND custkey < 200 GROUP BY GROUPING SETS ((pri), (shippriority, custkey))", + IGNORE_SAFE_CONSTANTS); + assertDifferentPlanHash( + "SELECT COUNT_IF(totalprice > 0) from orders WHERE custkey > 100 AND custkey < 200 GROUP BY orderkey, orderstatus", + "SELECT COUNT_IF(totalprice > 5) from orders WHERE custkey > 100 AND custkey < 250 GROUP BY orderkey, orderstatus", + IGNORE_SCAN_CONSTANTS); assertSamePlanHash( "SELECT COUNT_IF(totalprice > 0), 1 from (select *, shippriority/2 as pri from orders) WHERE custkey > 100 AND custkey < 200 GROUP BY GROUPING SETS ((pri), (shippriority, custkey))", "SELECT COUNT_IF(totalprice > 0), 2 from (select *, shippriority/4 as pri from orders) WHERE custkey > 100 AND custkey < 200 GROUP BY GROUPING SETS ((pri), (shippriority, custkey))", - REMOVE_SAFE_CONSTANTS); + IGNORE_SCAN_CONSTANTS); assertDifferentPlanHash( "SELECT COUNT(totalprice) from orders WHERE custkey > 100 AND custkey < 200 GROUP BY orderkey", @@ -132,15 +153,28 @@ public void testGroupBy() assertDifferentPlanHash( "SELECT COUNT(totalprice) from orders WHERE custkey > 100 AND custkey < 200 GROUP BY orderkey", "SELECT SUM(totalprice) from orders WHERE custkey > 100 AND custkey < 200 GROUP BY orderkey", - REMOVE_SAFE_CONSTANTS); + IGNORE_SAFE_CONSTANTS); + assertDifferentPlanHash( + "SELECT COUNT(totalprice) from orders WHERE custkey > 100 AND custkey < 200 GROUP BY orderkey", + "SELECT COUNT(DISTINCT totalprice) from orders WHERE custkey > 100 AND custkey < 200 GROUP BY orderkey", + IGNORE_SAFE_CONSTANTS); + assertDifferentPlanHash( + "SELECT COUNT_IF(totalprice > 0) from (select *, shippriority/2 as pri from orders) WHERE custkey > 100 AND custkey < 200 GROUP BY GROUPING SETS ((pri), (shippriority, custkey))", + "SELECT COUNT_IF(totalprice > 0) from (select *, shippriority/2 as pri from orders) WHERE custkey > 100 AND custkey < 250 GROUP BY GROUPING SETS ((pri), (custkey))", + IGNORE_SAFE_CONSTANTS); + + assertDifferentPlanHash( + "SELECT COUNT(totalprice) from orders WHERE custkey > 100 AND custkey < 200 GROUP BY orderkey", + "SELECT SUM(totalprice) from orders WHERE custkey > 100 AND custkey < 200 GROUP BY orderkey", + IGNORE_SCAN_CONSTANTS); assertDifferentPlanHash( "SELECT COUNT(totalprice) from orders WHERE custkey > 100 AND custkey < 200 GROUP BY orderkey", "SELECT COUNT(DISTINCT totalprice) from orders WHERE custkey > 100 AND custkey < 200 GROUP BY orderkey", - REMOVE_SAFE_CONSTANTS); + IGNORE_SCAN_CONSTANTS); assertDifferentPlanHash( "SELECT COUNT_IF(totalprice > 0) from (select *, shippriority/2 as pri from orders) WHERE custkey > 100 AND custkey < 200 GROUP BY GROUPING SETS ((pri), (shippriority, custkey))", "SELECT COUNT_IF(totalprice > 0) from (select *, shippriority/2 as pri from orders) WHERE custkey > 100 AND custkey < 250 GROUP BY GROUPING SETS ((pri), (custkey))", - REMOVE_SAFE_CONSTANTS); + IGNORE_SCAN_CONSTANTS); } @Test @@ -160,7 +194,12 @@ public void testUnnest() assertSamePlanHash( "SELECT a.custkey, t.e FROM (SELECT custkey, ARRAY[1, 2, 3, 4] AS my_array FROM orders) a CROSS JOIN UNNEST(my_array) AS t(e)", "SELECT a.custkey, t.e FROM (SELECT custkey, ARRAY[1, 2, 3] AS my_array FROM orders) a CROSS JOIN UNNEST(my_array) AS t(e)", - REMOVE_SAFE_CONSTANTS); + IGNORE_SAFE_CONSTANTS); + + assertSamePlanHash( + "SELECT a.custkey, t.e FROM (SELECT custkey, ARRAY[1, 2, 3, 4] AS my_array FROM orders) a CROSS JOIN UNNEST(my_array) AS t(e)", + "SELECT a.custkey, t.e FROM (SELECT custkey, ARRAY[1, 2, 3] AS my_array FROM orders) a CROSS JOIN UNNEST(my_array) AS t(e)", + IGNORE_SCAN_CONSTANTS); } @Test @@ -190,7 +229,11 @@ public void testUnion() assertSamePlanHash( "SELECT orderkey, custkey, 1 as x FROM orders where orderkey < 1000 UNION ALL SELECT orderkey, custkey, 1 as x FROM orders where orderkey >= 1000 and orderkey < 2000 UNION ALL SELECT orderkey, custkey, 1 as x FROM orders where orderkey >= 2000 and orderkey < 3000", "SELECT orderkey, custkey, 2 as x FROM orders where orderkey >= 2000 and orderkey < 3000 UNION ALL SELECT orderkey, custkey, 2 as x FROM orders where orderkey < 1000 UNION ALL SELECT orderkey, custkey, 2 as x FROM orders where orderkey >= 1000 and orderkey < 2000", - REMOVE_SAFE_CONSTANTS); + IGNORE_SAFE_CONSTANTS); + assertSamePlanHash( + "SELECT orderkey, custkey, 1 as x FROM orders where orderkey < 1000 UNION ALL SELECT orderkey, custkey, 1 as x FROM orders where orderkey >= 1000 and orderkey < 2000 UNION ALL SELECT orderkey, custkey, 1 as x FROM orders where orderkey >= 2000 and orderkey < 3000", + "SELECT orderkey, custkey, 2 as x FROM orders where orderkey >= 2000 and orderkey < 3000 UNION ALL SELECT orderkey, custkey, 2 as x FROM orders where orderkey < 1000 UNION ALL SELECT orderkey, custkey, 2 as x FROM orders where orderkey >= 1000 and orderkey < 2000", + IGNORE_SCAN_CONSTANTS); } @Test @@ -315,7 +358,8 @@ public void testLimit() assertSamePlanHash("SELECT * from nation LIMIT 1000", "SELECT * from nation LIMIT 1000", CONNECTOR); assertDifferentPlanHash("SELECT * from nation LIMIT 1000", "SELECT * from nation", CONNECTOR); assertDifferentPlanHash("SELECT * from nation LIMIT 1000", "SELECT * from nation LIMIT 10000", CONNECTOR); - assertDifferentPlanHash("SELECT * from nation LIMIT 1000", "SELECT * from nation LIMIT 10000", REMOVE_SAFE_CONSTANTS); + assertDifferentPlanHash("SELECT * from nation LIMIT 1000", "SELECT * from nation LIMIT 10000", IGNORE_SAFE_CONSTANTS); + assertDifferentPlanHash("SELECT * from nation LIMIT 1000", "SELECT * from nation LIMIT 10000", IGNORE_SCAN_CONSTANTS); } @Test diff --git a/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/TestPrestoSparkStatsCalculator.java b/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/TestPrestoSparkStatsCalculator.java index 505e5a3f9fb8d..7134c003ef522 100644 --- a/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/TestPrestoSparkStatsCalculator.java +++ b/presto-spark-base/src/test/java/com/facebook/presto/spark/planner/TestPrestoSparkStatsCalculator.java @@ -51,7 +51,7 @@ import java.util.Optional; import static com.facebook.presto.SystemSessionProperties.USE_HISTORY_BASED_PLAN_STATISTICS; -import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.REMOVE_SAFE_CONSTANTS; +import static com.facebook.presto.common.plan.PlanCanonicalizationStrategy.IGNORE_SAFE_CONSTANTS; import static com.facebook.presto.testing.TestingSession.testSessionBuilder; import static java.lang.Double.NaN; @@ -118,7 +118,7 @@ public void testUsesHboStatsWhenMatchRuntime() .registerVariable(planBuilder.variable("c1")) .filter(planBuilder.rowExpression("c1 IS NOT NULL"), planBuilder.values(planBuilder.variable("c1"))); - Optional hash = historyBasedPlanStatisticsCalculator.getPlanCanonicalInfoProvider().hash(session, statsEquivalentRemoteSource, REMOVE_SAFE_CONSTANTS, false); + Optional hash = historyBasedPlanStatisticsCalculator.getPlanCanonicalInfoProvider().hash(session, statsEquivalentRemoteSource, IGNORE_SAFE_CONSTANTS, false); InMemoryHistoryBasedPlanStatisticsProvider historyBasedPlanStatisticsProvider = (InMemoryHistoryBasedPlanStatisticsProvider) historyBasedPlanStatisticsCalculator.getHistoryBasedPlanStatisticsProvider().get(); historyBasedPlanStatisticsProvider.putStats(ImmutableMap.of( diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanNodeWithHash.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanNodeWithHash.java index 980575a48e79c..96cda044d7471 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanNodeWithHash.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanNodeWithHash.java @@ -22,7 +22,7 @@ public class PlanNodeWithHash { private final PlanNode planNode; - // An optional canonical hash of the corresponding plan node + // An optional canonical hash of the corresponding plan node. Hash strategy is part of `CanonicalPlan` which gets hashed, hence different strategies gives different hash private final Optional hash; public PlanNodeWithHash(PlanNode planNode, Optional hash)