From 0ed73ea03b014f8d4a2d804c547f08ebacea858c Mon Sep 17 00:00:00 2001 From: Feilong Liu Date: Tue, 19 Nov 2024 14:06:06 -0800 Subject: [PATCH] Move Delete TableWriter TableFinish node to SPI --- .../HistoryBasedPlanStatisticsTracker.java | 2 +- .../execution/scheduler/TableWriteInfo.java | 9 +- .../metadata/DelegatingMetadataManager.java | 1 + .../facebook/presto/metadata/Metadata.java | 1 + .../presto/metadata/MetadataManager.java | 1 + .../operator/StatisticsWriterOperator.java | 2 +- .../presto/operator/TableFinishOperator.java | 14 +- .../presto/sql/TemporaryTableUtil.java | 72 +++++++- .../sql/planner/BasePlanFragmenter.java | 4 +- .../sql/planner/CanonicalPlanGenerator.java | 4 +- .../sql/planner/GroupedExecutionTagger.java | 2 +- .../sql/planner/LocalExecutionPlanner.java | 8 +- .../presto/sql/planner/LogicalPlanner.java | 46 ++--- .../presto/sql/planner/OutputExtractor.java | 2 +- .../sql/planner/PlanFragmenterUtils.java | 4 +- .../presto/sql/planner/QueryPlanner.java | 2 +- .../sql/planner/SplitSourceFactory.java | 6 +- .../planner/StatisticsAggregationPlanner.java | 6 +- .../rule/PushTableWriteThroughUnion.java | 2 +- .../iterative/rule/RemoveEmptyDelete.java | 2 +- .../rule/RowExpressionRewriteRuleSet.java | 6 +- .../iterative/rule/ScaledWriterRule.java | 2 +- .../planner/optimizations/AddExchanges.java | 4 +- .../optimizations/AddLocalExchanges.java | 10 +- .../MetadataDeleteOptimizer.java | 4 +- .../optimizations/PropertyDerivations.java | 6 +- .../PruneUnreferencedOutputs.java | 8 +- .../optimizations/PushdownSubfields.java | 2 +- .../ReplicateSemiJoinInDelete.java | 2 +- .../StreamPropertyDerivations.java | 6 +- .../planner/optimizations/SymbolMapper.java | 8 +- .../UnaliasSymbolReferences.java | 6 +- .../sql/planner/plan/InternalPlanVisitor.java | 15 -- .../presto/sql/planner/plan/Patterns.java | 3 + .../planner/plan/StatisticAggregations.java | 170 ------------------ .../planner/plan/StatisticsWriterNode.java | 1 + .../planner/plan/TableWriterMergeNode.java | 1 + .../planner/planPrinter/IOPlanPrinter.java | 6 +- .../sql/planner/planPrinter/PlanPrinter.java | 8 +- .../sanity/ValidateDependenciesChecker.java | 10 +- .../facebook/presto/util/GraphvizPrinter.java | 6 +- .../presto/metadata/AbstractMockMetadata.java | 1 + .../operator/TestTableFinishOperator.java | 5 +- .../sql/planner/TestingWriterTarget.java | 2 +- .../planner/assertions/PlanMatchPattern.java | 2 +- .../assertions/TableWriterMatcher.java | 2 +- .../iterative/rule/test/PlanBuilder.java | 6 +- .../plan/TestStatisticsWriterNode.java | 1 + .../core/presto_protocol_core.cpp | 4 +- .../core/presto_protocol_core.yml | 6 +- .../presto_protocol/presto_protocol.yml | 6 +- .../PrestoSparkNativeTaskExecutorFactory.java | 2 +- .../facebook/presto/spi}/NewTableLayout.java | 12 +- .../facebook/presto/spi}/plan/DeleteNode.java | 20 +-- .../facebook/presto/spi/plan/PlanVisitor.java | 15 ++ .../spi/plan/StatisticAggregations.java | 111 ++++++++++++ .../plan/StatisticAggregationsDescriptor.java | 56 +++--- .../presto/spi}/plan/TableFinishNode.java | 29 ++- .../presto/spi}/plan/TableWriterNode.java | 47 +++-- 59 files changed, 400 insertions(+), 401 deletions(-) delete mode 100644 presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticAggregations.java rename {presto-main/src/main/java/com/facebook/presto/metadata => presto-spi/src/main/java/com/facebook/presto/spi}/NewTableLayout.java (86%) rename {presto-main/src/main/java/com/facebook/presto/sql/planner => presto-spi/src/main/java/com/facebook/presto/spi}/plan/DeleteNode.java (83%) create mode 100644 presto-spi/src/main/java/com/facebook/presto/spi/plan/StatisticAggregations.java rename {presto-main/src/main/java/com/facebook/presto/sql/planner => presto-spi/src/main/java/com/facebook/presto/spi}/plan/StatisticAggregationsDescriptor.java (72%) rename {presto-main/src/main/java/com/facebook/presto/sql/planner => presto-spi/src/main/java/com/facebook/presto/spi}/plan/TableFinishNode.java (85%) rename {presto-main/src/main/java/com/facebook/presto/sql/planner => presto-spi/src/main/java/com/facebook/presto/spi}/plan/TableWriterNode.java (91%) diff --git a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java index 20aeb989ffaf7..2c02e64c85cde 100644 --- a/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java +++ b/presto-main/src/main/java/com/facebook/presto/cost/HistoryBasedPlanStatisticsTracker.java @@ -29,6 +29,7 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeWithHash; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.statistics.Estimate; import com.facebook.presto.spi.statistics.HistoricalPlanStatistics; @@ -42,7 +43,6 @@ import com.facebook.presto.spi.statistics.TableWriterNodeStatistics; import com.facebook.presto.sql.planner.CanonicalPlan; import com.facebook.presto.sql.planner.PlanNodeCanonicalInfo; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.planPrinter.PlanNodeStats; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; diff --git a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TableWriteInfo.java b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TableWriteInfo.java index 34f168ec5e32b..d769550178449 100644 --- a/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TableWriteInfo.java +++ b/presto-main/src/main/java/com/facebook/presto/execution/scheduler/TableWriteInfo.java @@ -22,19 +22,18 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.Constraint; import com.facebook.presto.spi.TableHandle; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.JoinNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.SemiJoinNode; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.sql.planner.optimizations.PlanNodeSearcher; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode.WriterTarget; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.VerifyException; @@ -90,7 +89,7 @@ public static TableWriteInfo createTableWriteInfo(PlanNode planNode, Metadata me private static Optional createWriterTarget(Optional finishNodeOptional, Metadata metadata, Session session) { if (finishNodeOptional.isPresent()) { - WriterTarget target = finishNodeOptional.get().getTarget().orElseThrow(() -> new VerifyException("target is absent")); + TableWriterNode.WriterTarget target = finishNodeOptional.get().getTarget().orElseThrow(() -> new VerifyException("target is absent")); if (target instanceof TableWriterNode.CreateName) { TableWriterNode.CreateName create = (TableWriterNode.CreateName) target; return Optional.of(new ExecutionWriterTarget.CreateHandle(metadata.beginCreateTable(session, create.getConnectorId().getCatalogName(), create.getTableMetadata(), create.getLayout()), create.getSchemaTableName())); diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/DelegatingMetadataManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/DelegatingMetadataManager.java index cfc65f6bd61f8..eb9e184625fba 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/DelegatingMetadataManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/DelegatingMetadataManager.java @@ -27,6 +27,7 @@ import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.Constraint; import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.NewTableLayout; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.SystemTable; import com.facebook.presto.spi.TableHandle; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java b/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java index 01679c1900983..55d5958ffd2a1 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/Metadata.java @@ -28,6 +28,7 @@ import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.Constraint; import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.NewTableLayout; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.SystemTable; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java b/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java index c9b86244de8ce..26c0ce32e4338 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java +++ b/presto-main/src/main/java/com/facebook/presto/metadata/MetadataManager.java @@ -43,6 +43,7 @@ import com.facebook.presto.spi.Constraint; import com.facebook.presto.spi.MaterializedViewDefinition; import com.facebook.presto.spi.MaterializedViewStatus; +import com.facebook.presto.spi.NewTableLayout; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.SchemaTableName; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/StatisticsWriterOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/StatisticsWriterOperator.java index 509f4a7628251..5f75fd4aec2c8 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/StatisticsWriterOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/StatisticsWriterOperator.java @@ -19,8 +19,8 @@ import com.facebook.presto.common.block.BlockBuilder; import com.facebook.presto.common.type.Type; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.StatisticAggregationsDescriptor; import com.facebook.presto.spi.statistics.ComputedStatistics; -import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; import com.google.common.collect.ImmutableList; import java.util.Collection; diff --git a/presto-main/src/main/java/com/facebook/presto/operator/TableFinishOperator.java b/presto-main/src/main/java/com/facebook/presto/operator/TableFinishOperator.java index bcdab3dde9fe3..80d989c9bdeb4 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/TableFinishOperator.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/TableFinishOperator.java @@ -26,8 +26,8 @@ import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.connector.ConnectorOutputMetadata; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.StatisticAggregationsDescriptor; import com.facebook.presto.spi.statistics.ComputedStatistics; -import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableList; import com.google.common.collect.Streams; @@ -469,9 +469,9 @@ public long getFinalRowCount() { checkState(uncommittedRecoverableLifespanAndStageStates.isEmpty(), "All recoverable LifespanAndStage should be committed when fetching final row count"); return Streams.concat( - noCommitUnrecoverableLifespanAndStageStates.values().stream(), - taskCommitUnrecoverableLifespanAndStageStates.values().stream(), - committedRecoverableLifespanAndStages.values().stream()) + noCommitUnrecoverableLifespanAndStageStates.values().stream(), + taskCommitUnrecoverableLifespanAndStageStates.values().stream(), + committedRecoverableLifespanAndStages.values().stream()) .mapToLong(LifespanAndStageState::getRowCount) .sum(); } @@ -480,9 +480,9 @@ public List getFinalFragments() { checkState(uncommittedRecoverableLifespanAndStageStates.isEmpty(), "All recoverable LifespanAndStage should be committed when fetching final fragments"); return Streams.concat( - noCommitUnrecoverableLifespanAndStageStates.values().stream(), - taskCommitUnrecoverableLifespanAndStageStates.values().stream(), - committedRecoverableLifespanAndStages.values().stream()) + noCommitUnrecoverableLifespanAndStageStates.values().stream(), + taskCommitUnrecoverableLifespanAndStageStates.values().stream(), + committedRecoverableLifespanAndStages.values().stream()) .map(LifespanAndStageState::getFragments) .flatMap(List::stream) .collect(toImmutableList()); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/TemporaryTableUtil.java b/presto-main/src/main/java/com/facebook/presto/sql/TemporaryTableUtil.java index 2c8cec755b598..a88000274d3c5 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/TemporaryTableUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/TemporaryTableUtil.java @@ -16,8 +16,8 @@ import com.facebook.presto.Session; import com.facebook.presto.SystemSessionProperties; import com.facebook.presto.common.predicate.TupleDomain; +import com.facebook.presto.metadata.FunctionAndTypeManager; import com.facebook.presto.metadata.Metadata; -import com.facebook.presto.metadata.NewTableLayout; import com.facebook.presto.metadata.PartitioningMetadata; import com.facebook.presto.metadata.TableLayout; import com.facebook.presto.metadata.TableLayoutResult; @@ -25,11 +25,15 @@ import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorNewTableLayout; import com.facebook.presto.spi.Constraint; +import com.facebook.presto.spi.NewTableLayout; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.SourceLocation; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.TableMetadata; import com.facebook.presto.spi.VariableAllocator; +import com.facebook.presto.spi.function.AggregationFunctionImplementation; +import com.facebook.presto.spi.function.FunctionHandle; +import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.Partitioning; import com.facebook.presto.spi.plan.PartitioningHandle; @@ -37,7 +41,11 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.StatisticAggregations; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; +import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.ConstantExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -45,10 +53,7 @@ import com.facebook.presto.sql.planner.BasePlanFragmenter; import com.facebook.presto.sql.planner.StatisticsAggregationPlanner; import com.facebook.presto.sql.planner.plan.ExchangeNode; -import com.facebook.presto.sql.planner.plan.StatisticAggregations; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -318,14 +323,14 @@ public static TableFinishNode createTemporaryTableWriteWithExchanges( TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle); TableStatisticsMetadata statisticsMetadata = metadata.getStatisticsCollectionMetadataForWrite(session, catalogName, tableMetadata.getMetadata()); StatisticsAggregationPlanner.TableStatisticAggregation statisticsResult = statisticsAggregationPlanner.createStatisticsAggregation(statisticsMetadata, columnNameToVariable); - StatisticAggregations.Parts aggregations = statisticsResult.getAggregations().splitIntoPartialAndFinal(variableAllocator, metadata.getFunctionAndTypeManager()); + StatisticAggregations.Parts aggregations = splitIntoPartialAndFinal(statisticsResult.getAggregations(), variableAllocator, metadata.getFunctionAndTypeManager()); PlanNode tableWriterMerge; // Disabled by default. Enable when the column statistics are essential for future runtime adaptive plan optimizations boolean enableStatsCollectionForTemporaryTable = SystemSessionProperties.isEnableStatsCollectionForTemporaryTable(session); if (isTableWriterMergeOperatorEnabled(session)) { - StatisticAggregations.Parts localAggregations = aggregations.getPartialAggregation().splitIntoPartialAndIntermediate(variableAllocator, metadata.getFunctionAndTypeManager()); + StatisticAggregations.Parts localAggregations = splitIntoPartialAndIntermediate(aggregations.getPartialAggregation(), variableAllocator, metadata.getFunctionAndTypeManager()); tableWriterMerge = new TableWriterMergeNode( sourceLocation, idAllocator.getNextId(), @@ -384,4 +389,59 @@ public static TableFinishNode createTemporaryTableWriteWithExchanges( enableStatsCollectionForTemporaryTable ? Optional.of(aggregations.getFinalAggregation()) : Optional.empty(), enableStatsCollectionForTemporaryTable ? Optional.of(statisticsResult.getDescriptor()) : Optional.empty()); } + + public static StatisticAggregations.Parts splitIntoPartialAndFinal(StatisticAggregations statisticAggregations, VariableAllocator variableAllocator, FunctionAndTypeManager functionAndTypeManager) + { + return split(statisticAggregations, variableAllocator, functionAndTypeManager, false); + } + + public static StatisticAggregations.Parts splitIntoPartialAndIntermediate(StatisticAggregations statisticAggregations, VariableAllocator variableAllocator, FunctionAndTypeManager functionAndTypeManager) + { + return split(statisticAggregations, variableAllocator, functionAndTypeManager, true); + } + + private static StatisticAggregations.Parts split(StatisticAggregations statisticAggregations, VariableAllocator variableAllocator, FunctionAndTypeManager functionAndTypeManager, boolean intermediate) + { + ImmutableMap.Builder finalOrIntermediateAggregations = ImmutableMap.builder(); + ImmutableMap.Builder partialAggregations = ImmutableMap.builder(); + for (Map.Entry entry : statisticAggregations.getAggregations().entrySet()) { + AggregationNode.Aggregation originalAggregation = entry.getValue(); + FunctionHandle functionHandle = originalAggregation.getFunctionHandle(); + AggregationFunctionImplementation function = functionAndTypeManager.getAggregateFunctionImplementation(functionHandle); + + // create partial aggregation + VariableReferenceExpression partialVariable = variableAllocator.newVariable(entry.getValue().getCall().getSourceLocation(), functionAndTypeManager.getFunctionMetadata(functionHandle).getName().getObjectName(), function.getIntermediateType()); + partialAggregations.put(partialVariable, new AggregationNode.Aggregation( + new CallExpression( + originalAggregation.getCall().getSourceLocation(), + originalAggregation.getCall().getDisplayName(), + functionHandle, + function.getIntermediateType(), + originalAggregation.getArguments()), + originalAggregation.getFilter(), + originalAggregation.getOrderBy(), + originalAggregation.isDistinct(), + originalAggregation.getMask())); + + // create final aggregation + finalOrIntermediateAggregations.put(entry.getKey(), + new AggregationNode.Aggregation( + new CallExpression( + originalAggregation.getCall().getSourceLocation(), + originalAggregation.getCall().getDisplayName(), + functionHandle, + intermediate ? function.getIntermediateType() : function.getFinalType(), + ImmutableList.of(partialVariable)), + Optional.empty(), + Optional.empty(), + false, + Optional.empty())); + } + + StatisticAggregations finalOrIntermediateAggregation = new StatisticAggregations(finalOrIntermediateAggregations.build(), statisticAggregations.getGroupingVariables()); + return new StatisticAggregations.Parts( + intermediate ? Optional.empty() : Optional.of(finalOrIntermediateAggregation), + intermediate ? Optional.of(finalOrIntermediateAggregation) : Optional.empty(), + new StatisticAggregations(partialAggregations.build(), statisticAggregations.getGroupingVariables())); + } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/BasePlanFragmenter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/BasePlanFragmenter.java index a3da01b778bf4..18a2e199468cf 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/BasePlanFragmenter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/BasePlanFragmenter.java @@ -34,7 +34,9 @@ import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.StageExecutionDescriptor; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -45,8 +47,6 @@ import com.facebook.presto.sql.planner.plan.SequenceNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.sanity.PlanChecker; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java index 044b821f91c1d..08d4868e2ff26 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/CanonicalPlanGenerator.java @@ -44,7 +44,9 @@ import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.SortNode; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -59,8 +61,6 @@ import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SequenceNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/GroupedExecutionTagger.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/GroupedExecutionTagger.java index 59d4a233d6be7..b3a87ed704c16 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/GroupedExecutionTagger.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/GroupedExecutionTagger.java @@ -26,10 +26,10 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.WindowNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.RowNumberNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.google.common.base.VerifyException; import com.google.common.collect.ImmutableList; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java index 403875db37908..c1b80bdf1ad3b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LocalExecutionPlanner.java @@ -145,6 +145,7 @@ import com.facebook.presto.spi.plan.AggregationNode.Aggregation; import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; @@ -163,7 +164,10 @@ import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.SpatialJoinNode; import com.facebook.presto.spi.plan.StageExecutionDescriptor; +import com.facebook.presto.spi.plan.StatisticAggregationsDescriptor; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -192,7 +196,6 @@ import com.facebook.presto.sql.gen.PageFunctionCompiler; import com.facebook.presto.sql.planner.optimizations.IndexJoinOptimizer; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; @@ -204,11 +207,8 @@ import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; -import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.plan.UpdateNode; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java index 8c5a5c7a52a0a..f89c177452e10 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/LogicalPlanner.java @@ -18,17 +18,18 @@ import com.facebook.presto.common.predicate.TupleDomain; import com.facebook.presto.common.type.Type; import com.facebook.presto.metadata.Metadata; -import com.facebook.presto.metadata.NewTableLayout; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ColumnMetadata; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorTableMetadata; +import com.facebook.presto.spi.NewTableLayout; import com.facebook.presto.spi.PrestoException; import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.TableMetadata; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.OutputNode; import com.facebook.presto.spi.plan.Partitioning; @@ -36,7 +37,11 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.ProjectNode; +import com.facebook.presto.spi.plan.StatisticAggregations; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; +import com.facebook.presto.spi.plan.TableWriterNode.DeleteHandle; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -48,13 +53,8 @@ import com.facebook.presto.sql.analyzer.Scope; import com.facebook.presto.sql.parser.SqlParser; import com.facebook.presto.sql.planner.StatisticsAggregationPlanner.TableStatisticAggregation; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; -import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode.DeleteHandle; import com.facebook.presto.sql.planner.plan.UpdateNode; import com.facebook.presto.sql.tree.Analyze; import com.facebook.presto.sql.tree.Cast; @@ -93,16 +93,16 @@ import static com.facebook.presto.spi.StandardErrorCode.NOT_SUPPORTED; import static com.facebook.presto.spi.plan.AggregationNode.singleGroupingSet; import static com.facebook.presto.spi.plan.LimitNode.Step.FINAL; +import static com.facebook.presto.spi.plan.TableWriterNode.CreateName; +import static com.facebook.presto.spi.plan.TableWriterNode.InsertReference; +import static com.facebook.presto.spi.plan.TableWriterNode.RefreshMaterializedViewReference; +import static com.facebook.presto.spi.plan.TableWriterNode.UpdateTarget; import static com.facebook.presto.spi.statistics.TableStatisticType.ROW_COUNT; +import static com.facebook.presto.sql.TemporaryTableUtil.splitIntoPartialAndFinal; import static com.facebook.presto.sql.analyzer.ExpressionTreeUtils.createSymbolReference; import static com.facebook.presto.sql.analyzer.ExpressionTreeUtils.getSourceLocation; import static com.facebook.presto.sql.planner.PlannerUtils.newVariable; import static com.facebook.presto.sql.planner.TranslateExpressionsUtil.toRowExpression; -import static com.facebook.presto.sql.planner.plan.TableWriterNode.CreateName; -import static com.facebook.presto.sql.planner.plan.TableWriterNode.InsertReference; -import static com.facebook.presto.sql.planner.plan.TableWriterNode.RefreshMaterializedViewReference; -import static com.facebook.presto.sql.planner.plan.TableWriterNode.UpdateTarget; -import static com.facebook.presto.sql.planner.plan.TableWriterNode.WriterTarget; import static com.facebook.presto.sql.relational.Expressions.constant; import static com.facebook.presto.sql.tree.ExplainFormat.Type.TEXT; import static com.google.common.base.Preconditions.checkState; @@ -301,7 +301,7 @@ private RelationPlan createRefreshMaterializedViewPlan(Analysis analysis, Refres TableHandle tableHandle = viewAnalysis.getTarget(); List columnHandles = viewAnalysis.getColumns(); - WriterTarget target = new RefreshMaterializedViewReference(tableHandle, metadata.getTableMetadata(session, tableHandle).getTable()); + TableWriterNode.WriterTarget target = new RefreshMaterializedViewReference(tableHandle, metadata.getTableMetadata(session, tableHandle).getTable()); return buildInternalInsertPlan(tableHandle, columnHandles, viewAnalysis.getQuery(), analysis, target); } @@ -312,7 +312,7 @@ private RelationPlan createInsertPlan(Analysis analysis, Insert insertStatement) TableHandle tableHandle = insertAnalysis.getTarget(); List columnHandles = insertAnalysis.getColumns(); - WriterTarget target = new InsertReference(tableHandle, metadata.getTableMetadata(session, tableHandle).getTable()); + TableWriterNode.WriterTarget target = new InsertReference(tableHandle, metadata.getTableMetadata(session, tableHandle).getTable()); return buildInternalInsertPlan(tableHandle, columnHandles, insertStatement.getQuery(), analysis, target); } @@ -322,7 +322,7 @@ private RelationPlan buildInternalInsertPlan( List columnHandles, Query query, Analysis analysis, - WriterTarget target) + TableWriterNode.WriterTarget target) { TableMetadata tableMetadata = metadata.getTableMetadata(session, tableHandle); @@ -391,7 +391,7 @@ private RelationPlan buildInternalInsertPlan( private RelationPlan createTableWriterPlan( Analysis analysis, RelationPlan plan, - WriterTarget target, + TableWriterNode.WriterTarget target, List columnNames, List columnMetadataList, Optional writeTableLayout, @@ -423,7 +423,7 @@ private RelationPlan createTableWriterPlan( if (!statisticsMetadata.isEmpty()) { TableStatisticAggregation result = statisticsAggregationPlanner.createStatisticsAggregation(statisticsMetadata, columnToVariableMap); - StatisticAggregations.Parts aggregations = result.getAggregations().splitIntoPartialAndFinal(variableAllocator, metadata.getFunctionAndTypeManager()); + StatisticAggregations.Parts aggregations = splitIntoPartialAndFinal(result.getAggregations(), variableAllocator, metadata.getFunctionAndTypeManager()); TableFinishNode commitNode = new TableFinishNode( source.getSourceLocation(), @@ -589,13 +589,13 @@ private ConnectorTableMetadata createTableMetadata(QualifiedObjectName table, Li private RowExpression rowExpression(Expression expression, SqlPlannerContext context, Analysis analysis) { return toRowExpression( - expression, - metadata, - session, - sqlParser, - variableAllocator, - analysis, - context.getTranslatorContext()); + expression, + metadata, + session, + sqlParser, + variableAllocator, + analysis, + context.getTranslatorContext()); } private static List getOutputTableColumns(RelationPlan plan, Optional> columnAliases) diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java index 172f4e6052c36..196124e3f79f7 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/OutputExtractor.java @@ -17,9 +17,9 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.SequenceNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.google.common.base.VerifyException; import java.util.Optional; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenterUtils.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenterUtils.java index 5570b14859cb2..2304e0eb1cbe8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenterUtils.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/PlanFragmenterUtils.java @@ -28,14 +28,14 @@ import com.facebook.presto.spi.plan.PartitioningScheme; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; import com.facebook.presto.sql.planner.plan.MetadataDeleteNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java index bf257e0f7a59f..cde6af2ab9453 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java @@ -29,6 +29,7 @@ import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.AggregationNode.Aggregation; import com.facebook.presto.spi.plan.Assignments; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.LimitNode; import com.facebook.presto.spi.plan.Ordering; @@ -52,7 +53,6 @@ import com.facebook.presto.sql.analyzer.RelationType; import com.facebook.presto.sql.analyzer.Scope; import com.facebook.presto.sql.parser.SqlParser; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.GroupIdNode; import com.facebook.presto.sql.planner.plan.OffsetNode; import com.facebook.presto.sql.planner.plan.UpdateNode; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java index bdf96975c74ef..2396eb32b8265 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/SplitSourceFactory.java @@ -21,6 +21,7 @@ import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.JoinNode; @@ -35,7 +36,9 @@ import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.SpatialJoinNode; import com.facebook.presto.spi.plan.StageExecutionDescriptor; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -44,7 +47,6 @@ import com.facebook.presto.split.SplitSource; import com.facebook.presto.split.SplitSourceProvider; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; @@ -56,9 +58,7 @@ import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.google.common.collect.ImmutableList; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java index 305ad85f77b0b..b84a5e23d7b51 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/StatisticsAggregationPlanner.java @@ -22,6 +22,8 @@ import com.facebook.presto.spi.function.FunctionMetadata; import com.facebook.presto.spi.function.StandardFunctionResolution; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.StatisticAggregations; +import com.facebook.presto.spi.plan.StatisticAggregationsDescriptor; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; @@ -31,8 +33,6 @@ import com.facebook.presto.spi.statistics.TableStatisticsMetadata; import com.facebook.presto.sql.analyzer.FunctionAndTypeResolver; import com.facebook.presto.sql.analyzer.TypeSignatureProvider; -import com.facebook.presto.sql.planner.plan.StatisticAggregations; -import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; import com.facebook.presto.sql.relational.FunctionResolution; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -192,7 +192,7 @@ private ColumnStatisticsAggregation createColumnAggregationFromFunctionName(Colu } private ColumnStatisticsAggregation createColumnAggregation(ColumnStatisticMetadata columnStatisticMetadata, VariableReferenceExpression input, - Map columnNameToInputVariableNameMap) + Map columnNameToInputVariableNameMap) { if (columnStatisticMetadata.isSqlExpression()) { return createColumnAggregationFromSqlFunction(columnStatisticMetadata.getFunction(), input, columnNameToInputVariableNameMap); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushTableWriteThroughUnion.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushTableWriteThroughUnion.java index 006995355dfcc..09745d2619435 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushTableWriteThroughUnion.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/PushTableWriteThroughUnion.java @@ -19,11 +19,11 @@ import com.facebook.presto.matching.Pattern; import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.optimizations.SymbolMapper; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableListMultimap; import com.google.common.collect.ImmutableMap; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveEmptyDelete.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveEmptyDelete.java index 590248c77be1b..e53bec0b16163 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveEmptyDelete.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RemoveEmptyDelete.java @@ -15,9 +15,9 @@ import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.google.common.collect.ImmutableList; import java.util.Optional; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RowExpressionRewriteRuleSet.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RowExpressionRewriteRuleSet.java index 925c11a7e7547..9dd42cbc639e2 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RowExpressionRewriteRuleSet.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/RowExpressionRewriteRuleSet.java @@ -22,6 +22,9 @@ import com.facebook.presto.spi.plan.JoinNode; import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.SpatialJoinNode; +import com.facebook.presto.spi.plan.StatisticAggregations; +import com.facebook.presto.spi.plan.TableFinishNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.plan.WindowNode; import com.facebook.presto.spi.relation.CallExpression; @@ -29,9 +32,6 @@ import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.iterative.Rule; import com.facebook.presto.sql.planner.plan.ApplyNode; -import com.facebook.presto.sql.planner.plan.StatisticAggregations; -import com.facebook.presto.sql.planner.plan.TableFinishNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ScaledWriterRule.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ScaledWriterRule.java index f2d65ba86a37b..73fad6e2e7089 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ScaledWriterRule.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/iterative/rule/ScaledWriterRule.java @@ -16,8 +16,8 @@ import com.facebook.presto.Session; import com.facebook.presto.matching.Captures; import com.facebook.presto.matching.Pattern; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.sql.planner.iterative.Rule; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import java.util.Optional; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java index ec6da969b1185..5644d778ff417 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddExchanges.java @@ -45,7 +45,9 @@ import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.SpatialJoinNode; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -71,8 +73,6 @@ import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SequenceNode; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.google.common.annotations.VisibleForTesting; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java index c714022eb8749..9764defd1aaaf 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/AddLocalExchanges.java @@ -36,6 +36,9 @@ import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.SpatialJoinNode; +import com.facebook.presto.spi.plan.StatisticAggregations; +import com.facebook.presto.spi.plan.TableFinishNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.WindowNode; @@ -50,11 +53,8 @@ import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; import com.facebook.presto.sql.planner.plan.LateralJoinNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; -import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -79,6 +79,7 @@ import static com.facebook.presto.common.type.VarbinaryType.VARBINARY; import static com.facebook.presto.operator.aggregation.AggregationUtils.hasSingleNodeExecutionPreference; import static com.facebook.presto.operator.aggregation.AggregationUtils.isDecomposable; +import static com.facebook.presto.sql.TemporaryTableUtil.splitIntoPartialAndIntermediate; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_ARBITRARY_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION; import static com.facebook.presto.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION; @@ -554,7 +555,8 @@ public PlanWithProperties visitTableWriter(TableWriterNode originalTableWriterNo Optional statisticAggregations = originalTableWriterNode .getStatisticsAggregation() - .map(aggregations -> aggregations.splitIntoPartialAndIntermediate( + .map(aggregations -> splitIntoPartialAndIntermediate( + aggregations, variableAllocator, metadata.getFunctionAndTypeManager())); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MetadataDeleteOptimizer.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MetadataDeleteOptimizer.java index 412649f902335..2d132f80d6842 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MetadataDeleteOptimizer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/MetadataDeleteOptimizer.java @@ -17,15 +17,15 @@ import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.MetadataDeleteNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.google.common.collect.Iterables; import java.util.List; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java index 95a4fa3b5a46b..3b00ebd61a333 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PropertyDerivations.java @@ -24,6 +24,7 @@ import com.facebook.presto.spi.LocalProperty; import com.facebook.presto.spi.SortingProperty; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; @@ -40,7 +41,9 @@ import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.SpatialJoinNode; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.plan.WindowNode; @@ -51,7 +54,6 @@ import com.facebook.presto.sql.planner.optimizations.ActualProperties.Global; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; @@ -65,9 +67,7 @@ import com.facebook.presto.sql.planner.plan.SampleNode; import com.facebook.presto.sql.planner.plan.SequenceNode; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.relational.RowExpressionDomainTranslator; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java index 69bfed87d9814..adf739944fb31 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PruneUnreferencedOutputs.java @@ -22,6 +22,7 @@ import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.CteConsumerNode; import com.facebook.presto.spi.plan.CteProducerNode; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.ExceptNode; @@ -39,7 +40,10 @@ import com.facebook.presto.spi.plan.SetOperationNode; import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.SpatialJoinNode; +import com.facebook.presto.spi.plan.StatisticAggregations; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -50,7 +54,6 @@ import com.facebook.presto.sql.planner.VariablesExtractor; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; import com.facebook.presto.sql.planner.plan.GroupIdNode; @@ -60,11 +63,8 @@ import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SequenceNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; -import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.google.common.collect.ImmutableList; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java index 42c648fa2ded5..3c04454458e95 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/PushdownSubfields.java @@ -49,6 +49,7 @@ import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.SpatialJoinNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.WindowNode; @@ -66,7 +67,6 @@ import com.facebook.presto.sql.planner.plan.IndexJoinNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.relational.FunctionResolution; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ReplicateSemiJoinInDelete.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ReplicateSemiJoinInDelete.java index 87623fd7bc8d1..b1dd4ef8bf474 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ReplicateSemiJoinInDelete.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/ReplicateSemiJoinInDelete.java @@ -16,11 +16,11 @@ import com.facebook.presto.Session; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.WarningCollector; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import static com.facebook.presto.spi.plan.SemiJoinNode.DistributionType.REPLICATED; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java index 84be5c263f78e..375a86df1dd47 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/StreamPropertyDerivations.java @@ -19,6 +19,7 @@ import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.LocalProperty; import com.facebook.presto.spi.plan.AggregationNode; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.JoinNode; @@ -31,7 +32,9 @@ import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.SpatialJoinNode; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -40,7 +43,6 @@ import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; @@ -54,9 +56,7 @@ import com.facebook.presto.sql.planner.plan.SampleNode; import com.facebook.presto.sql.planner.plan.SequenceNode; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.google.common.collect.ImmutableBiMap; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java index cca820e193c3e..90e09b365193f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/SymbolMapper.java @@ -26,18 +26,18 @@ import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.plan.PlanNodeIdAllocator; +import com.facebook.presto.spi.plan.StatisticAggregations; +import com.facebook.presto.spi.plan.StatisticAggregationsDescriptor; +import com.facebook.presto.spi.plan.TableFinishNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.relation.CallExpression; import com.facebook.presto.spi.relation.RowExpression; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.Symbol; import com.facebook.presto.sql.planner.TypeProvider; -import com.facebook.presto.sql.planner.plan.StatisticAggregations; -import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.ExpressionRewriter; import com.facebook.presto.sql.tree.ExpressionTreeRewriter; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java index 4a6c97145414e..512945f285670 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/optimizations/UnaliasSymbolReferences.java @@ -23,6 +23,7 @@ import com.facebook.presto.spi.plan.CteConsumerNode; import com.facebook.presto.spi.plan.CteProducerNode; import com.facebook.presto.spi.plan.CteReferenceNode; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.ExceptNode; @@ -42,7 +43,9 @@ import com.facebook.presto.spi.plan.SetOperationNode; import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.SpatialJoinNode; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -54,7 +57,6 @@ import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; @@ -69,9 +71,7 @@ import com.facebook.presto.sql.planner.plan.SequenceNode; import com.facebook.presto.sql.planner.plan.SimplePlanRewriter; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.relational.RowExpressionDeterminismEvaluator; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/InternalPlanVisitor.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/InternalPlanVisitor.java index 4c182a68ef64a..e1935487c6abb 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/InternalPlanVisitor.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/InternalPlanVisitor.java @@ -52,21 +52,11 @@ public R visitOffset(OffsetNode node, C context) return visitPlan(node, context); } - public R visitTableWriter(TableWriterNode node, C context) - { - return visitPlan(node, context); - } - public R visitTableWriteMerge(TableWriterMergeNode node, C context) { return visitPlan(node, context); } - public R visitDelete(DeleteNode node, C context) - { - return visitPlan(node, context); - } - public R visitMetadataDelete(MetadataDeleteNode node, C context) { return visitPlan(node, context); @@ -77,11 +67,6 @@ public R visitUpdate(UpdateNode node, C context) return visitPlan(node, context); } - public R visitTableFinish(TableFinishNode node, C context) - { - return visitPlan(node, context); - } - public R visitStatisticsWriterNode(StatisticsWriterNode node, C context) { return visitPlan(node, context); diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java index 532f9896d1a25..df817f60799ec 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/Patterns.java @@ -19,6 +19,7 @@ import com.facebook.presto.spi.plan.CteConsumerNode; import com.facebook.presto.spi.plan.CteProducerNode; import com.facebook.presto.spi.plan.CteReferenceNode; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.FilterNode; import com.facebook.presto.spi.plan.IntersectNode; import com.facebook.presto.spi.plan.JoinNode; @@ -31,7 +32,9 @@ import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.SpatialJoinNode; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticAggregations.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticAggregations.java deleted file mode 100644 index d90dd1d9ab75d..0000000000000 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticAggregations.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.facebook.presto.sql.planner.plan; - -import com.facebook.presto.metadata.FunctionAndTypeManager; -import com.facebook.presto.spi.VariableAllocator; -import com.facebook.presto.spi.function.AggregationFunctionImplementation; -import com.facebook.presto.spi.function.FunctionHandle; -import com.facebook.presto.spi.plan.AggregationNode.Aggregation; -import com.facebook.presto.spi.relation.CallExpression; -import com.facebook.presto.spi.relation.VariableReferenceExpression; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; - -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import static com.google.common.base.Preconditions.checkArgument; -import static java.util.Objects.requireNonNull; - -public class StatisticAggregations -{ - // outputVariables indicates the order of aggregations in the output - private final List outputVariables; - private final Map aggregations; - private final List groupingVariables; - - @JsonCreator - public StatisticAggregations( - @JsonProperty("outputVariables") List outputVariables, - @JsonProperty("aggregations") Map aggregations, - @JsonProperty("groupingVariables") List groupingVariables) - { - this.outputVariables = ImmutableList.copyOf(requireNonNull(outputVariables, "outputVariables is null")); - this.aggregations = ImmutableMap.copyOf(requireNonNull(aggregations, "aggregations is null")); - this.groupingVariables = ImmutableList.copyOf(requireNonNull(groupingVariables, "groupingVariables is null")); - checkArgument(outputVariables.size() == aggregations.size(), "outputVariables and aggregations' sizes are different"); - } - - public StatisticAggregations( - Map aggregations, - List groupingVariables) - { - this.aggregations = ImmutableMap.copyOf(requireNonNull(aggregations, "aggregations is null")); - this.groupingVariables = ImmutableList.copyOf(requireNonNull(groupingVariables, "groupingVariables is null")); - this.outputVariables = ImmutableList.copyOf(aggregations.keySet()); - } - - @JsonProperty - public List getOutputVariables() - { - return outputVariables; - } - - @JsonProperty - public Map getAggregations() - { - return aggregations; - } - - @JsonProperty - public List getGroupingVariables() - { - return groupingVariables; - } - - public Parts splitIntoPartialAndFinal(VariableAllocator variableAllocator, FunctionAndTypeManager functionAndTypeManager) - { - return split(variableAllocator, functionAndTypeManager, false); - } - - public Parts splitIntoPartialAndIntermediate(VariableAllocator variableAllocator, FunctionAndTypeManager functionAndTypeManager) - { - return split(variableAllocator, functionAndTypeManager, true); - } - - private Parts split(VariableAllocator variableAllocator, FunctionAndTypeManager functionAndTypeManager, boolean intermediate) - { - ImmutableMap.Builder finalOrIntermediateAggregations = ImmutableMap.builder(); - ImmutableMap.Builder partialAggregations = ImmutableMap.builder(); - for (Map.Entry entry : aggregations.entrySet()) { - Aggregation originalAggregation = entry.getValue(); - FunctionHandle functionHandle = originalAggregation.getFunctionHandle(); - AggregationFunctionImplementation function = functionAndTypeManager.getAggregateFunctionImplementation(functionHandle); - - // create partial aggregation - VariableReferenceExpression partialVariable = variableAllocator.newVariable(entry.getValue().getCall().getSourceLocation(), functionAndTypeManager.getFunctionMetadata(functionHandle).getName().getObjectName(), function.getIntermediateType()); - partialAggregations.put(partialVariable, new Aggregation( - new CallExpression( - originalAggregation.getCall().getSourceLocation(), - originalAggregation.getCall().getDisplayName(), - functionHandle, - function.getIntermediateType(), - originalAggregation.getArguments()), - originalAggregation.getFilter(), - originalAggregation.getOrderBy(), - originalAggregation.isDistinct(), - originalAggregation.getMask())); - - // create final aggregation - finalOrIntermediateAggregations.put(entry.getKey(), - new Aggregation( - new CallExpression( - originalAggregation.getCall().getSourceLocation(), - originalAggregation.getCall().getDisplayName(), - functionHandle, - intermediate ? function.getIntermediateType() : function.getFinalType(), - ImmutableList.of(partialVariable)), - Optional.empty(), - Optional.empty(), - false, - Optional.empty())); - } - - StatisticAggregations finalOrIntermediateAggregation = new StatisticAggregations(finalOrIntermediateAggregations.build(), groupingVariables); - return new Parts( - intermediate ? Optional.empty() : Optional.of(finalOrIntermediateAggregation), - intermediate ? Optional.of(finalOrIntermediateAggregation) : Optional.empty(), - new StatisticAggregations(partialAggregations.build(), groupingVariables)); - } - - public static class Parts - { - private final Optional finalAggregation; - private final Optional intermediateAggregation; - private final StatisticAggregations partialAggregation; - - public Parts( - Optional finalAggregation, - Optional intermediateAggregation, - StatisticAggregations partialAggregation) - { - this.finalAggregation = requireNonNull(finalAggregation, "finalAggregation is null"); - this.intermediateAggregation = requireNonNull(intermediateAggregation, "intermediateAggregation is null"); - checkArgument( - finalAggregation.isPresent() ^ intermediateAggregation.isPresent(), - "only final or only intermediate aggregation is expected to be present"); - this.partialAggregation = requireNonNull(partialAggregation, "partialAggregation is null"); - } - - public StatisticAggregations getFinalAggregation() - { - return finalAggregation.orElseThrow(() -> new IllegalStateException("finalAggregation is not present")); - } - - public StatisticAggregations getIntermediateAggregation() - { - return intermediateAggregation.orElseThrow(() -> new IllegalStateException("intermediateAggregation is not present")); - } - - public StatisticAggregations getPartialAggregation() - { - return partialAggregation; - } - } -} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticsWriterNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticsWriterNode.java index 51d73a8987349..d30220bebd943 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticsWriterNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticsWriterNode.java @@ -17,6 +17,7 @@ import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.StatisticAggregationsDescriptor; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterMergeNode.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterMergeNode.java index c04aa78c190a9..a1457a4bc3d2b 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterMergeNode.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterMergeNode.java @@ -16,6 +16,7 @@ import com.facebook.presto.spi.SourceLocation; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.StatisticAggregations; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/IOPlanPrinter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/IOPlanPrinter.java index 8769fb98a2396..5b37bfc9bb0e8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/IOPlanPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/IOPlanPrinter.java @@ -34,11 +34,11 @@ import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.TableMetadata; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.IndexSourceNode; import com.facebook.presto.sql.planner.plan.InternalPlanVisitor; -import com.facebook.presto.sql.planner.plan.TableFinishNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode.WriterTarget; import com.facebook.presto.sql.planner.planPrinter.IOPlanPrinter.IOPlan.IOPlanBuilder; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; @@ -501,7 +501,7 @@ public Void visitIndexSource(IndexSourceNode node, IOPlanBuilder context) @Override public Void visitTableFinish(TableFinishNode node, IOPlanBuilder context) { - WriterTarget writerTarget = node.getTarget().orElseThrow(() -> new VerifyException("target is absent")); + TableWriterNode.WriterTarget writerTarget = node.getTarget().orElseThrow(() -> new VerifyException("target is absent")); context.setOutputTable(new CatalogSchemaTableName( writerTarget.getConnectorId().getCatalogName(), writerTarget.getSchemaTableName().getSchemaName(), diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java index d46c188058de9..6ab838d47e953 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/planPrinter/PlanPrinter.java @@ -40,6 +40,7 @@ import com.facebook.presto.spi.plan.CteConsumerNode; import com.facebook.presto.spi.plan.CteProducerNode; import com.facebook.presto.spi.plan.CteReferenceNode; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.ExceptNode; @@ -61,7 +62,10 @@ import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.SpatialJoinNode; import com.facebook.presto.spi.plan.StageExecutionDescriptor; +import com.facebook.presto.spi.plan.StatisticAggregations; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -77,7 +81,6 @@ import com.facebook.presto.sql.planner.optimizations.JoinNodeUtils; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; @@ -91,11 +94,8 @@ import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; import com.facebook.presto.sql.planner.plan.SequenceNode; -import com.facebook.presto.sql.planner.plan.StatisticAggregations; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.relational.FunctionResolution; diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java index bf5a5a2226dcb..b566528f6bd8d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/planner/sanity/ValidateDependenciesChecker.java @@ -21,6 +21,7 @@ import com.facebook.presto.spi.plan.CteConsumerNode; import com.facebook.presto.spi.plan.CteProducerNode; import com.facebook.presto.spi.plan.CteReferenceNode; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.ExceptNode; @@ -37,7 +38,11 @@ import com.facebook.presto.spi.plan.SetOperationNode; import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.SpatialJoinNode; +import com.facebook.presto.spi.plan.StatisticAggregationsDescriptor; +import com.facebook.presto.spi.plan.StatisticAggregationsDescriptor.ColumnStatisticsDescriptor; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -48,7 +53,6 @@ import com.facebook.presto.sql.planner.optimizations.WindowNodeUtil; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; @@ -63,12 +67,8 @@ import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; import com.facebook.presto.sql.planner.plan.SequenceNode; -import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; -import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor.ColumnStatisticsDescriptor; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.google.common.collect.ImmutableList; diff --git a/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java b/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java index 0ce2526c42fdd..f1d880ebf1804 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java +++ b/presto-main/src/main/java/com/facebook/presto/util/GraphvizPrinter.java @@ -22,6 +22,7 @@ import com.facebook.presto.spi.plan.AbstractJoinNode; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.FilterNode; @@ -36,7 +37,9 @@ import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.SpatialJoinNode; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -48,7 +51,6 @@ import com.facebook.presto.sql.planner.optimizations.JoinNodeUtils; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.ExplainAnalyzeNode; @@ -63,9 +65,7 @@ import com.facebook.presto.sql.planner.plan.SampleNode; import com.facebook.presto.sql.planner.plan.SequenceNode; import com.facebook.presto.sql.planner.plan.StatisticsWriterNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; import com.facebook.presto.sql.planner.plan.TableWriterMergeNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.TopNRowNumberNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.planner.planPrinter.RowExpressionFormatter; diff --git a/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java b/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java index 5ba7b34b2d772..995db6d5571ed 100644 --- a/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java +++ b/presto-main/src/test/java/com/facebook/presto/metadata/AbstractMockMetadata.java @@ -27,6 +27,7 @@ import com.facebook.presto.spi.ConnectorTableMetadata; import com.facebook.presto.spi.Constraint; import com.facebook.presto.spi.MaterializedViewDefinition; +import com.facebook.presto.spi.NewTableLayout; import com.facebook.presto.spi.QueryId; import com.facebook.presto.spi.SystemTable; import com.facebook.presto.spi.TableHandle; diff --git a/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java b/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java index ff5b789c8607b..9f4be760e4a84 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/TestTableFinishOperator.java @@ -28,9 +28,9 @@ import com.facebook.presto.spi.function.JavaAggregationFunctionImplementation; import com.facebook.presto.spi.plan.AggregationNode; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.StatisticAggregationsDescriptor; import com.facebook.presto.spi.statistics.ColumnStatisticMetadata; import com.facebook.presto.spi.statistics.ComputedStatistics; -import com.facebook.presto.sql.planner.plan.StatisticAggregationsDescriptor; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.ListenableFuture; @@ -232,7 +232,8 @@ public void testTableWriteCommit() // pages for successful recoverable grouped execution // expect lifespan committer to be called and pages published operator.addInput(rowPagesBuilder(inputTypes).row(null, null, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 3, LIFESPAN_COMMIT, false), 9).build().get(0)); - operator.addInput(rowPagesBuilder(inputTypes).row(11, new byte[] {10}, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 3, LIFESPAN_COMMIT, true), null).build().get(0)); + operator.addInput(rowPagesBuilder(inputTypes).row(11, new byte[] { + 10}, getTableCommitContextBytes(Lifespan.driverGroup(2), 2, 3, LIFESPAN_COMMIT, true), null).build().get(0)); assertEquals(getOnlyElement(pageSinkCommitter.getCommittedFragments()), ImmutableList.of(Slices.wrappedBuffer(new byte[] {10}))); assertThat(driverContext.getSystemMemoryUsage()).isGreaterThan(0); diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestingWriterTarget.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestingWriterTarget.java index 0ed382c524e36..863047690952b 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/TestingWriterTarget.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/TestingWriterTarget.java @@ -16,7 +16,7 @@ import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.SchemaTableName; -import com.facebook.presto.sql.planner.plan.TableWriterNode; +import com.facebook.presto.spi.plan.TableWriterNode; public class TestingWriterTarget extends TableWriterNode.WriterTarget diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java index 4ff9c15143597..e059c42e89a90 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/PlanMatchPattern.java @@ -40,6 +40,7 @@ import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.SortNode; import com.facebook.presto.spi.plan.SpatialJoinNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -63,7 +64,6 @@ import com.facebook.presto.sql.planner.plan.OffsetNode; import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.facebook.presto.sql.planner.plan.SequenceNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.tree.Expression; import com.facebook.presto.sql.tree.FunctionCall; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/TableWriterMatcher.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/TableWriterMatcher.java index d642a64f87ac6..415b3595ac788 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/TableWriterMatcher.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/assertions/TableWriterMatcher.java @@ -18,9 +18,9 @@ import com.facebook.presto.cost.StatsProvider; import com.facebook.presto.metadata.Metadata; import com.facebook.presto.spi.plan.PlanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.sql.planner.Symbol; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import java.util.List; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java index e9f22a59d7aaf..41674ca683335 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/iterative/rule/test/PlanBuilder.java @@ -32,6 +32,7 @@ import com.facebook.presto.spi.plan.AggregationNode.Step; import com.facebook.presto.spi.plan.Assignments; import com.facebook.presto.spi.plan.CteProducerNode; +import com.facebook.presto.spi.plan.DeleteNode; import com.facebook.presto.spi.plan.DistinctLimitNode; import com.facebook.presto.spi.plan.EquiJoinClause; import com.facebook.presto.spi.plan.ExceptNode; @@ -54,7 +55,9 @@ import com.facebook.presto.spi.plan.ProjectNode; import com.facebook.presto.spi.plan.SemiJoinNode; import com.facebook.presto.spi.plan.SortNode; +import com.facebook.presto.spi.plan.TableFinishNode; import com.facebook.presto.spi.plan.TableScanNode; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.plan.TopNNode; import com.facebook.presto.spi.plan.UnionNode; import com.facebook.presto.spi.plan.ValuesNode; @@ -71,7 +74,6 @@ import com.facebook.presto.sql.planner.TypeProvider; import com.facebook.presto.sql.planner.plan.ApplyNode; import com.facebook.presto.sql.planner.plan.AssignUniqueId; -import com.facebook.presto.sql.planner.plan.DeleteNode; import com.facebook.presto.sql.planner.plan.EnforceSingleRowNode; import com.facebook.presto.sql.planner.plan.ExchangeNode; import com.facebook.presto.sql.planner.plan.IndexJoinNode; @@ -81,8 +83,6 @@ import com.facebook.presto.sql.planner.plan.RemoteSourceNode; import com.facebook.presto.sql.planner.plan.RowNumberNode; import com.facebook.presto.sql.planner.plan.SampleNode; -import com.facebook.presto.sql.planner.plan.TableFinishNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.facebook.presto.sql.planner.plan.UnnestNode; import com.facebook.presto.sql.relational.FunctionResolution; import com.facebook.presto.sql.relational.SqlToRowExpressionTranslator; diff --git a/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestStatisticsWriterNode.java b/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestStatisticsWriterNode.java index b2d882888a5f8..5e28a77689e8c 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestStatisticsWriterNode.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/planner/plan/TestStatisticsWriterNode.java @@ -26,6 +26,7 @@ import com.facebook.presto.spi.TableHandle; import com.facebook.presto.spi.VariableAllocator; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.StatisticAggregationsDescriptor; import com.facebook.presto.spi.plan.ValuesNode; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.facebook.presto.spi.statistics.ColumnStatisticType; diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp index 763bb62e39426..6bfc625743dba 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.cpp @@ -685,7 +685,7 @@ void to_json(json& j, const std::shared_ptr& p) { j = *std::static_pointer_cast(p); return; } - if (type == "com.facebook.presto.sql.planner.plan.TableWriterNode") { + if (type == ".TableWriterNode") { j = *std::static_pointer_cast(p); return; } @@ -837,7 +837,7 @@ void from_json(const json& j, std::shared_ptr& p) { p = std::static_pointer_cast(k); return; } - if (type == "com.facebook.presto.sql.planner.plan.TableWriterNode") { + if (type == ".TableWriterNode") { std::shared_ptr k = std::make_shared(); j.get_to(*k); p = std::static_pointer_cast(k); diff --git a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml index 2099bf0845414..51669ad6891e0 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml +++ b/presto-native-execution/presto_cpp/presto_protocol/core/presto_protocol_core.yml @@ -144,7 +144,7 @@ AbstractClasses: - { name: SampleNode, key: com.facebook.presto.sql.planner.plan.SampleNode } - { name: SemiJoinNode, key: .SemiJoinNode } - { name: TableScanNode, key: .TableScanNode } - - { name: TableWriterNode, key: com.facebook.presto.sql.planner.plan.TableWriterNode } + - { name: TableWriterNode, key: .TableWriterNode } - { name: TableWriterMergeNode, key: com.facebook.presto.sql.planner.plan.TableWriterMergeNode } - { name: TopNNode, key: .TopNNode } - { name: TopNRowNumberNode, key: com.facebook.presto.sql.planner.plan.TopNRowNumberNode } @@ -217,7 +217,7 @@ JavaClasses: - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/RowNumberNode.java - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SampleNode.java - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TopNRowNumberNode.java - - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterNode.java + - presto-spi/src/main/java/com/facebook/presto/spi/plan/TableWriterNode.java - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterMergeNode.java - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/UnnestNode.java - presto-spi/src/main/java/com/facebook/presto/spi/plan/WindowNode.java @@ -255,7 +255,7 @@ JavaClasses: - presto-main/src/main/java/com/facebook/presto/cost/PlanNodeStatsEstimate.java - presto-main/src/main/java/com/facebook/presto/cost/JoinNodeStatsEstimate.java - presto-main/src/main/java/com/facebook/presto/cost/TableWriterNodeStatsEstimate.java - - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticAggregations.java + - presto-spi/src/main/java/com/facebook/presto/spi/plan/StatisticAggregations.java - presto-main/src/main/java/com/facebook/presto/cost/PartialAggregationStatsEstimate.java - presto-spi/src/main/java/com/facebook/presto/spi/plan/ProjectNode.java - presto-matching/src/test/java/com/facebook/presto/matching/example/rel/ProjectNode.java diff --git a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml index 4c68f6e8e0736..18c9afda02ece 100644 --- a/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml +++ b/presto-native-execution/presto_cpp/presto_protocol/presto_protocol.yml @@ -143,7 +143,7 @@ AbstractClasses: - { name: SampleNode, key: com.facebook.presto.sql.planner.plan.SampleNode } - { name: SemiJoinNode, key: .SemiJoinNode } - { name: TableScanNode, key: .TableScanNode } - - { name: TableWriterNode, key: com.facebook.presto.sql.planner.plan.TableWriterNode } + - { name: TableWriterNode, key: .TableWriterNode } - { name: TableWriterMergeNode, key: com.facebook.presto.sql.planner.plan.TableWriterMergeNode } - { name: TopNNode, key: .TopNNode } - { name: TopNRowNumberNode, key: com.facebook.presto.sql.planner.plan.TopNRowNumberNode } @@ -221,7 +221,7 @@ JavaClasses: - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/RowNumberNode.java - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/SampleNode.java - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TopNRowNumberNode.java - - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterNode.java + - presto-spi/src/main/java/com/facebook/presto/spi/plan/TableWriterNode.java - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterMergeNode.java - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/UnnestNode.java - presto-spi/src/main/java/com/facebook/presto/spi/plan/WindowNode.java @@ -299,7 +299,7 @@ JavaClasses: - presto-main/src/main/java/com/facebook/presto/cost/PlanNodeStatsEstimate.java - presto-main/src/main/java/com/facebook/presto/cost/JoinNodeStatsEstimate.java - presto-main/src/main/java/com/facebook/presto/cost/TableWriterNodeStatsEstimate.java - - presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticAggregations.java + - presto-spi/src/main/java/com/facebook/presto/spi/plan/StatisticAggregations.java - presto-main/src/main/java/com/facebook/presto/cost/PartialAggregationStatsEstimate.java - presto-spi/src/main/java/com/facebook/presto/spi/plan/ProjectNode.java - presto-matching/src/test/java/com/facebook/presto/matching/example/rel/ProjectNode.java diff --git a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/task/PrestoSparkNativeTaskExecutorFactory.java b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/task/PrestoSparkNativeTaskExecutorFactory.java index 17c3afc9f1164..1822a1b0f7d7e 100644 --- a/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/task/PrestoSparkNativeTaskExecutorFactory.java +++ b/presto-spark-base/src/main/java/com/facebook/presto/spark/execution/task/PrestoSparkNativeTaskExecutorFactory.java @@ -63,11 +63,11 @@ import com.facebook.presto.spi.plan.PlanFragmentId; import com.facebook.presto.spi.plan.PlanNode; import com.facebook.presto.spi.plan.PlanNodeId; +import com.facebook.presto.spi.plan.TableWriterNode; import com.facebook.presto.spi.security.TokenAuthenticator; import com.facebook.presto.split.RemoteSplit; import com.facebook.presto.sql.planner.PlanFragment; import com.facebook.presto.sql.planner.plan.RemoteSourceNode; -import com.facebook.presto.sql.planner.plan.TableWriterNode; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; diff --git a/presto-main/src/main/java/com/facebook/presto/metadata/NewTableLayout.java b/presto-spi/src/main/java/com/facebook/presto/spi/NewTableLayout.java similarity index 86% rename from presto-main/src/main/java/com/facebook/presto/metadata/NewTableLayout.java rename to presto-spi/src/main/java/com/facebook/presto/spi/NewTableLayout.java index ef11f1bf81bb4..bd9d743fe6a48 100644 --- a/presto-main/src/main/java/com/facebook/presto/metadata/NewTableLayout.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/NewTableLayout.java @@ -11,10 +11,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.metadata; +package com.facebook.presto.spi; -import com.facebook.presto.spi.ConnectorId; -import com.facebook.presto.spi.ConnectorNewTableLayout; import com.facebook.presto.spi.connector.ConnectorTransactionHandle; import com.facebook.presto.spi.plan.PartitioningHandle; import com.fasterxml.jackson.annotation.JsonCreator; @@ -24,7 +22,7 @@ import java.util.Objects; import java.util.Optional; -import static com.google.common.base.MoreObjects.toStringHelper; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; public class NewTableLayout @@ -91,10 +89,6 @@ public int hashCode() @Override public String toString() { - return toStringHelper(this) - .add("connectorId", connectorId) - .add("transactionHandle", transactionHandle) - .add("layout", layout) - .toString(); + return format("%s {connectorId=%s, transactionHandle=%s, layout=%s}", this.getClass().getSimpleName(), connectorId, transactionHandle, layout); } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/DeleteNode.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/DeleteNode.java similarity index 83% rename from presto-main/src/main/java/com/facebook/presto/sql/planner/plan/DeleteNode.java rename to presto-spi/src/main/java/com/facebook/presto/spi/plan/DeleteNode.java index 16a334977e230..b3dd87cf6e7e8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/DeleteNode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/DeleteNode.java @@ -11,27 +11,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.plan; +package com.facebook.presto.spi.plan; import com.facebook.presto.spi.SourceLocation; -import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import javax.annotation.concurrent.Immutable; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Optional; +import static com.facebook.presto.common.Utils.checkArgument; import static java.util.Objects.requireNonNull; @Immutable public class DeleteNode - extends InternalPlanNode + extends PlanNode { private final PlanNode source; private final VariableReferenceExpression rowId; @@ -60,7 +59,7 @@ public DeleteNode( this.source = requireNonNull(source, "source is null"); this.rowId = requireNonNull(rowId, "rowId is null"); - this.outputVariables = ImmutableList.copyOf(requireNonNull(outputVariables, "outputVariables is null")); + this.outputVariables = Collections.unmodifiableList(new ArrayList<>(requireNonNull(outputVariables, "outputVariables is null"))); } @JsonProperty @@ -85,11 +84,11 @@ public List getOutputVariables() @Override public List getSources() { - return ImmutableList.of(source); + return Collections.singletonList(source); } @Override - public R accept(InternalPlanVisitor visitor, C context) + public R accept(PlanVisitor visitor, C context) { return visitor.visitDelete(this, context); } @@ -97,7 +96,8 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { - return new DeleteNode(getSourceLocation(), getId(), getStatsEquivalentPlanNode(), Iterables.getOnlyElement(newChildren), rowId, outputVariables); + checkArgument(newChildren.size() == 1); + return new DeleteNode(getSourceLocation(), getId(), getStatsEquivalentPlanNode(), newChildren.get(0), rowId, outputVariables); } @Override diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java index 676a347697169..b353ba88768de 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/PlanVisitor.java @@ -129,4 +129,19 @@ public R visitMergeJoin(MergeJoinNode node, C context) { return visitPlan(node, context); } + + public R visitDelete(DeleteNode node, C context) + { + return visitPlan(node, context); + } + + public R visitTableWriter(TableWriterNode node, C context) + { + return visitPlan(node, context); + } + + public R visitTableFinish(TableFinishNode node, C context) + { + return visitPlan(node, context); + } } diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/plan/StatisticAggregations.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/StatisticAggregations.java new file mode 100644 index 0000000000000..d1071fcefad66 --- /dev/null +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/StatisticAggregations.java @@ -0,0 +1,111 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.spi.plan; + +import com.facebook.presto.spi.plan.AggregationNode.Aggregation; +import com.facebook.presto.spi.relation.VariableReferenceExpression; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static com.facebook.presto.common.Utils.checkArgument; +import static java.util.Objects.requireNonNull; + +public class StatisticAggregations +{ + // outputVariables indicates the order of aggregations in the output + private final List outputVariables; + private final Map aggregations; + private final List groupingVariables; + + @JsonCreator + public StatisticAggregations( + @JsonProperty("outputVariables") List outputVariables, + @JsonProperty("aggregations") Map aggregations, + @JsonProperty("groupingVariables") List groupingVariables) + { + this.outputVariables = Collections.unmodifiableList(new ArrayList<>(requireNonNull(outputVariables, "outputVariables is null"))); + this.aggregations = Collections.unmodifiableMap(new LinkedHashMap<>(requireNonNull(aggregations, "aggregations is null"))); + this.groupingVariables = Collections.unmodifiableList(new ArrayList<>(requireNonNull(groupingVariables, "groupingVariables is null"))); + checkArgument(outputVariables.size() == aggregations.size(), "outputVariables and aggregations' sizes are different"); + } + + public StatisticAggregations( + Map aggregations, + List groupingVariables) + { + this.aggregations = Collections.unmodifiableMap(new LinkedHashMap<>(requireNonNull(aggregations, "aggregations is null"))); + this.groupingVariables = Collections.unmodifiableList(new ArrayList<>(requireNonNull(groupingVariables, "groupingVariables is null"))); + this.outputVariables = Collections.unmodifiableList(new ArrayList<>(aggregations.keySet())); + } + + @JsonProperty + public List getOutputVariables() + { + return outputVariables; + } + + @JsonProperty + public Map getAggregations() + { + return aggregations; + } + + @JsonProperty + public List getGroupingVariables() + { + return groupingVariables; + } + + public static class Parts + { + private final Optional finalAggregation; + private final Optional intermediateAggregation; + private final StatisticAggregations partialAggregation; + + public Parts( + Optional finalAggregation, + Optional intermediateAggregation, + StatisticAggregations partialAggregation) + { + this.finalAggregation = requireNonNull(finalAggregation, "finalAggregation is null"); + this.intermediateAggregation = requireNonNull(intermediateAggregation, "intermediateAggregation is null"); + checkArgument( + finalAggregation.isPresent() ^ intermediateAggregation.isPresent(), + "only final or only intermediate aggregation is expected to be present"); + this.partialAggregation = requireNonNull(partialAggregation, "partialAggregation is null"); + } + + public StatisticAggregations getFinalAggregation() + { + return finalAggregation.orElseThrow(() -> new IllegalStateException("finalAggregation is not present")); + } + + public StatisticAggregations getIntermediateAggregation() + { + return intermediateAggregation.orElseThrow(() -> new IllegalStateException("intermediateAggregation is not present")); + } + + public StatisticAggregations getPartialAggregation() + { + return partialAggregation; + } + } +} diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticAggregationsDescriptor.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/StatisticAggregationsDescriptor.java similarity index 72% rename from presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticAggregationsDescriptor.java rename to presto-spi/src/main/java/com/facebook/presto/spi/plan/StatisticAggregationsDescriptor.java index 7f7db1692d0a1..2651f4c454e0d 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/StatisticAggregationsDescriptor.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/StatisticAggregationsDescriptor.java @@ -11,22 +11,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.plan; +package com.facebook.presto.spi.plan; import com.facebook.presto.spi.statistics.ColumnStatisticMetadata; import com.facebook.presto.spi.statistics.TableStatisticType; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.function.Function; +import java.util.stream.Collectors; -import static com.google.common.base.MoreObjects.toStringHelper; -import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; public class StatisticAggregationsDescriptor @@ -46,8 +46,8 @@ public StatisticAggregationsDescriptor( @JsonProperty("tableStatistics") Map tableStatistics, @JsonProperty("columnStatistics") List> columnStatistics) { - this.grouping = ImmutableMap.copyOf(requireNonNull(grouping, "grouping is null")); - this.tableStatistics = ImmutableMap.copyOf(requireNonNull(tableStatistics, "tableStatistics is null")); + this.grouping = Collections.unmodifiableMap(new LinkedHashMap<>(requireNonNull(grouping, "grouping is null"))); + this.tableStatistics = Collections.unmodifiableMap(new LinkedHashMap<>(requireNonNull(tableStatistics, "tableStatistics is null"))); this.columnStatistics = requireNonNull(columnStatistics, "columnStatistics is null"); } @@ -56,10 +56,8 @@ public StatisticAggregationsDescriptor( Map tableStatistics, Map columnStatistics) { - this(grouping, tableStatistics, ImmutableList.>builder() - .addAll(columnStatistics.entrySet().stream() - .map(e -> new ColumnStatisticsDescriptor<>(e.getKey(), e.getValue())).iterator()) - .build()); + this(grouping, tableStatistics, Collections.unmodifiableList(columnStatistics.entrySet().stream() + .map(e -> new ColumnStatisticsDescriptor<>(e.getKey(), e.getValue())).collect(Collectors.toList()))); } @JsonProperty @@ -104,11 +102,7 @@ public int hashCode() @Override public String toString() { - return toStringHelper(this) - .add("grouping", grouping) - .add("tableStatistics", tableStatistics) - .add("columnStatistics", columnStatistics) - .toString(); + return format("%s {grouping=%s, tableStatistics=%s, columnStatistics=%s}", this.getClass().getSimpleName(), grouping, tableStatistics, columnStatistics); } public static Builder builder() @@ -118,28 +112,33 @@ public static Builder builder() public StatisticAggregationsDescriptor map(Function mapper) { + Map newMap = new LinkedHashMap<>(); + this.getColumnStatistics().forEach(x -> + { + if (newMap.containsKey(x.getMetadata())) { + throw new IllegalStateException(format("Duplicate key %s", x)); + } + newMap.put(x.getMetadata(), x.getItem()); + }); return new StatisticAggregationsDescriptor<>( map(this.getGrouping(), mapper), map(this.getTableStatistics(), mapper), - map(this.getColumnStatistics().stream() - .collect(toImmutableMap( - ColumnStatisticsDescriptor::getMetadata, - ColumnStatisticsDescriptor::getItem)), + map(Collections.unmodifiableMap(newMap), mapper)); } private static Map map(Map input, Function mapper) { - return input.entrySet() + return Collections.unmodifiableMap(input.entrySet() .stream() - .collect(toImmutableMap(Map.Entry::getKey, entry -> mapper.apply(entry.getValue()))); + .collect(Collectors.toMap(Map.Entry::getKey, entry -> mapper.apply(entry.getValue()), (e1, e2) -> { throw new IllegalStateException(format("Duplicate key %s", e1)); }, LinkedHashMap::new))); } public static class Builder { - private final ImmutableMap.Builder grouping = ImmutableMap.builder(); - private final ImmutableMap.Builder tableStatistics = ImmutableMap.builder(); - private final ImmutableMap.Builder columnStatistics = ImmutableMap.builder(); + private final Map grouping = new LinkedHashMap<>(); + private final Map tableStatistics = new LinkedHashMap<>(); + private final Map columnStatistics = new LinkedHashMap<>(); public void addGrouping(String column, T key) { @@ -158,7 +157,7 @@ public void addColumnStatistic(ColumnStatisticMetadata statisticMetadata, T key) public StatisticAggregationsDescriptor build() { - return new StatisticAggregationsDescriptor<>(grouping.build(), tableStatistics.build(), columnStatistics.build()); + return new StatisticAggregationsDescriptor<>(Collections.unmodifiableMap(grouping), Collections.unmodifiableMap(tableStatistics), Collections.unmodifiableMap(columnStatistics)); } } @@ -211,10 +210,7 @@ public boolean equals(Object o) @Override public String toString() { - return toStringHelper(this) - .add("metadata", metadata) - .add("item", item) - .toString(); + return format("%s {metadata=%s, item=%s}", this.getClass().getSimpleName(), metadata, item); } } } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableFinishNode.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/TableFinishNode.java similarity index 85% rename from presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableFinishNode.java rename to presto-spi/src/main/java/com/facebook/presto/spi/plan/TableFinishNode.java index 566168defb0c8..3e510c2123fa8 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableFinishNode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/TableFinishNode.java @@ -11,33 +11,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.plan; +package com.facebook.presto.spi.plan; import com.facebook.presto.spi.SourceLocation; -import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import javax.annotation.concurrent.Immutable; +import java.util.Collections; import java.util.List; import java.util.Optional; -import static com.facebook.presto.sql.planner.plan.TableWriterNode.WriterTarget; -import static com.google.common.base.Preconditions.checkArgument; +import static com.facebook.presto.common.Utils.checkArgument; import static java.util.Objects.requireNonNull; @Immutable public class TableFinishNode - extends InternalPlanNode + extends PlanNode { private final PlanNode source; - private final Optional target; + private final Optional target; private final VariableReferenceExpression rowCountVariable; private final Optional statisticsAggregation; private final Optional> statisticsAggregationDescriptor; @@ -47,7 +43,7 @@ public TableFinishNode( Optional sourceLocation, @JsonProperty("id") PlanNodeId id, @JsonProperty("source") PlanNode source, - @JsonProperty("target") Optional target, + @JsonProperty("target") Optional target, @JsonProperty("rowCountVariable") VariableReferenceExpression rowCountVariable, @JsonProperty("statisticsAggregation") Optional statisticsAggregation, @JsonProperty("statisticsAggregationDescriptor") Optional> statisticsAggregationDescriptor) @@ -60,7 +56,7 @@ public TableFinishNode( PlanNodeId id, Optional statsEquivalentPlanNode, PlanNode source, - Optional target, + Optional target, VariableReferenceExpression rowCountVariable, Optional statisticsAggregation, Optional> statisticsAggregationDescriptor) @@ -83,7 +79,7 @@ public PlanNode getSource() } @JsonIgnore - public Optional getTarget() + public Optional getTarget() { return target; } @@ -109,17 +105,17 @@ public Optional> ge @Override public List getSources() { - return ImmutableList.of(source); + return Collections.singletonList(source); } @Override public List getOutputVariables() { - return ImmutableList.of(rowCountVariable); + return Collections.singletonList(rowCountVariable); } @Override - public R accept(InternalPlanVisitor visitor, C context) + public R accept(PlanVisitor visitor, C context) { return visitor.visitTableFinish(this, context); } @@ -127,11 +123,12 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { + checkArgument(newChildren.size() == 1); return new TableFinishNode( getSourceLocation(), getId(), getStatsEquivalentPlanNode(), - Iterables.getOnlyElement(newChildren), + newChildren.get(0), target, rowCountVariable, statisticsAggregation, diff --git a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterNode.java b/presto-spi/src/main/java/com/facebook/presto/spi/plan/TableWriterNode.java similarity index 91% rename from presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterNode.java rename to presto-spi/src/main/java/com/facebook/presto/spi/plan/TableWriterNode.java index 459dd4a425f35..98f5a39e4cabc 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/planner/plan/TableWriterNode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/plan/TableWriterNode.java @@ -11,38 +11,36 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.facebook.presto.sql.planner.plan; +package com.facebook.presto.spi.plan; -import com.facebook.presto.metadata.NewTableLayout; import com.facebook.presto.spi.ColumnHandle; import com.facebook.presto.spi.ConnectorId; import com.facebook.presto.spi.ConnectorTableMetadata; +import com.facebook.presto.spi.NewTableLayout; import com.facebook.presto.spi.SchemaTableName; import com.facebook.presto.spi.SourceLocation; import com.facebook.presto.spi.TableHandle; -import com.facebook.presto.spi.plan.PartitioningScheme; -import com.facebook.presto.spi.plan.PlanNode; -import com.facebook.presto.spi.plan.PlanNodeId; import com.facebook.presto.spi.relation.VariableReferenceExpression; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Iterables; import javax.annotation.concurrent.Immutable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; -import static com.google.common.base.Preconditions.checkArgument; +import static com.facebook.presto.common.Utils.checkArgument; +import static java.lang.String.format; import static java.util.Objects.requireNonNull; @Immutable public class TableWriterNode - extends InternalPlanNode + extends PlanNode { private final PlanNode source; private final Optional target; @@ -112,22 +110,22 @@ public TableWriterNode( this.rowCountVariable = requireNonNull(rowCountVariable, "rowCountVariable is null"); this.fragmentVariable = requireNonNull(fragmentVariable, "fragmentVariable is null"); this.tableCommitContextVariable = requireNonNull(tableCommitContextVariable, "tableCommitContextVariable is null"); - this.columns = ImmutableList.copyOf(columns); - this.columnNames = ImmutableList.copyOf(columnNames); - this.notNullColumnVariables = ImmutableSet.copyOf(requireNonNull(notNullColumnVariables, "notNullColumns is null")); + this.columns = Collections.unmodifiableList(new ArrayList<>(columns)); + this.columnNames = Collections.unmodifiableList(new ArrayList<>(columnNames)); + this.notNullColumnVariables = Collections.unmodifiableSet(new LinkedHashSet<>(requireNonNull(notNullColumnVariables, "notNullColumns is null"))); this.tablePartitioningScheme = requireNonNull(tablePartitioningScheme, "partitioningScheme is null"); this.preferredShufflePartitioningScheme = requireNonNull(preferredShufflePartitioningScheme, "preferredShufflePartitioningScheme is null"); this.statisticsAggregation = requireNonNull(statisticsAggregation, "statisticsAggregation is null"); - ImmutableList.Builder outputs = ImmutableList.builder() - .add(rowCountVariable) - .add(fragmentVariable) - .add(tableCommitContextVariable); + List outputsList = new ArrayList<>(); + outputsList.add(rowCountVariable); + outputsList.add(fragmentVariable); + outputsList.add(tableCommitContextVariable); statisticsAggregation.ifPresent(aggregation -> { - outputs.addAll(aggregation.getGroupingVariables()); - outputs.addAll(aggregation.getAggregations().keySet()); + outputsList.addAll(aggregation.getGroupingVariables()); + outputsList.addAll(aggregation.getAggregations().keySet()); }); - this.outputs = outputs.build(); + this.outputs = Collections.unmodifiableList(outputsList); this.taskCountIfScaledWriter = requireNonNull(taskCountIfScaledWriter, "taskCountIfScaledWriter is null"); this.isTemporaryTableWriter = requireNonNull(isTemporaryTableWriter, "isTemporaryTableWriter is null"); } @@ -201,7 +199,7 @@ public Optional getStatisticsAggregation() @Override public List getSources() { - return ImmutableList.of(source); + return Collections.singletonList(source); } @Override @@ -223,7 +221,7 @@ public Optional getIsTemporaryTableWriter() } @Override - public R accept(InternalPlanVisitor visitor, C context) + public R accept(PlanVisitor visitor, C context) { return visitor.visitTableWriter(this, context); } @@ -231,11 +229,12 @@ public R accept(InternalPlanVisitor visitor, C context) @Override public PlanNode replaceChildren(List newChildren) { + checkArgument(newChildren.size() == 1); return new TableWriterNode( getSourceLocation(), getId(), getStatsEquivalentPlanNode(), - Iterables.getOnlyElement(newChildren), + newChildren.get(0), target, rowCountVariable, fragmentVariable, @@ -451,7 +450,7 @@ public UpdateTarget( { this.handle = requireNonNull(handle, "handle is null"); this.schemaTableName = requireNonNull(schemaTableName, "schemaTableName is null"); - checkArgument(updatedColumns.size() == updatedColumnHandles.size(), "updatedColumns size %s must equal updatedColumnHandles size %s", updatedColumns.size(), updatedColumnHandles.size()); + checkArgument(updatedColumns.size() == updatedColumnHandles.size(), format("updatedColumns size %s must equal updatedColumnHandles size %s", updatedColumns.size(), updatedColumnHandles.size())); this.updatedColumns = requireNonNull(updatedColumns, "updatedColumns is null"); this.updatedColumnHandles = requireNonNull(updatedColumnHandles, "updatedColumnHandles is null"); }