diff --git a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/RelationId.java b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/RelationId.java index bdac0594f242d..b4ff1ad5f3daf 100644 --- a/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/RelationId.java +++ b/presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/RelationId.java @@ -53,6 +53,11 @@ public boolean isAnonymous() return sourceNode == null; } + public Node getSourceNode() + { + return sourceNode; + } + @Override public boolean equals(Object o) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analyzer.java index 742e68453174a..35d1793e3e500 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/Analyzer.java @@ -115,7 +115,7 @@ public Analysis analyzeSemantic(Statement statement, boolean isDescribe) metadataExtractor.populateMetadataHandle(session, rewrittenStatement, analysis.getMetadataHandle()); StatementAnalyzer analyzer = new StatementAnalyzer(analysis, metadata, sqlParser, accessControl, session, warningCollector); analyzer.analyze(rewrittenStatement, Optional.empty()); - analyzeForUtilizedColumns(analysis, analysis.getStatement()); + analyzeForUtilizedColumns(analysis, analysis.getStatement(), warningCollector); analysis.populateTableColumnAndSubfieldReferencesForAccessControl(isCheckAccessControlOnUtilizedColumnsOnly(session), isCheckAccessControlWithSubfields(session)); return analysis; } diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/UtilizedColumnsAnalyzer.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/UtilizedColumnsAnalyzer.java index 895145bff7975..965396ecd390f 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/UtilizedColumnsAnalyzer.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/UtilizedColumnsAnalyzer.java @@ -15,6 +15,8 @@ import com.facebook.airlift.log.Logger; import com.facebook.presto.common.QualifiedObjectName; +import com.facebook.presto.spi.PrestoWarning; +import com.facebook.presto.spi.WarningCollector; import com.facebook.presto.spi.analyzer.AccessControlInfo; import com.facebook.presto.sql.tree.AliasedRelation; import com.facebook.presto.sql.tree.Cube; @@ -37,6 +39,7 @@ import com.facebook.presto.sql.tree.Lateral; import com.facebook.presto.sql.tree.Node; import com.facebook.presto.sql.tree.NodeRef; +import com.facebook.presto.sql.tree.QualifiedName; import com.facebook.presto.sql.tree.Query; import com.facebook.presto.sql.tree.QuerySpecification; import com.facebook.presto.sql.tree.Relation; @@ -48,7 +51,10 @@ import com.facebook.presto.sql.tree.Union; import com.facebook.presto.sql.tree.Unnest; import com.facebook.presto.sql.tree.Values; +import com.facebook.presto.sql.tree.With; +import com.facebook.presto.sql.tree.WithQuery; import com.google.common.collect.HashMultimap; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.util.HashMap; @@ -57,28 +63,29 @@ import java.util.Map.Entry; import java.util.Set; +import static com.facebook.presto.spi.StandardWarningCode.UTILIZED_COLUMN_ANALYSIS_FAILED; +import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Sets.intersection; -import static java.lang.String.format; /** * Finds all utilized columns in the query. Utilized columns are those that would have an "impact" on the query's results. - * + *
* For example, in the query: - * SELECT nationkey FROM (SELECT * FROM nation WHERE name = 'USA') + * SELECT nationkey FROM (SELECT * FROM nation WHERE name = 'USA') * Even though all the columns in table nation are referenced by the query (in the SELECT * part), only the columns * "name" and "nationkey" have an "impact" on the query's results. - * + *
* The high-level algorithm works as follows: * 1. Find all fields referenced in all clauses of the outermost SELECT query, and add them to an explore list. * 2. For each field reference F in the explore list, find its referenced relation R. * 3. If R is a SELECT query: - * a. Find the SELECT item expression that F references. Add all fields referenced by that expression to the explore list. - * b. Add all fields referenced by every other clause of the SELECT query to the explore list. + * a. Find the SELECT item expression that F references. Add all fields referenced by that expression to the explore list. + * b. Add all fields referenced by every other clause of the SELECT query to the explore list. * 4. Otherwise, - * a. Add F's referenced field to a referenced fields list. - * b. For each child of R, find the corresponding child of F, and add it to the explore list. + * a. Add F's referenced field to a referenced fields list. + * b. For each child of R, find the corresponding child of F, and add it to the explore list. * 5. Repeat from step 2 for all fields in the explore list, until all have been resolved to a base table relation. - * + *
* The referenced fields list at the end of this algorithm will contain all the columns referenced by the query, that impact the output.
* Step 3a is where fields that do not impact the output are pruned.
*/
@@ -88,14 +95,16 @@ public class UtilizedColumnsAnalyzer
private final Analysis analysis;
- public static void analyzeForUtilizedColumns(Analysis analysis, Node node)
+ public static void analyzeForUtilizedColumns(Analysis analysis, Node node, WarningCollector warningCollector)
{
UtilizedColumnsAnalyzer analyzer = new UtilizedColumnsAnalyzer(analysis);
try {
analyzer.analyze(node);
}
catch (Exception e) {
- LOG.debug(e, format("Error in analyzing utilized columns, falling back to access control on all columns: %s", analysis.getStatement()));
+ warningCollector.add(new PrestoWarning(
+ UTILIZED_COLUMN_ANALYSIS_FAILED,
+ "Error in analyzing utilized columns for access control, falling back to checking access on all columns: " + e.getMessage()));
analysis.getTableColumnReferences().forEach(analysis::addUtilizedTableColumnReferences);
}
}
@@ -271,6 +280,23 @@ protected Void visitQuerySpecification(QuerySpecification querySpec, Context con
return null;
}
+ @Override
+ protected Void visitWith(With node, Context context)
+ {
+ ImmutableList.copyOf(node.getQueries()).reverse().forEach(query -> process(query, context));
+
+ return null;
+ }
+
+ @Override
+ protected Void visitWithQuery(WithQuery withQuery, Context context)
+ {
+ context.copyFieldIdsToExploreForWithQuery(withQuery);
+ process(withQuery.getQuery(), context);
+
+ return null;
+ }
+
@Override
protected Void visitSampledRelation(SampledRelation sampledRelation, Context context)
{
@@ -493,5 +519,22 @@ private void addFieldIdToExplore(FieldId fieldId)
{
fieldsToExplore.put(fieldId.getRelationId(), fieldId);
}
+
+ // Associate the relation from the with clause with the fieldIdsToExplore that we collected for it
+ // when processing the main part of the query
+ public void copyFieldIdsToExploreForWithQuery(WithQuery withQuery)
+ {
+ QualifiedName name = QualifiedName.of(withQuery.getName().getValue());
+ List