Skip to content

Commit f54c498

Browse files
pdabre12pdabre12
committed
[native] Introduce native function namespace SPI
1 parent 092dd36 commit f54c498

File tree

55 files changed

+1617
-143
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+1617
-143
lines changed

‎pom.xml

+6
Original file line numberDiff line numberDiff line change
@@ -1018,6 +1018,12 @@
10181018
<version>${project.version}</version>
10191019
</dependency>
10201020

1021+
<dependency>
1022+
<groupId>com.facebook.presto</groupId>
1023+
<artifactId>presto-native-sidecar-plugin</artifactId>
1024+
<version>${project.version}</version>
1025+
</dependency>
1026+
10211027
<dependency>
10221028
<groupId>com.facebook.hive</groupId>
10231029
<artifactId>hive-dwrf</artifactId>

‎presto-function-namespace-managers-common/src/main/java/com/facebook/presto/functionNamespace/AbstractSqlInvokedFunctionNamespaceManager.java

+6-2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import com.facebook.presto.common.QualifiedObjectName;
1919
import com.facebook.presto.common.block.BlockEncodingSerde;
2020
import com.facebook.presto.common.function.SqlFunctionResult;
21+
import com.facebook.presto.common.type.Type;
2122
import com.facebook.presto.common.type.TypeManager;
2223
import com.facebook.presto.common.type.UserDefinedType;
2324
import com.facebook.presto.functionNamespace.execution.SqlFunctionExecutors;
@@ -192,7 +193,7 @@ public Optional<UserDefinedType> getUserDefinedType(QualifiedObjectName typeName
192193
}
193194

194195
@Override
195-
public final FunctionHandle getFunctionHandle(Optional<? extends FunctionNamespaceTransactionHandle> transactionHandle, Signature signature)
196+
public FunctionHandle getFunctionHandle(Optional<? extends FunctionNamespaceTransactionHandle> transactionHandle, Signature signature)
196197
{
197198
checkCatalog(signature.getName());
198199
// This is the only assumption in this class that we're dealing with sql-invoked regular function.
@@ -363,10 +364,13 @@ protected AggregationFunctionImplementation sqlInvokedFunctionToAggregationImple
363364
"Need aggregationMetadata to get aggregation function implementation");
364365

365366
AggregationFunctionMetadata aggregationMetadata = function.getAggregationMetadata().get();
367+
List<Type> parameters = function.getSignature().getArgumentTypes().stream().map(
368+
(typeManager::getType)).collect(toImmutableList());
366369
return new SqlInvokedAggregationFunctionImplementation(
367370
typeManager.getType(aggregationMetadata.getIntermediateType()),
368371
typeManager.getType(function.getSignature().getReturnType()),
369-
aggregationMetadata.isOrderSensitive());
372+
aggregationMetadata.isOrderSensitive(),
373+
parameters);
370374
default:
371375
throw new IllegalStateException(format("Unknown function implementation type: %s", implementationType));
372376
}

‎presto-function-namespace-managers/src/main/java/com/facebook/presto/functionNamespace/rest/RestBasedFunctionNamespaceManager.java

+1
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ private SqlInvokedFunction createSqlInvokedFunction(String functionName, JsonBas
143143
jsonBasedUdfFunctionMetaData.getDocString(),
144144
jsonBasedUdfFunctionMetaData.getRoutineCharacteristics(),
145145
"",
146+
jsonBasedUdfFunctionMetaData.getVariableArity(),
146147
functionVersion,
147148
jsonBasedUdfFunctionMetaData.getFunctionKind(),
148149
functionId,

‎presto-hive-function-namespace/src/main/java/com/facebook/presto/hive/functions/HiveFunctionHandle.java

+8
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,14 @@
1515
package com.facebook.presto.hive.functions;
1616

1717
import com.facebook.presto.common.CatalogSchemaName;
18+
import com.facebook.presto.common.type.TypeSignature;
1819
import com.facebook.presto.spi.function.FunctionHandle;
1920
import com.facebook.presto.spi.function.FunctionKind;
2021
import com.facebook.presto.spi.function.Signature;
2122
import com.fasterxml.jackson.annotation.JsonCreator;
2223
import com.fasterxml.jackson.annotation.JsonProperty;
2324

25+
import java.util.List;
2426
import java.util.Objects;
2527

2628
import static java.util.Objects.requireNonNull;
@@ -54,6 +56,12 @@ public FunctionKind getKind()
5456
return signature.getKind();
5557
}
5658

59+
@Override
60+
public List<TypeSignature> getArgumentTypes()
61+
{
62+
return signature.getArgumentTypes();
63+
}
64+
5765
@JsonProperty
5866
public Signature getSignature()
5967
{

‎presto-hive-function-namespace/src/test/java/com/facebook/presto/hive/functions/HiveFunctionsTestUtils.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ public static TestingPrestoServer createTestingPrestoServer()
3737
functionAndTypeManager.loadFunctionNamespaceManager(
3838
"hive-functions",
3939
"hive",
40-
getNamespaceManagerCreationProperties());
40+
getNamespaceManagerCreationProperties(),
41+
server.getPluginNodeManager());
4142
server.refreshNodes();
4243
return server;
4344
}

‎presto-hive-function-namespace/src/test/java/com/facebook/presto/hive/functions/TestHiveAggregationFunctions.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,8 @@ private static TestingPrestoServer createServer()
149149
functionAndTypeManager.loadFunctionNamespaceManager(
150150
"hive-functions",
151151
"hive",
152-
Collections.emptyMap());
152+
Collections.emptyMap(),
153+
server.getPluginNodeManager());
153154
server.refreshNodes();
154155
return server;
155156
}

‎presto-iceberg/src/main/java/com/facebook/presto/iceberg/optimizer/IcebergMetadataOptimizer.java

+13-17
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,6 @@
1313
*/
1414
package com.facebook.presto.iceberg.optimizer;
1515

16-
import com.facebook.presto.common.CatalogSchemaName;
17-
import com.facebook.presto.common.QualifiedObjectName;
1816
import com.facebook.presto.common.predicate.NullableValue;
1917
import com.facebook.presto.common.predicate.TupleDomain;
2018
import com.facebook.presto.common.predicate.TupleDomain.ColumnDomain;
@@ -58,7 +56,6 @@
5856
import com.facebook.presto.spi.relation.VariableReferenceExpression;
5957
import com.google.common.collect.ImmutableList;
6058
import com.google.common.collect.ImmutableMap;
61-
import com.google.common.collect.ImmutableSet;
6259
import com.google.common.collect.Iterables;
6360
import com.google.common.collect.Lists;
6461

@@ -67,7 +64,7 @@
6764
import java.util.List;
6865
import java.util.Map;
6966
import java.util.Optional;
70-
import java.util.Set;
67+
import java.util.function.Predicate;
7168

7269
import static com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT;
7370
import static com.facebook.presto.iceberg.IcebergSessionProperties.getRowsForMetadataOptimizationThreshold;
@@ -81,16 +78,10 @@
8178
public class IcebergMetadataOptimizer
8279
implements ConnectorPlanOptimizer
8380
{
84-
public static final CatalogSchemaName DEFAULT_NAMESPACE = new CatalogSchemaName("presto", "default");
85-
private static final Set<QualifiedObjectName> ALLOWED_FUNCTIONS = ImmutableSet.of(
86-
QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "max"),
87-
QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "min"),
88-
QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "approx_distinct"));
89-
9081
// Min/Max could be folded into LEAST/GREATEST
91-
private static final Map<QualifiedObjectName, QualifiedObjectName> AGGREGATION_SCALAR_MAPPING = ImmutableMap.of(
92-
QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "max"), QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "greatest"),
93-
QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "min"), QualifiedObjectName.valueOf(DEFAULT_NAMESPACE, "least"));
82+
private static final Map<String, String> AGGREGATION_SCALAR_MAPPING = ImmutableMap.of(
83+
"max", "greatest",
84+
"min", "least");
9485

9586
private final FunctionMetadataManager functionMetadataManager;
9687
private final TypeManager typeManager;
@@ -137,6 +128,7 @@ private static class Optimizer
137128
private final RowExpressionService rowExpressionService;
138129
private final StandardFunctionResolution functionResolution;
139130
private final int rowsForMetadataOptimizationThreshold;
131+
private final List<Predicate<FunctionHandle>> allowedFunctionsPredicates;
140132

141133
private Optimizer(ConnectorSession connectorSession,
142134
PlanNodeIdAllocator idAllocator,
@@ -156,15 +148,19 @@ private Optimizer(ConnectorSession connectorSession,
156148
this.functionResolution = functionResolution;
157149
this.typeManager = typeManager;
158150
this.rowsForMetadataOptimizationThreshold = rowsForMetadataOptimizationThreshold;
151+
this.allowedFunctionsPredicates = ImmutableList.of(
152+
functionResolution::isMaxFunction,
153+
functionResolution::isMinFunction,
154+
functionResolution::isApproximateCountDistinctFunction);
159155
}
160156

161157
@Override
162158
public PlanNode visitAggregation(AggregationNode node, RewriteContext<Void> context)
163159
{
164160
// supported functions are only MIN/MAX/APPROX_DISTINCT or distinct aggregates
165161
for (Aggregation aggregation : node.getAggregations().values()) {
166-
QualifiedObjectName functionName = functionMetadataManager.getFunctionMetadata(aggregation.getFunctionHandle()).getName();
167-
if (!ALLOWED_FUNCTIONS.contains(functionName) && !aggregation.isDistinct()) {
162+
if (allowedFunctionsPredicates.stream().noneMatch(
163+
pred -> pred.test(aggregation.getFunctionHandle())) && !aggregation.isDistinct()) {
168164
return context.defaultRewrite(node);
169165
}
170166
}
@@ -270,7 +266,7 @@ private boolean isReducible(AggregationNode node, List<VariableReferenceExpressi
270266
}
271267
for (Aggregation aggregation : node.getAggregations().values()) {
272268
FunctionMetadata functionMetadata = functionMetadataManager.getFunctionMetadata(aggregation.getFunctionHandle());
273-
if (!AGGREGATION_SCALAR_MAPPING.containsKey(functionMetadata.getName()) ||
269+
if (!AGGREGATION_SCALAR_MAPPING.containsKey(functionMetadata.getName().getObjectName()) ||
274270
functionMetadata.getArgumentTypes().size() > 1 ||
275271
!inputs.containsAll(aggregation.getCall().getArguments())) {
276272
return false;
@@ -340,7 +336,7 @@ private RowExpression evaluateMinMax(FunctionMetadata aggregationFunctionMetadat
340336
return new ConstantExpression(Optional.empty(), null, returnType);
341337
}
342338

343-
String scalarFunctionName = AGGREGATION_SCALAR_MAPPING.get(aggregationFunctionMetadata.getName()).getObjectName();
339+
String scalarFunctionName = AGGREGATION_SCALAR_MAPPING.get(aggregationFunctionMetadata.getName().getObjectName());
344340
while (arguments.size() > 1) {
345341
List<RowExpression> reducedArguments = new ArrayList<>();
346342
// We fold for every 100 values because GREATEST/LEAST has argument count limit

‎presto-main/src/main/java/com/facebook/presto/metadata/BuiltInFunctionHandle.java

+8
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,14 @@
1414
package com.facebook.presto.metadata;
1515

1616
import com.facebook.presto.common.CatalogSchemaName;
17+
import com.facebook.presto.common.type.TypeSignature;
1718
import com.facebook.presto.spi.function.FunctionHandle;
1819
import com.facebook.presto.spi.function.FunctionKind;
1920
import com.facebook.presto.spi.function.Signature;
2021
import com.fasterxml.jackson.annotation.JsonCreator;
2122
import com.fasterxml.jackson.annotation.JsonProperty;
2223

24+
import java.util.List;
2325
import java.util.Objects;
2426

2527
import static java.util.Objects.requireNonNull;
@@ -54,6 +56,12 @@ public FunctionKind getKind()
5456
return signature.getKind();
5557
}
5658

59+
@Override
60+
public List<TypeSignature> getArgumentTypes()
61+
{
62+
return signature.getArgumentTypes();
63+
}
64+
5765
@Override
5866
public CatalogSchemaName getCatalogSchemaName()
5967
{

‎presto-main/src/main/java/com/facebook/presto/metadata/BuiltInTypeAndFunctionNamespaceManager.java

+4-45
Original file line numberDiff line numberDiff line change
@@ -236,7 +236,6 @@
236236
import com.facebook.presto.spi.function.SqlInvokedFunction;
237237
import com.facebook.presto.spi.function.SqlInvokedScalarFunctionImplementation;
238238
import com.facebook.presto.sql.analyzer.FunctionsConfig;
239-
import com.facebook.presto.sql.analyzer.TypeSignatureProvider;
240239
import com.facebook.presto.type.BigintOperators;
241240
import com.facebook.presto.type.BooleanOperators;
242241
import com.facebook.presto.type.CharOperators;
@@ -345,7 +344,6 @@
345344
import static com.facebook.presto.geospatial.SphericalGeographyType.SPHERICAL_GEOGRAPHY;
346345
import static com.facebook.presto.geospatial.type.BingTileType.BING_TILE;
347346
import static com.facebook.presto.geospatial.type.GeometryType.GEOMETRY;
348-
import static com.facebook.presto.metadata.SignatureBinder.applyBoundVariables;
349347
import static com.facebook.presto.operator.aggregation.AlternativeArbitraryAggregationFunction.ALTERNATIVE_ANY_VALUE_AGGREGATION;
350348
import static com.facebook.presto.operator.aggregation.AlternativeArbitraryAggregationFunction.ALTERNATIVE_ARBITRARY_AGGREGATION;
351349
import static com.facebook.presto.operator.aggregation.AlternativeMaxAggregationFunction.ALTERNATIVE_MAX;
@@ -461,7 +459,6 @@
461459
import static com.facebook.presto.spi.function.FunctionKind.SCALAR;
462460
import static com.facebook.presto.spi.function.FunctionKind.WINDOW;
463461
import static com.facebook.presto.spi.function.SqlFunctionVisibility.HIDDEN;
464-
import static com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypeSignatures;
465462
import static com.facebook.presto.sql.planner.LiteralEncoder.MAGIC_LITERAL_FUNCTION_PREFIX;
466463
import static com.facebook.presto.type.ArrayParametricType.ARRAY;
467464
import static com.facebook.presto.type.CodePointsType.CODE_POINTS;
@@ -521,7 +518,6 @@
521518
import static com.facebook.presto.type.Re2JRegexpType.RE2J_REGEXP;
522519
import static com.facebook.presto.type.RowParametricType.ROW;
523520
import static com.facebook.presto.type.SfmSketchType.SFM_SKETCH;
524-
import static com.facebook.presto.type.TypeUtils.resolveTypes;
525521
import static com.facebook.presto.type.khyperloglog.KHyperLogLogType.K_HYPER_LOG_LOG;
526522
import static com.facebook.presto.type.setdigest.SetDigestType.SET_DIGEST;
527523
import static com.google.common.base.Preconditions.checkArgument;
@@ -1343,47 +1339,11 @@ private SpecializedFunctionKey getSpecializedFunctionKey(Signature signature)
13431339

13441340
private SpecializedFunctionKey doGetSpecializedFunctionKey(Signature signature)
13451341
{
1346-
Iterable<SqlFunction> candidates = getFunctions(null, signature.getName());
1347-
// search for exact match
1348-
Type returnType = functionAndTypeManager.getType(signature.getReturnType());
1349-
List<TypeSignatureProvider> argumentTypeSignatureProviders = fromTypeSignatures(signature.getArgumentTypes());
1350-
for (SqlFunction candidate : candidates) {
1351-
Optional<BoundVariables> boundVariables = new SignatureBinder(functionAndTypeManager, candidate.getSignature(), false)
1352-
.bindVariables(argumentTypeSignatureProviders, returnType);
1353-
if (boundVariables.isPresent()) {
1354-
return new SpecializedFunctionKey(candidate, boundVariables.get(), argumentTypeSignatureProviders.size());
1355-
}
1356-
}
1357-
1358-
// TODO: hack because there could be "type only" coercions (which aren't necessarily included as implicit casts),
1359-
// so do a second pass allowing "type only" coercions
1360-
List<Type> argumentTypes = resolveTypes(signature.getArgumentTypes(), functionAndTypeManager);
1361-
for (SqlFunction candidate : candidates) {
1362-
SignatureBinder binder = new SignatureBinder(functionAndTypeManager, candidate.getSignature(), true);
1363-
Optional<BoundVariables> boundVariables = binder.bindVariables(argumentTypeSignatureProviders, returnType);
1364-
if (!boundVariables.isPresent()) {
1365-
continue;
1366-
}
1367-
Signature boundSignature = applyBoundVariables(candidate.getSignature(), boundVariables.get(), argumentTypes.size());
1368-
1369-
if (!functionAndTypeManager.isTypeOnlyCoercion(functionAndTypeManager.getType(boundSignature.getReturnType()), returnType)) {
1370-
continue;
1371-
}
1372-
boolean nonTypeOnlyCoercion = false;
1373-
for (int i = 0; i < argumentTypes.size(); i++) {
1374-
Type expectedType = functionAndTypeManager.getType(boundSignature.getArgumentTypes().get(i));
1375-
if (!functionAndTypeManager.isTypeOnlyCoercion(argumentTypes.get(i), expectedType)) {
1376-
nonTypeOnlyCoercion = true;
1377-
break;
1378-
}
1379-
}
1380-
if (nonTypeOnlyCoercion) {
1381-
continue;
1382-
}
1383-
1384-
return new SpecializedFunctionKey(candidate, boundVariables.get(), argumentTypes.size());
1385-
}
1342+
return functionAndTypeManager.getSpecializedFunctionKey(signature);
1343+
}
13861344

1345+
public SpecializedFunctionKey doGetSpecializedFunctionKeyForMagicLiteralFunctions(Signature signature, FunctionAndTypeManager functionAndTypeManager)
1346+
{
13871347
// TODO: this is a hack and should be removed
13881348
if (signature.getNameSuffix().startsWith(MAGIC_LITERAL_FUNCTION_PREFIX)) {
13891349
List<TypeSignature> parameterTypes = signature.getArgumentTypes();
@@ -1406,7 +1366,6 @@ private SpecializedFunctionKey doGetSpecializedFunctionKey(Signature signature)
14061366
.build(),
14071367
1);
14081368
}
1409-
14101369
throw new PrestoException(FUNCTION_IMPLEMENTATION_MISSING, format("%s not found", signature));
14111370
}
14121371

0 commit comments

Comments
 (0)