Merge remote-tracking branch 'origin/main' into partiql-eval

CHANGELOG.md

@@ -28,6 +28,8 @@ Thank you to all who have contributed!
 ## [Unreleased]
 
 ### Added
+- Adds the ability to define a user-defined-function in ConnectorMetadata
+- Move ConnectorMetadata map from PartiQLPlanner to PartiQLPlanner.Session for planner re-use.
 
 ### Changed
 
@@ -41,7 +43,7 @@ Thank you to all who have contributed!
 
 ### Contributors
 Thank you to all who have contributed!
-- @<your-username>
+- @rchowell
 
 ## [0.14.0-alpha] - 2023-12-15
 

partiql-cli/src/main/kotlin/org/partiql/cli/Main.kt

@@ -52,12 +52,8 @@ object Debug {
 
     private val root = Paths.get(System.getProperty("user.home")).resolve(".partiql/local")
 
-    private val planner = PartiQLPlanner.builder()
-        .catalogs(
-            "local" to LocalConnector.Metadata(root)
-        )
-        .build()
     private val parser = PartiQLParser.default()
+    private val planner = PartiQLPlanner.default()
 
     // !!
     // IMPLEMENT DEBUG BEHAVIOR HERE
@@ -76,6 +72,9 @@ object Debug {
         val sess = PartiQLPlanner.Session(
             queryId = UUID.randomUUID().toString(),
             userId = "debug",
+            catalogs = mapOf(
+                "local" to LocalConnector.Metadata(root)
+            )
         )
         val result = planner.plan(statement, sess).plan
         out.info("-- Plan ----------")

partiql-planner/src/main/kotlin/org/partiql/planner/PartiQLPlanner.kt

@@ -4,6 +4,7 @@ import org.partiql.ast.Statement
 import org.partiql.errors.Problem
 import org.partiql.errors.ProblemCallback
 import org.partiql.plan.PartiQLPlan
+import org.partiql.spi.connector.ConnectorMetadata
 import java.time.Instant
 
 /**
@@ -45,6 +46,7 @@ public interface PartiQLPlanner {
         public val userId: String,
         public val currentCatalog: String? = null,
         public val currentDirectory: List<String> = emptyList(),
+        public val catalogs: Map<String, ConnectorMetadata> = emptyMap(),
         public val instant: Instant = Instant.now(),
     )
 

partiql-planner/src/main/kotlin/org/partiql/planner/PartiQLPlannerBuilder.kt

@@ -1,51 +1,41 @@
 package org.partiql.planner
 
-import org.partiql.spi.connector.ConnectorMetadata
-
 /**
  * PartiQLPlannerBuilder is used to programmatically construct a [PartiQLPlanner] implementation.
  *
  * Usage:
  *      PartiQLPlanner.builder()
- *                    .addCatalog("foo", FooConnector())
- *                    .addCatalog("bar", BarConnector())
- *                    .builder()
+ *                    .addPass(myPass)
+ *                    .build()
  */
 public class PartiQLPlannerBuilder {
 
-    private var headers: MutableList<Header> = mutableListOf(PartiQLHeader)
-    private var catalogs: MutableMap<String, ConnectorMetadata> = mutableMapOf()
-    private var passes: List<PartiQLPlannerPass> = emptyList()
+    private val passes: MutableList<PartiQLPlannerPass> = mutableListOf()
 
     /**
      * Build the builder, return an implementation of a [PartiQLPlanner].
      *
      * @return
      */
-    public fun build(): PartiQLPlanner = PartiQLPlannerDefault(headers, catalogs, passes)
+    public fun build(): PartiQLPlanner = PartiQLPlannerDefault(passes)
 
     /**
-     * Java style method for assigning a Catalog name to [ConnectorMetadata].
+     * Java style method for adding a planner pass to this planner builder.
      *
-     * @param catalog
-     * @param metadata
+     * @param pass
      * @return
      */
-    public fun addCatalog(catalog: String, metadata: ConnectorMetadata): PartiQLPlannerBuilder = this.apply {
-        this.catalogs[catalog] = metadata
+    public fun addPass(pass: PartiQLPlannerPass): PartiQLPlannerBuilder = this.apply {
+        this.passes.add(pass)
     }
 
     /**
-     * Kotlin style method for assigning Catalog names to [ConnectorMetadata].
+     * Kotlin style method for adding a list of planner passes to this planner builder.
      *
-     * @param catalogs
+     * @param passes
      * @return
      */
-    public fun catalogs(vararg catalogs: Pair<String, ConnectorMetadata>): PartiQLPlannerBuilder = this.apply {
-        this.catalogs = mutableMapOf(*catalogs)
-    }
-
-    public fun passes(passes: List<PartiQLPlannerPass>): PartiQLPlannerBuilder = this.apply {
-        this.passes = passes
+    public fun addPasses(vararg passes: PartiQLPlannerPass): PartiQLPlannerBuilder = this.apply {
+        this.passes.addAll(passes)
     }
 }

partiql-planner/src/main/kotlin/org/partiql/planner/PartiQLPlannerDefault.kt

@@ -8,14 +8,11 @@ import org.partiql.planner.internal.ir.PartiQLVersion
 import org.partiql.planner.internal.transforms.AstToPlan
 import org.partiql.planner.internal.transforms.PlanTransform
 import org.partiql.planner.internal.typer.PlanTyper
-import org.partiql.spi.connector.ConnectorMetadata
 
 /**
  * Default PartiQL logical query planner.
  */
 internal class PartiQLPlannerDefault(
-    private val headers: List<Header>,
-    private val catalogs: Map<String, ConnectorMetadata>,
     private val passes: List<PartiQLPlannerPass>,
 ) : PartiQLPlanner {
 
@@ -24,8 +21,9 @@ internal class PartiQLPlannerDefault(
         session: PartiQLPlanner.Session,
         onProblem: ProblemCallback,
     ): PartiQLPlanner.Result {
+
         // 0. Initialize the planning environment
-        val env = Env(headers, catalogs, session)
+        val env = Env(session)
 
         // 1. Normalize
         val ast = statement.normalize()

partiql-planner/src/main/kotlin/org/partiql/planner/internal/Env.kt

@@ -1,6 +1,5 @@
 package org.partiql.planner.internal
 
-import org.partiql.planner.Header
 import org.partiql.planner.PartiQLPlanner
 import org.partiql.planner.internal.ir.Agg
 import org.partiql.planner.internal.ir.Catalog
@@ -20,6 +19,7 @@ import org.partiql.spi.connector.ConnectorSession
 import org.partiql.types.StaticType
 import org.partiql.types.StructType
 import org.partiql.types.TupleConstraint
+import org.partiql.types.function.FunctionSignature
 
 /**
  * Handle for associating a catalog name with catalog related metadata objects.
@@ -77,16 +77,18 @@ internal sealed interface ResolvedVar {
      *
      * @property type              Resolved StaticType
      * @property ordinal           Index offset in [TypeEnv]
-     * @property replacementSteps  Path steps to replace.
-     * @property depth             The depth/level of the path match.
+     * @property resolvedSteps     The fully resolved path steps.s
      */
     class Local(
         override val type: StaticType,
         override val ordinal: Int,
         val rootType: StaticType,
-        val replacementSteps: List<BindingName>,
-        override val depth: Int
-    ) : ResolvedVar
+        val resolvedSteps: List<BindingName>,
+    ) : ResolvedVar {
+        // the depth are always going to be 1 because this is local variable.
+        // the global path, however the path length maybe, going to be replaced by a binding name.
+        override val depth: Int = 1
+    }
 
     /**
      * Metadata for a resolved global variable
@@ -100,7 +102,7 @@ internal sealed interface ResolvedVar {
         override val type: StaticType,
         override val ordinal: Int,
         override val depth: Int,
-        val position: Int
+        val position: Int,
     ) : ResolvedVar
 }
 
@@ -120,13 +122,9 @@ internal enum class ResolutionStrategy {
 /**
  * PartiQL Planner Global Environment of Catalogs backed by given plugins.
  *
- * @property headers        List of namespaced definitions
- * @property catalogs       List of plugins for global resolution
  * @property session        Session details
  */
 internal class Env(
-    private val headers: List<Header>,
-    private val connectors: Map<String, ConnectorMetadata>,
     private val session: PartiQLPlanner.Session,
 ) {
 
@@ -135,24 +133,44 @@ internal class Env(
      */
     public val catalogs = mutableListOf<Catalog>()
 
+    /**
+     * Catalog Metadata for this query session.
+     */
+    private val connectors = session.catalogs
+
     /**
      * Encapsulate all function resolving logic within [FnResolver].
+     *
+     * TODO we should be using a search_path for resolving functions. This is not possible at the moment, so we flatten
+     *      all builtin functions to live at the top-level. At the moment, we could technically use this to have
+     *      single-level `catalog`.`function`() syntax but that is out-of-scope for this commit.
      */
-    public val fnResolver = FnResolver(headers)
+    public val fnResolver = FnResolver(object : Header() {
+
+        override val namespace: String = "builtins"
+
+        override val functions: List<FunctionSignature.Scalar> =
+            PartiQLHeader.functions + connectors.values.flatMap { it.functions }
+
+        override val operators: List<FunctionSignature.Scalar> =
+            PartiQLHeader.operators + connectors.values.flatMap { it.operators }
+
+        override val aggregations: List<FunctionSignature.Aggregation> =
+            PartiQLHeader.aggregations + connectors.values.flatMap { it.aggregations }
+    })
 
     private val connectorSession = object : ConnectorSession {
         override fun getQueryId(): String = session.queryId
         override fun getUserId(): String = session.userId
     }
 
     /**
-
-     * Leverages a [FunctionResolver] to find a matching function defined in the [Header] scalar function catalog.
+     * Leverages a [FnResolver] to find a matching function defined in the [Header] scalar function catalog.
      */
     internal fun resolveFn(fn: Fn.Unresolved, args: List<Rex>) = fnResolver.resolveFn(fn, args)
 
     /**
-     * Leverages a [FunctionResolver] to find a matching function defined in the [Header] aggregation function catalog.
+     * Leverages a [FnResolver] to find a matching function defined in the [Header] aggregation function catalog.
      */
     internal fun resolveAgg(agg: Agg.Unresolved, args: List<Rex>) = fnResolver.resolveAgg(agg, args)
 
@@ -211,7 +229,11 @@ internal class Env(
             getObjectHandle(cat, catalogPath)?.let { handle ->
                 getObjectDescriptor(handle).let { type ->
                     val depth = calculateMatched(originalPath, catalogPath, handle.second.absolutePath)
-                    val (catalogIndex, valueIndex) = getOrAddCatalogValue(handle.first, handle.second.absolutePath.steps, type)
+                    val (catalogIndex, valueIndex) = getOrAddCatalogValue(
+                        handle.first,
+                        handle.second.absolutePath.steps,
+                        type
+                    )
                     // Return resolution metadata
                     ResolvedVar.Global(type, catalogIndex, depth, valueIndex)
                 }
@@ -222,7 +244,11 @@ internal class Env(
     /**
      * @return a [Pair] where [Pair.first] is the catalog index and [Pair.second] is the value index within that catalog
      */
-    private fun getOrAddCatalogValue(catalogName: String, valuePath: List<String>, valueType: StaticType): Pair<Int, Int> {
+    private fun getOrAddCatalogValue(
+        catalogName: String,
+        valuePath: List<String>,
+        valueType: StaticType,
+    ): Pair<Int, Int> {
         val catalogIndex = getOrAddCatalog(catalogName)
         val symbols = catalogs[catalogIndex].symbols
         return symbols.indexOfFirst { value ->
@@ -233,7 +259,7 @@ internal class Env(
                     catalogs[catalogIndex] = catalogs[catalogIndex].copy(
                         symbols = symbols + listOf(Catalog.Symbol(valuePath, valueType))
                     )
-                    catalogIndex to 0
+                    catalogIndex to catalogs[catalogIndex].symbols.lastIndex
                 }
                 else -> {
                     catalogIndex to index
@@ -325,19 +351,25 @@ internal class Env(
         locals.forEachIndexed { ordinal, binding ->
             val root = path.steps[0]
             if (root.isEquivalentTo(binding.name)) {
-                return ResolvedVar.Local(binding.type, ordinal, binding.type, emptyList(), 1)
+                return ResolvedVar.Local(binding.type, ordinal, binding.type, path.steps)
             }
         }
 
         // 2. Check if this variable is referencing a struct field, carrying ordinals
         val matches = mutableListOf<ResolvedVar.Local>()
         for (ordinal in locals.indices) {
             val rootType = locals[ordinal].type
+            val pathPrefix = BindingName(locals[ordinal].name, BindingCase.SENSITIVE)
             if (rootType is StructType) {
                 val varType = inferStructLookup(rootType, path)
                 if (varType != null) {
                     // we found this path within a struct!
-                    val match = ResolvedVar.Local(varType.resolvedType, ordinal, rootType, varType.replacementPath.steps, varType.replacementPath.steps.size)
+                    val match = ResolvedVar.Local(
+                        varType.resolvedType,
+                        ordinal,
+                        rootType,
+                        listOf(pathPrefix) + varType.replacementPath.steps,
+                    )
                     matches.add(match)
                 }
             }
@@ -380,7 +412,7 @@ internal class Env(
      */
     private class ResolvedPath(
         val replacementPath: BindingPath,
-        val resolvedType: StaticType
+        val resolvedType: StaticType,
     )
 
     /**
@@ -413,7 +445,7 @@ internal class Env(
                 }
             }
             // 3. Struct is open
-            else -> null
+            else -> key to StaticType.ANY
         }
     }
 

partiql-planner/src/main/kotlin/org/partiql/planner/Header.kt → partiql-planner/src/main/kotlin/org/partiql/planner/internal/Header.kt

@@ -1,4 +1,4 @@
-package org.partiql.planner
+package org.partiql.planner.internal
 
 import org.partiql.planner.internal.typer.TypeLattice
 import org.partiql.types.function.FunctionParameter

partiql-planner/src/main/kotlin/org/partiql/planner/PartiQLHeader.kt → partiql-planner/src/main/kotlin/org/partiql/planner/internal/PartiQLHeader.kt

@@ -1,4 +1,4 @@
-package org.partiql.planner
+package org.partiql.planner.internal
 
 import org.partiql.ast.DatetimeField
 import org.partiql.types.function.FunctionParameter

partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/FnResolver.kt

@@ -1,6 +1,6 @@
 package org.partiql.planner.internal.typer
 
-import org.partiql.planner.Header
+import org.partiql.planner.internal.Header
 import org.partiql.planner.internal.ir.Agg
 import org.partiql.planner.internal.ir.Fn
 import org.partiql.planner.internal.ir.Identifier
@@ -109,7 +109,7 @@ internal sealed class FnMatch<T : FunctionSignature> {
  * at the moment to keep that information (derived from the current TypeLattice) with the [FnResolver].
  */
 @OptIn(PartiQLValueExperimental::class)
-internal class FnResolver(private val headers: List<Header>) {
+internal class FnResolver(private val header: Header) {
 
     /**
      * All headers use the same type lattice (we don't have a design for plugging type systems at the moment).
@@ -140,10 +140,10 @@ internal class FnResolver(private val headers: List<Header>) {
         val (casts, unsafeCasts) = casts()
         unsafeCastSet = unsafeCasts
         // combine all header definitions
-        val fns = headers.flatMap { it.functions }
+        val fns = header.functions
         functions = fns.toFnMap()
-        operators = (headers.flatMap { it.operators } + casts).toFnMap()
-        aggregations = headers.flatMap { it.aggregations }.toFnMap()
+        operators = (header.operators + casts).toFnMap()
+        aggregations = header.aggregations.toFnMap()
     }
 
     /**