Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Partiql eval wildcard #1374

Merged
merged 9 commits into from
Mar 7, 2024
Merged

Partiql eval wildcard #1374

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
3524ca0
wildcard
yliuuuu Feb 15, 2024
7fd5966
roll back scan modeling
yliuuuu Feb 20, 2024
8f585c1
rebase
yliuuuu Feb 20, 2024
28aec15
rebase clean up
yliuuuu Feb 20, 2024
4ec9ff8
address comments
yliuuuu Feb 26, 2024
4a6c961
rebase
yliuuuu Feb 26, 2024
c71c11f
address feedback
yliuuuu Feb 28, 2024
490c708
add test cases
yliuuuu Feb 29, 2024
5d676bb
flatten
yliuuuu Feb 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 105 additions & 0 deletions partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1289,4 +1289,109 @@ class PartiQLEngineDefaultTest {
""".trimIndent(),
expected = boolValue(false)
).assert()

@Test
// TODO: Add to conformance tests
fun wildCard() =
SuccessTestCase(
input = """
[
{ 'id':'5',
'books':[
{ 'title':'A',
'price':5.0,
'authors': [{'name': 'John'}, {'name': 'Doe'}]
},
{ 'title':'B',
'price':2.0,
'authors': [{'name': 'Zoe'}, {'name': 'Bill'}]
}
]
},
{ 'id':'6',
'books':[
{ 'title':'A',
'price':5.0,
'authors': [{'name': 'John'}, {'name': 'Doe'}]
},
{ 'title':'E',
'price':2.0,
'authors': [{'name': 'Zoe'}, {'name': 'Bill'}]
}
]
},
{ 'id':7,
'books':[]
}
][*].books[*].authors[*].name
""".trimIndent(),
expected = bagValue(
listOf(
stringValue("John"), stringValue("Doe"), stringValue("Zoe"), stringValue("Bill"),
stringValue("John"), stringValue("Doe"), stringValue("Zoe"), stringValue("Bill")
)
)
).assert()

@Test
// TODO: add to conformance tests
// Note that the existing pipeline produced identical result when supplying with
// SELECT VALUE v2.name FROM e as v0, v0.books as v1, unpivot v1.authors as v2;
// But it produces different result when supplying with e[*].books[*].authors.*
// <<
// <<{ 'name': 'John'},{'name': 'Doe'} >>,
// ...
// >>
fun unpivot() =
SuccessTestCase(
input = """
[
{ 'id':'5',
'books':[
{ 'title':'A',
'price':5.0,
'authors': {
'first': {'name': 'John'},
'second': {'name': 'Doe'}
}
},
{ 'title':'B',
'price':2.0,
'authors': {
'first': {'name': 'Zoe'},
'second': {'name': 'Bill'}
}
}
]
},
{ 'id':'6',
'books':[
{ 'title':'A',
'price':5.0,
'authors': {
'first': {'name': 'John'},
'second': {'name': 'Doe'}
}
},
{ 'title':'E',
'price':2.0,
'authors': {
'first': {'name': 'Zoe'},
'second': {'name': 'Bill'}
}
}
]
},
{ 'id':7,
'books':[]
}
][*].books[*].authors.*.name
""".trimIndent(),
expected = bagValue(
listOf(
stringValue("John"), stringValue("Doe"), stringValue("Zoe"), stringValue("Bill"),
stringValue("John"), stringValue("Doe"), stringValue("Zoe"), stringValue("Bill")
)
)
).assert()
}
1 change: 0 additions & 1 deletion partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/RelConverter.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,6 @@ internal object RelConverter {
}

// Helpers

private fun convertScan(rex: Rex, binding: Rel.Binding): Rel {
val schema = listOf(binding)
val props = emptySet<Rel.Prop>()
Expand Down
177 changes: 149 additions & 28 deletions partiql-planner/src/main/kotlin/org/partiql/planner/internal/transforms/RexConverter.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,17 @@ import org.partiql.ast.Type
import org.partiql.ast.visitor.AstBaseVisitor
import org.partiql.planner.internal.Env
import org.partiql.planner.internal.ir.Identifier
import org.partiql.planner.internal.ir.Rel
import org.partiql.planner.internal.ir.Rex
import org.partiql.planner.internal.ir.builder.plan
import org.partiql.planner.internal.ir.identifierQualified
import org.partiql.planner.internal.ir.identifierSymbol
import org.partiql.planner.internal.ir.rel
import org.partiql.planner.internal.ir.relBinding
import org.partiql.planner.internal.ir.relOpJoin
import org.partiql.planner.internal.ir.relOpScan
import org.partiql.planner.internal.ir.relOpUnpivot
import org.partiql.planner.internal.ir.relType
import org.partiql.planner.internal.ir.rex
import org.partiql.planner.internal.ir.rexOpCallUnresolved
import org.partiql.planner.internal.ir.rexOpCastUnresolved
Expand All @@ -36,17 +43,20 @@ import org.partiql.planner.internal.ir.rexOpLit
import org.partiql.planner.internal.ir.rexOpPathIndex
import org.partiql.planner.internal.ir.rexOpPathKey
import org.partiql.planner.internal.ir.rexOpPathSymbol
import org.partiql.planner.internal.ir.rexOpSelect
import org.partiql.planner.internal.ir.rexOpStruct
import org.partiql.planner.internal.ir.rexOpStructField
import org.partiql.planner.internal.ir.rexOpSubquery
import org.partiql.planner.internal.ir.rexOpTupleUnion
import org.partiql.planner.internal.ir.rexOpVarLocal
import org.partiql.planner.internal.ir.rexOpVarUnresolved
import org.partiql.planner.internal.typer.toNonNullStaticType
import org.partiql.planner.internal.typer.toStaticType
import org.partiql.types.StaticType
import org.partiql.value.PartiQLValueExperimental
import org.partiql.value.PartiQLValueType
import org.partiql.value.StringValue
import org.partiql.value.boolValue
import org.partiql.value.int32Value
import org.partiql.value.int64Value
import org.partiql.value.io.PartiQLValueIonReaderBuilder
Expand Down Expand Up @@ -249,41 +259,152 @@ internal object RexConverter {
else -> root to node.steps
}

// Return wrapped path
return when (newSteps.isEmpty()) {
true -> newRoot
false -> newSteps.fold(newRoot) { current, step ->
val path = when (step) {
is Expr.Path.Step.Index -> {
val key = visitExprCoerce(step.key, context)
when (val astKey = step.key) {
is Expr.Lit -> when (astKey.value) {
is StringValue -> rexOpPathKey(current, key)
else -> rexOpPathIndex(current, key)
}
is Expr.Cast -> when (astKey.asType is Type.String) {
true -> rexOpPathKey(current, key)
false -> rexOpPathIndex(current, key)
}
if (newSteps.isEmpty()) {
return newRoot
}

val fromList = mutableListOf<Rel>()

var varRefIndex = 0 // tracking var ref index

val pathNavi = newSteps.fold(newRoot) { current, step ->
val path = when (step) {
is Expr.Path.Step.Index -> {
val key = visitExprCoerce(step.key, context)
val op = when (val astKey = step.key) {
is Expr.Lit -> when (astKey.value) {
is StringValue -> rexOpPathKey(current, key)
else -> rexOpPathIndex(current, key)
}
}
is Expr.Path.Step.Symbol -> {
val identifier = AstToPlan.convert(step.symbol)
when (identifier.caseSensitivity) {
Identifier.CaseSensitivity.SENSITIVE -> rexOpPathKey(
current,
rexString(identifier.symbol)
)
Identifier.CaseSensitivity.INSENSITIVE -> rexOpPathSymbol(current, identifier.symbol)

is Expr.Cast -> when (astKey.asType is Type.String) {
true -> rexOpPathKey(current, key)
false -> rexOpPathIndex(current, key)
}

else -> rexOpPathIndex(current, key)
}
is Expr.Path.Step.Unpivot -> error("Unpivot path not supported yet")
is Expr.Path.Step.Wildcard -> error("Wildcard path not supported yet")
op
}

is Expr.Path.Step.Symbol -> {
val identifier = AstToPlan.convert(step.symbol)
val op = when (identifier.caseSensitivity) {
Identifier.CaseSensitivity.SENSITIVE -> rexOpPathKey(
current,
rexString(identifier.symbol)
)

Identifier.CaseSensitivity.INSENSITIVE -> rexOpPathSymbol(current, identifier.symbol)
}
op
}

// Unpivot and Wildcard steps trigger the rewrite
// According to spec Section 4.3
// ew1p1...wnpn
// rewrite to:
// SELECT VALUE v_n.p_n
// FROM
// u_1 e as v_1
// u_2 @v_1.p_1 as v_2
// ...
// u_n @v_(n-1).p_(n-1) as v_n
// The From clause needs to be rewritten to
// Join <------------------- schema: [(k_1), v_1, (k_2), v_2, ..., (k_(n-1)) v_(n-1)]
// / \
// ... un @v_(n-1).p_(n-1) <-- stack: [global, typeEnv: [outer: [global], schema: [(k_1), v_1, (k_2), v_2, ..., (k_(n-1)) v_(n-1)]]]
// Join <----------------------- schema: [(k_1), v_1, (k_2), v_2, (k_3), v_3]
// / \
// u_2 @v_1.p_1 as v2 <------- stack: [global, typeEnv: [outer: [global], schema: [(k_1), v_1, (k_2), v_2]]]
// JOIN <---------------------------- schema: [(k_1), v_1, (k_2), v_2]
// / \
// u_1 e as v_1 < ----\----------------------- stack: [global]
// u_2 @v_1.p_1 as v2 <------ stack: [global, typeEnv: [outer: [global], schema: [(k_1), v_1]]]
// while doing the traversal, instead of passing the stack,
// each join will produce its own schema and pass the schema as a type Env.
// The (k_i) indicate the possible key binding produced by unpivot.
// We calculate the var ref on the fly.
is Expr.Path.Step.Unpivot -> {
// Unpivot produces two binding, in this context we want the value,
// which always going to be the second binding
val op = rexOpVarLocal(1, varRefIndex + 1)
varRefIndex += 2
val index = fromList.size
fromList.add(relFromUnpivot(current, index))
op
}
is Expr.Path.Step.Wildcard -> {
// Scan produce only one binding
val op = rexOpVarLocal(1, varRefIndex)
varRefIndex += 1
val index = fromList.size
fromList.add(relFromDefault(current, index))
op
}
rex(StaticType.ANY, path)
}
rex(StaticType.ANY, path)
}

if (fromList.size == 0) return pathNavi
val fromNode = fromList.reduce { acc, scan ->
val schema = acc.type.schema + scan.type.schema
val props = emptySet<Rel.Prop>()
val type = relType(schema, props)
rel(type, relOpJoin(acc, scan, rex(StaticType.BOOL, rexOpLit(boolValue(true))), Rel.Op.Join.Type.INNER))
}

// compute the ref used by select construct
// always going to be the last binding
val selectRef = fromNode.type.schema.size - 1

val constructor = when (val op = pathNavi.op) {
is Rex.Op.Path.Index -> rex(pathNavi.type, rexOpPathIndex(rex(op.root.type, rexOpVarLocal(0, selectRef)), op.key))
is Rex.Op.Path.Key -> rex(pathNavi.type, rexOpPathKey(rex(op.root.type, rexOpVarLocal(0, selectRef)), op.key))
is Rex.Op.Path.Symbol -> rex(pathNavi.type, rexOpPathSymbol(rex(op.root.type, rexOpVarLocal(0, selectRef)), op.key))
is Rex.Op.Var.Local -> rex(pathNavi.type, rexOpVarLocal(0, selectRef))
else -> throw IllegalStateException()
}
val op = rexOpSelect(constructor, fromNode)
return rex(StaticType.ANY, op)
}

/**
* Construct Rel(Scan([path])).
*
* The constructed rel would produce one binding: _v$[index]
*/
private fun relFromDefault(path: Rex, index: Int): Rel {
val schema = listOf(
relBinding(
name = "_v$index", // fresh variable
type = path.type
)
)
val props = emptySet<Rel.Prop>()
val relType = relType(schema, props)
return rel(relType, relOpScan(path))
}

/**
* Construct Rel(Unpivot([path])).
*
* The constructed rel would produce two bindings: _k$[index] and _v$[index]
*/
private fun relFromUnpivot(path: Rex, index: Int): Rel {
val schema = listOf(
relBinding(
name = "_k$index", // fresh variable
type = StaticType.STRING
),
relBinding(
name = "_v$index", // fresh variable
type = path.type
)
)
val props = emptySet<Rel.Prop>()
val relType = relType(schema, props)
return rel(relType, relOpUnpivot(path))
}

private fun rexString(str: String) = rex(StaticType.STRING, rexOpLit(stringValue(str)))
Expand Down
28 changes: 16 additions & 12 deletions partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/PlanTyper.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,11 +145,6 @@ internal class PlanTyper(
return rel(type, op)
}

override fun visitRelOpErr(node: Rel.Op.Err, ctx: Rel.Type?): Rel {
val type = ctx ?: relType(emptyList(), emptySet())
return rel(type, node)
}

/**
* The output schema of a `rel.op.scan_index` is the value binding and index binding.
*/
Expand All @@ -176,15 +171,19 @@ internal class PlanTyper(
val kType = STRING

// value type, possibly coerced.
val vType = when (val t = rex.type) {
is StructType -> {
if (t.contentClosed || t.constraints.contains(TupleConstraint.Open(false))) {
unionOf(t.fields.map { it.value }.toSet()).flatten()
} else {
ANY
val vType = rex.type.allTypes.map { type ->
when (type) {
is StructType -> {
if (type.contentClosed || type.constraints.contains(TupleConstraint.Open(false))) {
unionOf(type.fields.map { it.value }.toSet()).flatten()
} else {
ANY
}
}
else -> type
}
else -> t
}.let {
unionOf(it.toSet()).flatten()
}

// rewrite
Expand All @@ -193,6 +192,11 @@ internal class PlanTyper(
return rel(type, op)
}

override fun visitRelOpErr(node: Rel.Op.Err, ctx: Rel.Type?): Rel {
val type = ctx ?: relType(emptyList(), emptySet())
return rel(type, node)
}

override fun visitRelOpDistinct(node: Rel.Op.Distinct, ctx: Rel.Type?): Rel {
val input = visitRel(node.input, ctx)
return rel(input.type, relOpDistinct(input))
Expand Down
2 changes: 1 addition & 1 deletion partiql-planner/src/main/kotlin/org/partiql/planner/internal/typer/TypeEnv.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ internal data class TypeEnv(
internal fun getScope(depth: Int): TypeEnv {
return when (depth) {
0 -> this
else -> outer.reversed()[depth - 1]
else -> outer[outer.size - depth]
}
}

Expand Down
Loading