From 16931ad423ae8a3a84eb195d2745feddae5bf1ee Mon Sep 17 00:00:00 2001 From: Alan Cai Date: Fri, 19 Jan 2024 17:39:41 -0800 Subject: [PATCH 1/5] Add RelSort and comparator between PartiQLValue --- .../org/partiql/eval/internal/Compiler.kt | 76 ++-- .../eval/internal/operator/rel/RelSort.kt | 64 +++ .../internal/operator/rex/ExprCollection.kt | 18 +- .../eval/internal/operator/rex/ExprSelect.kt | 7 +- .../eval/internal/util/NumberExtensions.kt | 154 +++++++ .../internal/util/PartiQLValueComparator.kt | 272 +++++++++++ .../eval/internal/PartiQLEngineDefaultTest.kt | 51 ++- .../util/PartiQLValueComparatorTest.kt | 429 ++++++++++++++++++ .../kotlin/org/partiql/value/PartiQLValue.kt | 5 +- 9 files changed, 1034 insertions(+), 42 deletions(-) create mode 100644 partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt create mode 100644 partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/NumberExtensions.kt create mode 100644 partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/PartiQLValueComparator.kt create mode 100644 partiql-eval/src/test/kotlin/org/partiql/eval/internal/util/PartiQLValueComparatorTest.kt diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt index 1fc701ad8f..bf3d439d59 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt @@ -11,6 +11,7 @@ import org.partiql.eval.internal.operator.rel.RelJoinRight import org.partiql.eval.internal.operator.rel.RelProject import org.partiql.eval.internal.operator.rel.RelScan import org.partiql.eval.internal.operator.rel.RelScanIndexed +import org.partiql.eval.internal.operator.rel.RelSort import org.partiql.eval.internal.operator.rex.ExprCase import org.partiql.eval.internal.operator.rex.ExprCollection import org.partiql.eval.internal.operator.rex.ExprGlobal @@ -31,74 +32,77 @@ import org.partiql.plan.Statement import org.partiql.plan.visitor.PlanBaseVisitor import org.partiql.spi.connector.ConnectorBindings import org.partiql.spi.connector.ConnectorObjectPath +import org.partiql.types.StaticType import org.partiql.value.PartiQLValueExperimental import java.lang.IllegalStateException internal class Compiler( private val plan: PartiQLPlan, private val catalogs: Map, -) : PlanBaseVisitor() { +) : PlanBaseVisitor() { // Current ctx is a StaticType to differentiate between collection types for Rex.Op.Collection fun compile(): Operator.Expr { - return visitPartiQLPlan(plan, Unit) + return visitPartiQLPlan(plan, null) } - override fun defaultReturn(node: PlanNode, ctx: Unit): Operator { + override fun defaultReturn(node: PlanNode, ctx: StaticType?): Operator { TODO("Not yet implemented") } - override fun visitRexOpErr(node: Rex.Op.Err, ctx: Unit): Operator { + override fun visitRexOpErr(node: Rex.Op.Err, ctx: StaticType?): Operator { throw IllegalStateException(node.message) } - override fun visitRelOpErr(node: Rel.Op.Err, ctx: Unit): Operator { + override fun visitRelOpErr(node: Rel.Op.Err, ctx: StaticType?): Operator { throw IllegalStateException(node.message) } // TODO: Re-look at - override fun visitPartiQLPlan(node: PartiQLPlan, ctx: Unit): Operator.Expr { + override fun visitPartiQLPlan(node: PartiQLPlan, ctx: StaticType?): Operator.Expr { return visitStatement(node.statement, ctx) as Operator.Expr } // TODO: Re-look at - override fun visitStatementQuery(node: Statement.Query, ctx: Unit): Operator.Expr { + override fun visitStatementQuery(node: Statement.Query, ctx: StaticType?): Operator.Expr { return visitRex(node.root, ctx) } // REX - override fun visitRex(node: Rex, ctx: Unit): Operator.Expr { - return super.visitRexOp(node.op, ctx) as Operator.Expr + override fun visitRex(node: Rex, ctx: StaticType?): Operator.Expr { + return super.visitRexOp(node.op, node.type) as Operator.Expr } - override fun visitRexOpCollection(node: Rex.Op.Collection, ctx: Unit): Operator { + override fun visitRexOpCollection(node: Rex.Op.Collection, ctx: StaticType?): Operator { val values = node.values.map { visitRex(it, ctx) } - return ExprCollection(values) + val type = ctx ?: error("No type provided in ctx") + return ExprCollection(values, type) } - override fun visitRexOpStruct(node: Rex.Op.Struct, ctx: Unit): Operator { + override fun visitRexOpStruct(node: Rex.Op.Struct, ctx: StaticType?): Operator { val fields = node.fields.map { ExprStruct.Field(visitRex(it.k, ctx), visitRex(it.v, ctx)) } return ExprStruct(fields) } - override fun visitRexOpSelect(node: Rex.Op.Select, ctx: Unit): Operator { + override fun visitRexOpSelect(node: Rex.Op.Select, ctx: StaticType?): Operator { val rel = visitRel(node.rel, ctx) + val ordered = node.rel.type.props.contains(Rel.Prop.ORDERED) val constructor = visitRex(node.constructor, ctx) - return ExprSelect(rel, constructor) + return ExprSelect(rel, constructor, ordered) } - override fun visitRexOpPivot(node: Rex.Op.Pivot, ctx: Unit): Operator { + override fun visitRexOpPivot(node: Rex.Op.Pivot, ctx: StaticType?): Operator { val rel = visitRel(node.rel, ctx) val key = visitRex(node.key, ctx) val value = visitRex(node.value, ctx) return ExprPivot(rel, key, value) } - override fun visitRexOpVar(node: Rex.Op.Var, ctx: Unit): Operator { + override fun visitRexOpVar(node: Rex.Op.Var, ctx: StaticType?): Operator { return ExprVar(node.ref) } - override fun visitRexOpGlobal(node: Rex.Op.Global, ctx: Unit): Operator { + override fun visitRexOpGlobal(node: Rex.Op.Global, ctx: StaticType?): Operator { val catalog = plan.catalogs[node.ref.catalog] val symbol = catalog.symbols[node.ref.symbol] val path = ConnectorObjectPath(symbol.path) @@ -106,19 +110,19 @@ internal class Compiler( return ExprGlobal(path, bindings) } - override fun visitRexOpPathKey(node: Rex.Op.Path.Key, ctx: Unit): Operator { + override fun visitRexOpPathKey(node: Rex.Op.Path.Key, ctx: StaticType?): Operator { val root = visitRex(node.root, ctx) val key = visitRex(node.key, ctx) return ExprPathKey(root, key) } - override fun visitRexOpPathSymbol(node: Rex.Op.Path.Symbol, ctx: Unit): Operator { + override fun visitRexOpPathSymbol(node: Rex.Op.Path.Symbol, ctx: StaticType?): Operator { val root = visitRex(node.root, ctx) val symbol = node.key return ExprPathSymbol(root, symbol) } - override fun visitRexOpPathIndex(node: Rex.Op.Path.Index, ctx: Unit): Operator { + override fun visitRexOpPathIndex(node: Rex.Op.Path.Index, ctx: StaticType?): Operator { val root = visitRex(node.root, ctx) val index = visitRex(node.key, ctx) return ExprPathIndex(root, index) @@ -126,32 +130,32 @@ internal class Compiler( // REL - override fun visitRel(node: Rel, ctx: Unit): Operator.Relation { + override fun visitRel(node: Rel, ctx: StaticType?): Operator.Relation { return super.visitRelOp(node.op, ctx) as Operator.Relation } - override fun visitRelOpScan(node: Rel.Op.Scan, ctx: Unit): Operator { + override fun visitRelOpScan(node: Rel.Op.Scan, ctx: StaticType?): Operator { val rex = visitRex(node.rex, ctx) return RelScan(rex) } - override fun visitRelOpProject(node: Rel.Op.Project, ctx: Unit): Operator { + override fun visitRelOpProject(node: Rel.Op.Project, ctx: StaticType?): Operator { val input = visitRel(node.input, ctx) val projections = node.projections.map { visitRex(it, ctx) } return RelProject(input, projections) } - override fun visitRelOpScanIndexed(node: Rel.Op.ScanIndexed, ctx: Unit): Operator { + override fun visitRelOpScanIndexed(node: Rel.Op.ScanIndexed, ctx: StaticType?): Operator { val rex = visitRex(node.rex, ctx) return RelScanIndexed(rex) } - override fun visitRexOpTupleUnion(node: Rex.Op.TupleUnion, ctx: Unit): Operator { + override fun visitRexOpTupleUnion(node: Rex.Op.TupleUnion, ctx: StaticType?): Operator { val args = node.args.map { visitRex(it, ctx) }.toTypedArray() return ExprTupleUnion(args) } - override fun visitRelOpJoin(node: Rel.Op.Join, ctx: Unit): Operator { + override fun visitRelOpJoin(node: Rel.Op.Join, ctx: StaticType?): Operator { val lhs = visitRel(node.lhs, ctx) val rhs = visitRel(node.rhs, ctx) val condition = visitRex(node.rex, ctx) @@ -163,7 +167,7 @@ internal class Compiler( } } - override fun visitRexOpCase(node: Rex.Op.Case, ctx: Unit): Operator { + override fun visitRexOpCase(node: Rex.Op.Case, ctx: StaticType?): Operator { val branches = node.branches.map { branch -> visitRex(branch.condition, ctx) to visitRex(branch.rex, ctx) } @@ -172,23 +176,33 @@ internal class Compiler( } @OptIn(PartiQLValueExperimental::class) - override fun visitRexOpLit(node: Rex.Op.Lit, ctx: Unit): Operator { + override fun visitRexOpLit(node: Rex.Op.Lit, ctx: StaticType?): Operator { return ExprLiteral(node.value) } - override fun visitRelOpDistinct(node: Rel.Op.Distinct, ctx: Unit): Operator { + override fun visitRelOpDistinct(node: Rel.Op.Distinct, ctx: StaticType?): Operator { val input = visitRel(node.input, ctx) return RelDistinct(input) } - override fun visitRelOpFilter(node: Rel.Op.Filter, ctx: Unit): Operator { + override fun visitRelOpFilter(node: Rel.Op.Filter, ctx: StaticType?): Operator { val input = visitRel(node.input, ctx) val condition = visitRex(node.predicate, ctx) return RelFilter(input, condition) } - override fun visitRelOpExclude(node: Rel.Op.Exclude, ctx: Unit): Operator { + override fun visitRelOpExclude(node: Rel.Op.Exclude, ctx: StaticType?): Operator { val input = visitRel(node.input, ctx) return RelExclude(input, node.paths) } + + override fun visitRelOpSort(node: Rel.Op.Sort, ctx: StaticType?): Operator { + val input = visitRel(node.input, ctx) + val compiledSpecs = node.specs.map { spec -> + val expr = visitRex(spec.rex, ctx) + val order = spec.order + expr to order + } + return RelSort(input, compiledSpecs) + } } diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt new file mode 100644 index 0000000000..26facc7952 --- /dev/null +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt @@ -0,0 +1,64 @@ +package org.partiql.eval.internal.operator.rel + +import org.partiql.eval.internal.Record +import org.partiql.eval.internal.operator.Operator +import org.partiql.eval.internal.util.PartiQLValueComparator +import org.partiql.plan.Rel +import org.partiql.value.PartiQLValueExperimental + +internal class RelSort( + val input: Operator.Relation, + val specs: List> + +) : Operator.Relation { + private var records: MutableList = mutableListOf() + private var init: Boolean = false + + private val nullsFirstComparator = PartiQLValueComparator(nullOrder = PartiQLValueComparator.NullOrder.FIRST) + private val nullsLastComparator = PartiQLValueComparator(nullOrder = PartiQLValueComparator.NullOrder.LAST) + + override fun open() { + input.open() + init = false + records = mutableListOf() + } + + @OptIn(PartiQLValueExperimental::class) + val comparator = object : Comparator { + override fun compare(l: Record, r: Record): Int { + specs.forEach { spec -> + val lVal = spec.first.eval(l) + val rVal = spec.first.eval(r) + + val cmpResult = when (spec.second) { + Rel.Op.Sort.Order.ASC_NULLS_FIRST -> nullsFirstComparator.compare(lVal, rVal) + Rel.Op.Sort.Order.ASC_NULLS_LAST -> nullsLastComparator.compare(lVal, rVal) + Rel.Op.Sort.Order.DESC_NULLS_FIRST -> nullsLastComparator.compare(rVal, lVal) + Rel.Op.Sort.Order.DESC_NULLS_LAST -> nullsFirstComparator.compare(rVal, lVal) + } + if (cmpResult != 0) { + return cmpResult + } + } + return 0 // Equal + } + } + + override fun next(): Record? { + if (!init) { + while (true) { + val row = input.next() ?: break + records.add(row) + } + records.sortWith(comparator) + } + return when (records.isEmpty()) { + true -> null + else -> records.removeAt(0) + } + } + + override fun close() { + input.close() + } +} diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rex/ExprCollection.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rex/ExprCollection.kt index e784dd37e1..45ad113271 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rex/ExprCollection.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rex/ExprCollection.kt @@ -2,18 +2,28 @@ package org.partiql.eval.internal.operator.rex import org.partiql.eval.internal.Record import org.partiql.eval.internal.operator.Operator +import org.partiql.types.BagType +import org.partiql.types.ListType +import org.partiql.types.SexpType +import org.partiql.types.StaticType import org.partiql.value.PartiQLValue import org.partiql.value.PartiQLValueExperimental import org.partiql.value.bagValue +import org.partiql.value.listValue +import org.partiql.value.sexpValue internal class ExprCollection( - private val values: List + private val values: List, + private val type: StaticType ) : Operator.Expr { @PartiQLValueExperimental override fun eval(record: Record): PartiQLValue { - return bagValue( - values.map { it.eval(record) } - ) + return when (type) { + is BagType -> bagValue(values.map { it.eval(record) }) + is SexpType -> sexpValue(values.map { it.eval(record) }) + is ListType -> listValue(values.map { it.eval(record) }) + else -> error("Unsupported type for collection $type") + } } } diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rex/ExprSelect.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rex/ExprSelect.kt index eb71d4435c..11829eb7ef 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rex/ExprSelect.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rex/ExprSelect.kt @@ -5,6 +5,7 @@ import org.partiql.eval.internal.operator.Operator import org.partiql.value.PartiQLValue import org.partiql.value.PartiQLValueExperimental import org.partiql.value.bagValue +import org.partiql.value.listValue /** * Invoke the constructor over all inputs. @@ -15,6 +16,7 @@ import org.partiql.value.bagValue internal class ExprSelect( val input: Operator.Relation, val constructor: Operator.Expr, + val ordered: Boolean ) : Operator.Expr { /** @@ -31,6 +33,9 @@ internal class ExprSelect( elements.add(e) } input.close() - return bagValue(elements) + return when (ordered) { + true -> listValue(elements) + false -> bagValue(elements) + } } } diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/NumberExtensions.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/NumberExtensions.kt new file mode 100644 index 0000000000..f7f3f98e43 --- /dev/null +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/NumberExtensions.kt @@ -0,0 +1,154 @@ +package org.partiql.eval.internal.util + +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at: + * + * http://aws.amazon.com/apache2.0/ + * + * or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific + * language governing permissions and limitations under the License. + */ + +import com.amazon.ion.Decimal +import java.math.BigDecimal +import java.math.MathContext +import java.math.RoundingMode + +/** + * Essentially the same as partiql-lang's NumberExtensions.kt. Key differences are the following: + * - Adding [Int] and [Float] branch when casing on the [Number] + * - TODO support for integer types smaller than [Int] + */ + +// TODO should this be configurable? +private val MATH_CONTEXT = MathContext(38, RoundingMode.HALF_EVEN) + +/** + * Factory function to create a [BigDecimal] using correct precision, use it in favor of native BigDecimal constructors + * and factory methods + */ +internal fun bigDecimalOf(num: Number, mc: MathContext = MATH_CONTEXT): BigDecimal = when (num) { + is Decimal -> num + is Int -> BigDecimal(num, mc) + is Long -> BigDecimal(num, mc) + is Float -> BigDecimal(num.toDouble(), mc) + is Double -> BigDecimal(num, mc) + is BigDecimal -> num + else -> throw IllegalArgumentException("Unsupported number type: $num, ${num.javaClass}") +} + +private val CONVERSION_MAP = mapOf>, Class>( + setOf(Int::class.javaObjectType, Int::class.javaObjectType) to Int::class.javaObjectType, + setOf(Int::class.javaObjectType, Long::class.javaObjectType) to Long::class.javaObjectType, + setOf(Int::class.javaObjectType, Float::class.javaObjectType) to Float::class.javaObjectType, + setOf(Int::class.javaObjectType, Double::class.javaObjectType) to Double::class.javaObjectType, + setOf(Int::class.javaObjectType, BigDecimal::class.javaObjectType) to BigDecimal::class.javaObjectType, + + setOf(Float::class.javaObjectType, Float::class.javaObjectType) to Float::class.javaObjectType, + // Float w/ long -> Double + setOf(Float::class.javaObjectType, Long::class.javaObjectType) to Double::class.javaObjectType, + setOf(Float::class.javaObjectType, Double::class.javaObjectType) to Double::class.javaObjectType, + setOf(Float::class.javaObjectType, BigDecimal::class.javaObjectType) to BigDecimal::class.javaObjectType, + + setOf(Long::class.javaObjectType, Long::class.javaObjectType) to Long::class.javaObjectType, + setOf(Long::class.javaObjectType, Double::class.javaObjectType) to Double::class.javaObjectType, + setOf(Long::class.javaObjectType, BigDecimal::class.javaObjectType) to BigDecimal::class.javaObjectType, + + setOf(Double::class.javaObjectType, Double::class.javaObjectType) to Double::class.javaObjectType, + setOf(Double::class.javaObjectType, BigDecimal::class.javaObjectType) to BigDecimal::class.javaObjectType, + + setOf(BigDecimal::class.javaObjectType, BigDecimal::class.javaObjectType) to BigDecimal::class.javaObjectType +) + +private val CONVERTERS = mapOf, (Number) -> Number>( + Int::class.javaObjectType to Number::toInt, + Long::class.javaObjectType to Number::toLong, + Float::class.javaObjectType to Number::toFloat, + Double::class.javaObjectType to Number::toDouble, + BigDecimal::class.java to { num -> + when (num) { + is Int -> bigDecimalOf(num) + is Long -> bigDecimalOf(num) + is Float -> bigDecimalOf(num) + is Double -> bigDecimalOf(num) + is BigDecimal -> bigDecimalOf(num) + else -> throw IllegalArgumentException( + "Unsupported number for decimal conversion: $num" + ) + } + } +) + +internal fun Number.isZero() = when (this) { + // using compareTo instead of equals for BigDecimal because equality also checks same scale + is Int -> this == 0 + is Long -> this == 0L + is Float -> this == 0.0f || this == -0.0f + is Double -> this == 0.0 || this == -0.0 + is BigDecimal -> BigDecimal.ZERO.compareTo(this) == 0 + else -> throw IllegalStateException("$this") +} + +@Suppress("UNCHECKED_CAST") +/** Provides a narrowing or widening operator on supported numbers. */ +internal fun Number.coerce(type: Class): T where T : Number { + val conv = CONVERTERS[type] ?: throw IllegalArgumentException("No converter for $type") + return conv(this) as T +} + +/** + * Implements a very simple number tower to convert two numbers to their arithmetic + * compatible type. + * + * This is only supported on limited types needed by the expression system. + */ +internal fun coerceNumbers(first: Number, second: Number): Pair { + fun typeFor(n: Number): Class<*> = if (n is Decimal) { + BigDecimal::class.javaObjectType + } else { + n.javaClass + } + + val type = CONVERSION_MAP[setOf(typeFor(first), typeFor(second))] + ?: throw IllegalArgumentException("No coercion support for ${typeFor(first)} to ${typeFor(second)}") + + return Pair(first.coerce(type), second.coerce(type)) +} + +internal operator fun Number.compareTo(other: Number): Int { + val (first, second) = coerceNumbers(this, other) + return when (first) { + is Int -> first.compareTo(second as Int) + is Long -> first.compareTo(second as Long) + is Float -> first.compareTo(second as Float) + is Double -> first.compareTo(second as Double) + is BigDecimal -> first.compareTo(second as BigDecimal) + else -> throw IllegalStateException() + } +} + +internal val Number.isNaN + get() = when (this) { + is Float -> isNaN() + is Double -> isNaN() + else -> false + } + +internal val Number.isNegInf + get() = when (this) { + is Float -> isInfinite() && this < 0 + is Double -> isInfinite() && this < 0 + else -> false + } + +internal val Number.isPosInf + get() = when (this) { + is Float -> isInfinite() && this > 0 + is Double -> isInfinite() && this > 0 + else -> false + } diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/PartiQLValueComparator.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/PartiQLValueComparator.kt new file mode 100644 index 0000000000..17d8233f77 --- /dev/null +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/PartiQLValueComparator.kt @@ -0,0 +1,272 @@ +package org.partiql.eval.internal.util + +import org.partiql.value.BagValue +import org.partiql.value.BlobValue +import org.partiql.value.BoolValue +import org.partiql.value.ClobValue +import org.partiql.value.DateValue +import org.partiql.value.ListValue +import org.partiql.value.MissingValue +import org.partiql.value.NullValue +import org.partiql.value.NumericValue +import org.partiql.value.PartiQLValue +import org.partiql.value.PartiQLValueExperimental +import org.partiql.value.ScalarValue +import org.partiql.value.SexpValue +import org.partiql.value.StructValue +import org.partiql.value.TextValue +import org.partiql.value.TimeValue +import org.partiql.value.TimestampValue + +/** + * Provides a total, natural ordering over [PartiQLValue] as defined by section 12.2 of the PartiQL specification + * (https://partiql.org/assets/PartiQL-Specification.pdf#subsection.12.2). PartiQL treats Ion typed nulls as `NULL` + * for the purposes of comparisons and Ion annotations are not considered for comparison purposes. + * + * The ordering rules are as follows: + * + * * [NullValue] and [MissingValue] are always first or last and compare equally. In other words, + * comparison cannot distinguish between `NULL` or `MISSING`. + * * The [BoolValue] values follow with `false` coming before `true`. + * * The [NumericValue] types come next ordered by their numerical value irrespective + * of precision or specific type. + * For `FLOAT` special values, `nan` comes before `-inf`, which comes before all normal + * numeric values, which is followed by `+inf`. + * * [DateValue] values follow and are compared by the date from earliest to latest. + * * [TimeValue] values follow and are compared by the time of the day (point of time in a day of 24 hours) + * from earliest to latest. Note that time without time zone is not directly comparable with time with time zone. + * * [TimestampValue] values follow and are compared by the point of time irrespective of precision and + * local UTC offset. + * * The [TextValue] types come next ordered by their lexicographical ordering by + * Unicode scalar irrespective of their specific type. + * * The [BlobValue] and [ClobValue] types follow and are ordered by their lexicographical ordering + * by octet. + * * [ListValue] comes next, and their values compare lexicographically based on their + * child elements recursively based on this definition. + * * [SexpValue] follows and compares within its type similar to `LIST`. + * * [StructValue] values follow and compare lexicographically based on the *sorted* + * (as defined by this definition) members, as pairs of field name and the member value. + * * [BagValue] values come finally (except with [NullOrder.LAST]), and their values + * compare lexicographically based on the *sorted* child elements. + * + * @param nullOrder that places [NullValue], [MissingValue], and typed Ion null values first or last + */ +@OptIn(PartiQLValueExperimental::class) +internal class PartiQLValueComparator(private val nullOrder: NullOrder) : Comparator { + /** Whether null values come first or last. */ + enum class NullOrder { + FIRST, + LAST + } + + private val EQUAL = 0 + private val LESS = -1 + private val GREATER = 1 + + private fun PartiQLValue.isNullOrMissing(): Boolean = this is NullValue || this is MissingValue || this.isNull + private fun PartiQLValue.isLob(): Boolean = this is BlobValue || this is ClobValue + + private val structFieldComparator = object : Comparator> { + override fun compare(left: Pair, right: Pair): Int { + val cmpKey = left.first.compareTo(right.first) + if (cmpKey != 0) { + return cmpKey + } + return compare(left.second, right.second) + } + } + + private fun compareInternal(l: PartiQLValue, r: PartiQLValue, nullsFirst: NullOrder): Int { + if (l.withoutAnnotations() == r.withoutAnnotations()) { + return EQUAL + } + + when { + l.isNullOrMissing() && r.isNullOrMissing() -> return EQUAL + l.isNullOrMissing() -> return when (nullsFirst) { + NullOrder.FIRST -> LESS + NullOrder.LAST -> GREATER + } + r.isNullOrMissing() -> return when (nullsFirst) { + NullOrder.FIRST -> GREATER + NullOrder.LAST -> LESS + } + } + + // BOOL comparator + when { + l is BoolValue && r is BoolValue -> { + val lv = l.value!! + val rv = r.value!! + return when { + !lv -> LESS + else -> GREATER + } + } + l is BoolValue -> return LESS + r is BoolValue -> return GREATER + } + + // NUMBER comparator + when { + l is NumericValue<*> && r is NumericValue<*> -> { + val lv = l.value!! + val rv = r.value!! + return when { + lv.isNaN && rv.isNaN -> EQUAL + lv.isNaN -> LESS + rv.isNaN -> GREATER + lv.isNegInf && rv.isNegInf -> EQUAL + lv.isNegInf -> LESS + rv.isNegInf -> GREATER + lv.isPosInf && rv.isPosInf -> EQUAL + lv.isPosInf -> GREATER + rv.isPosInf -> LESS + lv.isZero() && rv.isZero() -> return EQUAL + else -> lv.compareTo(rv) + } + } + l is NumericValue<*> -> return LESS + r is NumericValue<*> -> return GREATER + } + + // DATE + when { + l is DateValue && r is DateValue -> { + val lv = l.value!! + val rv = r.value!! + return lv.compareTo(rv) + } + l is DateValue -> return LESS + r is DateValue -> return GREATER + } + + // TIME + when { + l is TimeValue && r is TimeValue -> { + val lv = l.value!! + val rv = r.value!! + return lv.compareTo(rv) + } + l is TimeValue -> return LESS + r is TimeValue -> return GREATER + } + + // TIMESTAMP + when { + l is TimestampValue && r is TimestampValue -> { + val lv = l.value!! + val rv = r.value!! + return lv.compareTo(rv) + } + l is TimestampValue -> return LESS + r is TimestampValue -> return GREATER + } + + // TEXT + when { + l is TextValue<*> && r is TextValue<*> -> { + val lv = l.string!! + val rv = r.string!! + return lv.compareTo(rv) + } + l is TextValue<*> -> return LESS + r is TextValue<*> -> return GREATER + } + + // LOB + when { + l.isLob() && r.isLob() -> { + val lv = ((l as ScalarValue<*>).value) as ByteArray + val rv = ((r as ScalarValue<*>).value) as ByteArray + val commonLen = minOf(lv.size, rv.size) + for (i in 0 until commonLen) { + val lOctet = lv[i].toInt() and 0xFF + val rOctet = rv[i].toInt() and 0xFF + val diff = lOctet - rOctet + if (diff != 0) { + return diff + } + } + return lv.size - rv.size + } + l.isLob() -> return LESS + r.isLob() -> return GREATER + } + + // LIST + when { + l is ListValue<*> && r is ListValue<*> -> { + return compareOrdered(l, r, this) + } + l is ListValue<*> -> return LESS + r is ListValue<*> -> return GREATER + } + + // SEXP + when { + l is SexpValue<*> && r is SexpValue<*> -> { + return compareOrdered(l, r, this) + } + l is SexpValue<*> -> return LESS + r is SexpValue<*> -> return GREATER + } + + // STRUCT + when { + l is StructValue<*> && r is StructValue<*> -> { + val entriesL = l.entries + val entriesR = r.entries + return compareUnordered(entriesL, entriesR, structFieldComparator) + } + l is StructValue<*> -> return LESS + r is StructValue<*> -> return GREATER + } + + // BAG + when { + l is BagValue<*> && r is BagValue<*> -> { + return compareUnordered(l, r, this) + } + l is BagValue<*> -> return LESS + r is BagValue<*> -> return GREATER + } + throw IllegalStateException("Could not compare: $l and $r") + } + + private fun compareOrdered( + l: Iterable, + r: Iterable, + elementComparator: Comparator + ): Int { + val lIter = l.iterator() + val rIter = r.iterator() + while (lIter.hasNext() && rIter.hasNext()) { + val lVal = lIter.next() + val rVal = rIter.next() + val result = elementComparator.compare(lVal, rVal) + if (result != 0) { + return result + } + } + return when { + lIter.hasNext() -> GREATER + rIter.hasNext() -> LESS + else -> EQUAL + } + } + + private fun compareUnordered( + l: Iterable, + r: Iterable, + elementComparator: Comparator + ): Int { + val sortedL = l.sortedWith(elementComparator) + val sortedR = r.sortedWith(elementComparator) + return compareOrdered(sortedL, sortedR, elementComparator) + } + + override fun compare(l: PartiQLValue, r: PartiQLValue): Int { + return compareInternal(l, r, nullOrder) + } +} diff --git a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt index 706fa6d907..cf60db5d64 100644 --- a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt +++ b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt @@ -18,6 +18,7 @@ import org.partiql.value.boolValue import org.partiql.value.int32Value import org.partiql.value.int64Value import org.partiql.value.io.PartiQLValueIonWriterBuilder +import org.partiql.value.listValue import org.partiql.value.missingValue import org.partiql.value.nullValue import org.partiql.value.stringValue @@ -276,7 +277,7 @@ class PartiQLEngineDefaultTest { structValue( "a" to structValue( "b" to structValue( - "c" to bagValue( // TODO: should be ListValue; currently, Rex.ExprCollection doesn't return lists + "c" to listValue( structValue( "field_y" to int32Value(0) ), @@ -291,7 +292,53 @@ class PartiQLEngineDefaultTest { ) ) ) - ) + ), + SuccessTestCase( + input = "SELECT * FROM <<{'a': 10, 'b': 1}, {'a': 1, 'b': 2}>> AS t ORDER BY t.a;", + expected = listValue( + structValue("a" to int32Value(1), "b" to int32Value(2)), + structValue("a" to int32Value(10), "b" to int32Value(1)) + ) + ), + SuccessTestCase( + input = "SELECT * FROM <<{'a': 10, 'b': 1}, {'a': 1, 'b': 2}>> AS t ORDER BY t.a DESC;", + expected = listValue( + structValue("a" to int32Value(10), "b" to int32Value(1)), + structValue("a" to int32Value(1), "b" to int32Value(2)) + ) + ), + SuccessTestCase( + input = "SELECT * FROM <<{'a': NULL, 'b': 1}, {'a': 1, 'b': 2}, {'a': 3, 'b': 4}>> AS t ORDER BY t.a NULLS LAST;", + expected = listValue( + structValue("a" to int32Value(1), "b" to int32Value(2)), + structValue("a" to int32Value(3), "b" to int32Value(4)), + structValue("a" to nullValue(), "b" to int32Value(1)) + ) + ), + SuccessTestCase( + input = "SELECT * FROM <<{'a': NULL, 'b': 1}, {'a': 1, 'b': 2}, {'a': 3, 'b': 4}>> AS t ORDER BY t.a NULLS FIRST;", + expected = listValue( + structValue("a" to nullValue(), "b" to int32Value(1)), + structValue("a" to int32Value(1), "b" to int32Value(2)), + structValue("a" to int32Value(3), "b" to int32Value(4)) + ) + ), + SuccessTestCase( + input = "SELECT * FROM <<{'a': NULL, 'b': 1}, {'a': 1, 'b': 2}, {'a': 3, 'b': 4}>> AS t ORDER BY t.a DESC NULLS LAST;", + expected = listValue( + structValue("a" to int32Value(3), "b" to int32Value(4)), + structValue("a" to int32Value(1), "b" to int32Value(2)), + structValue("a" to nullValue(), "b" to int32Value(1)) + ) + ), + SuccessTestCase( + input = "SELECT * FROM <<{'a': NULL, 'b': 1}, {'a': 1, 'b': 2}, {'a': 3, 'b': 4}>> AS t ORDER BY t.a DESC NULLS FIRST;", + expected = listValue( + structValue("a" to nullValue(), "b" to int32Value(1)), + structValue("a" to int32Value(3), "b" to int32Value(4)), + structValue("a" to int32Value(1), "b" to int32Value(2)) + ) + ), ) } public class SuccessTestCase @OptIn(PartiQLValueExperimental::class) constructor( diff --git a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/util/PartiQLValueComparatorTest.kt b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/util/PartiQLValueComparatorTest.kt new file mode 100644 index 0000000000..66f8e77e14 --- /dev/null +++ b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/util/PartiQLValueComparatorTest.kt @@ -0,0 +1,429 @@ +package org.partiql.eval.internal.util + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.partiql.value.BagValue +import org.partiql.value.ListValue +import org.partiql.value.PartiQLValue +import org.partiql.value.PartiQLValueExperimental +import org.partiql.value.StructValue +import org.partiql.value.bagValue +import org.partiql.value.blobValue +import org.partiql.value.boolValue +import org.partiql.value.clobValue +import org.partiql.value.dateValue +import org.partiql.value.datetime.DateTimeValue.date +import org.partiql.value.datetime.DateTimeValue.time +import org.partiql.value.datetime.DateTimeValue.timestamp +import org.partiql.value.datetime.TimeZone +import org.partiql.value.decimalValue +import org.partiql.value.float32Value +import org.partiql.value.float64Value +import org.partiql.value.int32Value +import org.partiql.value.int64Value +import org.partiql.value.listValue +import org.partiql.value.missingValue +import org.partiql.value.nullValue +import org.partiql.value.sexpValue +import org.partiql.value.stringValue +import org.partiql.value.structValue +import org.partiql.value.symbolValue +import org.partiql.value.timeValue +import org.partiql.value.timestampValue +import org.partiql.value.toIon +import java.math.BigDecimal +import java.util.Base64 +import java.util.Random + +@OptIn(PartiQLValueExperimental::class) +class PartiQLValueComparatorTest { + class EquivValues(vararg val values: PartiQLValue) + + private val nullsFirstComparator = PartiQLValueComparator(nullOrder = PartiQLValueComparator.NullOrder.FIRST) + private val nullsLastComparator = PartiQLValueComparator(nullOrder = PartiQLValueComparator.NullOrder.LAST) + + // TODO consider replacing linear congruential generator with something else (e.g. xorshift) + // RNG for fuzz testing the sort orders, the seed is arbitrary but static for determinism + private val SEED = 0x59CF3400BEF36A67 + + private val emptyList: ListValue = listValue(emptyList()) + private val emptyBag: BagValue = bagValue(emptyList()) + private fun emptyStruct(annotations: List = emptyList()): StructValue = structValue(annotations = annotations) + + private fun base64Decode(s: String): ByteArray = Base64.getDecoder().decode(s) + + // Checks that [allValues], when shuffled and sorted using [comparator], follow same ordering as [allValues] + private fun checkAllEquivalent(allValues: List, comparator: PartiQLValueComparator) { + val shuffledValues = allValues.shuffled(Random(SEED)) + val sortedAfterShuffle = shuffledValues.sortedWith(comparator) + assertEquals(allValues.size, sortedAfterShuffle.size) + allValues.zip(sortedAfterShuffle) + .forEach { + assertEquals(0, comparator.compare(it.first, it.second), "${it.first.toIon()} != ${it.second.toIon()}") + } + } + + @Test + fun testNullsFirst() { + val sortedValsNullsFirst = (nullValues + nonNullPartiQLValue).flatMap { + it.values.asIterable() + } + checkAllEquivalent(sortedValsNullsFirst, nullsFirstComparator) + } + + @Test + fun testNullsLast() { + val sortedValsNullsLast = (nonNullPartiQLValue + nullValues).flatMap { + it.values.asIterable() + } + checkAllEquivalent(sortedValsNullsLast, nullsLastComparator) + } + + @Test + fun checkEquivalenceClasses() { + // Checks that all the values in an [EquivValues] are equivalent using both comparators + (nullValues + nonNullPartiQLValue).forEach { + val values = it.values + values.forEach { v1 -> + values.forEach { v2 -> + assertEquals(0, nullsFirstComparator.compare(v1, v2), "${v1.toIon()} != ${v1.toIon()}") + assertEquals(0, nullsLastComparator.compare(v1, v2), "${v1.toIon()} != ${v1.toIon()}") + } + } + } + } + + private val nullValues = listOf( + EquivValues( + nullValue(), // null + missingValue(), // missing + nullValue(annotations = listOf("a")), // `a::null` + missingValue(annotations = listOf("a")), // `a::missing` + int32Value(null), // `null.int`, + structValue(null) // `null.struct` + ) + ) + + private val nonNullPartiQLValue = listOf( + EquivValues( + boolValue(false), + boolValue(false, annotations = listOf("b")) + ), + EquivValues( + boolValue(true, annotations = listOf("c")), + boolValue(true) + ), + EquivValues( + // make sure there are at least two nan + float32Value(Float.NaN), + float64Value(Double.NaN), + ), + EquivValues( + // make sure there are at least two nan + float32Value(Float.NEGATIVE_INFINITY), + float64Value(Double.NEGATIVE_INFINITY), + ), + EquivValues( + float32Value(-1e1000f), + float64Value(-1e1000) + ), + EquivValues( + float32Value(-5e-1f), + float64Value(-5e-1), + decimalValue(BigDecimal("-0.50000000000000000000000000")), + float32Value(-0.5e0f), + float64Value(-0.5e0) + ), + EquivValues( + decimalValue(BigDecimal("-0.0")), + decimalValue(BigDecimal("-0.0000000000")), + float32Value(0e0f), + float64Value(0e0), + float32Value(-0e0f), + float64Value(-0e0), + decimalValue(BigDecimal("0e10000")), + int32Value(0), + int32Value(-0), + int64Value(0), + int64Value(-0) + ), + EquivValues( + float32Value(5e9f), + float64Value(5e9), + // 5000000000 does not fit into int32 + int64Value(5000000000), + int64Value(0x12a05f200), + float32Value(5.0e9f), + float64Value(5.0e9), + decimalValue(BigDecimal("5e9")), + decimalValue(BigDecimal("5.00000e9")), + ), + EquivValues( + // make sure there are at least two +inf + float32Value(Float.POSITIVE_INFINITY), + float64Value(Double.POSITIVE_INFINITY), + ), + EquivValues( + dateValue(date(year = 1992, month = 8, day = 22)) + ), + EquivValues( + dateValue(date(year = 2021, month = 8, day = 22)) + ), + EquivValues( + timeValue(time(hour = 12, minute = 12, second = 12, timeZone = TimeZone.UnknownTimeZone)), + timeValue(time(hour = 12, minute = 12, second = 12, nano = 0, timeZone = TimeZone.UnknownTimeZone)), + timeValue(time(hour = 12, minute = 12, second = 12, timeZone = TimeZone.UnknownTimeZone)), + // time second precision handled by time constructor + timeValue(time(hour = 12, minute = 12, second = 12, timeZone = TimeZone.UtcOffset.of(0))), + ), + EquivValues( + timeValue(time(hour = 12, minute = 12, second = 12, nano = 100000000, timeZone = TimeZone.UnknownTimeZone)), + ), + EquivValues( + timeValue(time(hour = 12, minute = 12, second = 12, nano = 0, timeZone = TimeZone.UtcOffset.of(-8, 0))), + timeValue(time(hour = 12, minute = 12, second = 12, timeZone = TimeZone.UtcOffset.of(-8, 0))), + ), + EquivValues( + timeValue(time(hour = 12, minute = 12, second = 12, nano = 100000000, timeZone = TimeZone.UtcOffset.of(-9, 0))), + ), + EquivValues( + timestampValue(timestamp(year = 2017, timeZone = TimeZone.UtcOffset.of(0, 0))), // `2017T` + timestampValue(timestamp(year = 2017, month = 1, timeZone = TimeZone.UtcOffset.of(0, 0))), // `2017-01T` + timestampValue(timestamp(year = 2017, month = 1, day = 1, timeZone = TimeZone.UtcOffset.of(0, 0))), // `2017-01-01T` + timestampValue(timestamp(year = 2017, month = 1, day = 1, hour = 0, minute = 0, second = 0, timeZone = TimeZone.UtcOffset.of(0, 0))), // `2017-01-01T00:00-00:00` + timestampValue(timestamp(year = 2017, month = 1, day = 1, hour = 1, minute = 0, second = 0, timeZone = TimeZone.UtcOffset.of(1, 0))) // `2017-01-01T01:00+01:00` + ), + EquivValues( + timestampValue(timestamp(year = 2017, month = 1, day = 1, hour = 1, minute = 0, second = 0, timeZone = TimeZone.UtcOffset.of(0, 0))) // `2017-01-01T01:00Z` + ), + EquivValues( + stringValue(value = ""), + stringValue(value = "", annotations = listOf("foobar")), + symbolValue(value = ""), + symbolValue(value = "", annotations = listOf("foobar")) + ), + EquivValues( + stringValue(value = "A"), + stringValue(value = "A", annotations = listOf("foobar")), + symbolValue(value = "A"), + symbolValue(value = "A", annotations = listOf("foobar")) + ), + EquivValues( + stringValue(value = "AA"), + symbolValue(value = "AA"), + ), + EquivValues( + stringValue(value = "a"), + symbolValue(value = "a"), + ), + EquivValues( + stringValue(value = "azzzzzzz"), + symbolValue(value = "azzzzzzz"), + ), + EquivValues( + stringValue(value = "z"), + symbolValue(value = "z"), + ), + // TODO add a UTF-16 order breaker here to verify we're doing the right thing + EquivValues( + stringValue(value = "\uD83D\uDCA9"), + symbolValue(value = "\uD83D\uDCA9"), + ), + EquivValues( + blobValue(base64Decode("")), // `{{}}` + clobValue("".toByteArray()) // `{{\"\"}}` + ), + EquivValues( + blobValue(base64Decode("QQ==")), // `{{QQ==}}` + clobValue("A".toByteArray()) // `{{\"A\"}}` + ), + EquivValues( + blobValue(base64Decode("YWFhYWFhYWFhYWFhYQ==")), // `{{YWFhYWFhYWFhYWFhYQ==}}` + clobValue("aaaaaaaaaaaaa".toByteArray()) // `{{"aaaaaaaaaaaaa"}}` + ), + EquivValues( + emptyList, // [] + listValue(emptyList(), annotations = listOf("z", "x", "y")) // `z::x::y::[]` + ), + EquivValues( + listValue(boolValue(false), emptyStruct()) // [false, {}] + ), + EquivValues( + listValue(boolValue(true)) // [true] + ), + EquivValues( + listValue(boolValue(true), boolValue(true)) // [true, true] + ), + EquivValues( + listValue(boolValue(true), int32Value(100)) // [true, 100] + ), + EquivValues( + listValue(listOf(listValue(int32Value(1)))) // [[1]] + ), + EquivValues( + listValue(listOf(listValue(int32Value(1), int32Value(1)))) // [[1, 1]] + ), + EquivValues( + listValue(listOf(listValue(int32Value(1), int32Value(2)))) // [[1, 2]] + ), + EquivValues( + listValue(listOf(listValue(int32Value(2), int32Value(1)))) // [[2, 1]] + ), + EquivValues( + listValue(listOf(listValue(listOf(listValue(int32Value(1)))))) // [[[1]]] + ), + EquivValues( + sexpValue(emptyList(), annotations = listOf("a", "b", "c")) // `a::b::c::()` + ), + EquivValues( + sexpValue(float32Value(1f)), // "`a::b::c::(1e0)`" + sexpValue(float64Value(1.0), annotations = listOf("a", "b", "c")), // "`a::b::c::(1e0)`" + sexpValue(int32Value(1)), // `(1)` + sexpValue(decimalValue(BigDecimal("1.0000000000000"))) // `(1.0000000000000)` + ), + EquivValues( + sexpValue(timestampValue(timestamp(year = 2012)), float32Value(Float.NaN)) // `(2012T nan)` + ), + EquivValues( + sexpValue(timestampValue(timestamp(year = 2012)), int32Value(1), int32Value(2), int32Value(3)) // `(2012T 1 2 3)` + ), + EquivValues( + sexpValue(listOf(listValue(emptyList()))) // `([])` + ), + EquivValues( + sexpValue(emptyList, emptyList) // `([] [])` + ), + EquivValues( + emptyStruct(), // {} + emptyStruct(annotations = listOf("m", "n", "o")) // `m::n::o::{}` + ), + EquivValues( + structValue( // {'a': true, 'b': 1000, 'c': false} + "a" to boolValue(true), "b" to int32Value(1000), "c" to boolValue(false) + ), + structValue( // {'b': `1e3`, 'a': true, 'c': false} + "b" to float32Value(1000f), "a" to boolValue(true), "c" to boolValue(false) + ) + ), + EquivValues( + structValue( // {'b': 1000, 'c': false} + "b" to int32Value(1000), "c" to boolValue(false) + ), + structValue( // {'c': false, 'b': 1.00000000e3} + "c" to boolValue(false), "b" to decimalValue(BigDecimal("1.00000000e3")) + ) + ), + EquivValues( + structValue( // {'c': false} + "c" to boolValue(false) + ) + ), + EquivValues( + structValue( // {'d': 1, 'f': 2} + "d" to int32Value(1), "f" to int32Value(2) + ) + ), + EquivValues( + structValue( // {'d': 2, 'e': 3, 'f': 4} + "d" to int32Value(2), + "e" to int32Value(3), + "f" to int32Value(4) + ) + ), + EquivValues( + structValue( // {'d': 3, 'e': 2} + "d" to int32Value(3), + "e" to int32Value(2) + ) + ), + EquivValues( + structValue( // { 'm': [1, 1], 'n': [1, 1]} + "m" to listValue(int32Value(1), int32Value(1)), + "n" to listValue(int32Value(1), int32Value(1)) + ) + ), + EquivValues( + structValue( // { 'm': [1, 1], 'n': [1, 2]} + "m" to listValue(int32Value(1), int32Value(1)), + "n" to listValue(int32Value(1), int32Value(2)) + ) + ), + EquivValues( + structValue( // { 'm': [1, 1], 'n': [2, 2]} + "m" to listValue(int32Value(1), int32Value(1)), + "n" to listValue(int32Value(2), int32Value(2)) + ) + ), + EquivValues( + structValue( // { 'm': [1, 2], 'n': [2, 2]} + "m" to listValue(int32Value(1), int32Value(2)), + "n" to listValue(int32Value(2), int32Value(2)) + ) + ), + EquivValues( + structValue( // { 'm': [2, 2], 'n': [2, 2]} + "m" to listValue(int32Value(1), int32Value(2)), + "n" to listValue(int32Value(2), int32Value(2)) + ) + ), + EquivValues( + structValue( // { 'm': <<1, 1>>, 'n': []} + "m" to bagValue(int32Value(1), int32Value(1)), + "n" to emptyList + ) + ), + EquivValues( + structValue( // { 'm': <<1, 1>>, 'n': <<>>} + "m" to bagValue(int32Value(1), int32Value(1)), + "n" to emptyBag + ) + ), + EquivValues( // {'x': 1, 'y': 2} + structValue( + "x" to int32Value(1), + "y" to int32Value(2) + ) + ), + EquivValues( // {'x': 1, 'y': 2, 'z': 1} + structValue( + "x" to int32Value(1), + "y" to int32Value(2), + "z" to int32Value(1) + ) + ), + EquivValues( // <<>> + emptyBag + ), + EquivValues( + // The ordered values are: true, true, 1 + // <<1, true, true>> + bagValue(int32Value(1), boolValue(true), boolValue(true)) + ), + EquivValues( + // The ordered values are: true, true, 1, 1, 1 + // <> + bagValue(boolValue(true), int32Value(1), decimalValue(BigDecimal("1.0")), float32Value(1e0f), boolValue(true)) + ), + EquivValues( // <<1>> + bagValue(int32Value(1)) + ), + EquivValues( // <<1, 1>> + bagValue(int32Value(1), int32Value(1)) + ), + EquivValues( // << [] >> + bagValue(listOf(emptyList)) + ), + EquivValues( // << {}, [] >> + bagValue(emptyStruct(), emptyList) + ), + EquivValues( // << {} >> + bagValue(emptyStruct()) + ), + EquivValues( // << <<>> >> + bagValue(listOf(emptyBag)) + ), + EquivValues( // << <<>>, <<>> >> + bagValue(emptyBag, emptyBag) + ) + ) +} diff --git a/partiql-types/src/main/kotlin/org/partiql/value/PartiQLValue.kt b/partiql-types/src/main/kotlin/org/partiql/value/PartiQLValue.kt index be2ba90a55..10a9a8fa56 100644 --- a/partiql-types/src/main/kotlin/org/partiql/value/PartiQLValue.kt +++ b/partiql-types/src/main/kotlin/org/partiql/value/PartiQLValue.kt @@ -267,13 +267,10 @@ public abstract class SymbolValue : TextValue() { } @PartiQLValueExperimental -public abstract class ClobValue : TextValue() { +public abstract class ClobValue : ScalarValue { override val type: PartiQLValueType = PartiQLValueType.CLOB - override val string: String? - get() = value?.toString(Charsets.UTF_8) - abstract override fun copy(annotations: Annotations): ClobValue abstract override fun withAnnotations(annotations: Annotations): ClobValue From 50f1007e0899211aef0843af56aea152a3884ddc Mon Sep 17 00:00:00 2001 From: Alan Cai Date: Mon, 22 Jan 2024 11:15:48 -0800 Subject: [PATCH 2/5] Apply commits from partiql-plugin-impl branch to fix build --- .../planner/util/PlanNodeEquivalentVisitor.kt | 7 ------ .../partiql/runner/executor/EvalExecutor.kt | 22 ++++++++----------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/partiql-planner/src/test/kotlin/org/partiql/planner/util/PlanNodeEquivalentVisitor.kt b/partiql-planner/src/test/kotlin/org/partiql/planner/util/PlanNodeEquivalentVisitor.kt index 04179370b6..7c6d84e497 100644 --- a/partiql-planner/src/test/kotlin/org/partiql/planner/util/PlanNodeEquivalentVisitor.kt +++ b/partiql-planner/src/test/kotlin/org/partiql/planner/util/PlanNodeEquivalentVisitor.kt @@ -125,13 +125,6 @@ class PlanNodeEquivalentVisitor : PlanBaseVisitor() { return true } - override fun visitRelOpExcludeStepCollIndex(node: Rel.Op.Exclude.Step.CollIndex, ctx: PlanNode): Boolean { - if (!super.visitRelOpExcludeStepCollIndex(node, ctx)) return false - ctx as Rel.Op.Exclude.Step.CollIndex - if (node.index != ctx.index) return false - return true - } - override fun visitRelOpErr(node: Rel.Op.Err, ctx: PlanNode): Boolean { if (!super.visitRelOpErr(node, ctx)) return false ctx as Rel.Op.Err diff --git a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/executor/EvalExecutor.kt b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/executor/EvalExecutor.kt index cc1a3c0aae..417eae52e0 100644 --- a/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/executor/EvalExecutor.kt +++ b/test/partiql-tests-runner/src/test/kotlin/org/partiql/runner/executor/EvalExecutor.kt @@ -12,7 +12,6 @@ import org.partiql.eval.PartiQLStatement import org.partiql.lang.eval.CompileOptions import org.partiql.parser.PartiQLParser import org.partiql.planner.PartiQLPlanner -import org.partiql.planner.PartiQLPlannerBuilder import org.partiql.plugins.memory.MemoryBindings import org.partiql.plugins.memory.MemoryConnector import org.partiql.runner.ION @@ -27,20 +26,9 @@ import org.partiql.value.toIon @OptIn(PartiQLValueExperimental::class) class EvalExecutor( - private val connector: Connector, private val session: PartiQLPlanner.Session, ) : TestExecutor, PartiQLResult> { - private val planner = PartiQLPlannerBuilder() - .addCatalog( - "test", - connector.getMetadata(object : ConnectorSession { - override fun getQueryId(): String = session.queryId - override fun getUserId(): String = session.userId - }) - ) - .build() - override fun prepare(statement: String): PartiQLStatement<*> { val stmt = parser.parse(statement).root val plan = planner.plan(stmt, session) @@ -53,6 +41,7 @@ class EvalExecutor( override fun fromIon(value: IonValue): PartiQLResult { val partiql = PartiQLValueIonReaderBuilder.standard().build(value.toIonElement()).read() + return PartiQLResult.Value(partiql) } @@ -72,6 +61,7 @@ class EvalExecutor( companion object { val parser = PartiQLParser.default() + val planner = PartiQLPlanner.default() val engine = PartiQLEngine.default() } @@ -88,8 +78,14 @@ class EvalExecutor( queryId = "query", userId = "user", currentCatalog = catalog, + catalogs = mapOf( + "test" to connector.getMetadata(object : ConnectorSession { + override fun getQueryId(): String = "query" + override fun getUserId(): String = "user" + }) + ) ) - return EvalExecutor(connector, session) + return EvalExecutor(session) } /** From 48455b72aebe7f7f1240ad750ac6f307de2ad376 Mon Sep 17 00:00:00 2001 From: Alan Cai Date: Mon, 22 Jan 2024 11:57:11 -0800 Subject: [PATCH 3/5] Some test corrections; additional comments --- .../eval/internal/operator/rel/RelSort.kt | 2 ++ .../eval/internal/util/NumberExtensions.kt | 5 ++-- .../eval/internal/PartiQLEngineDefaultTest.kt | 8 +++++++ .../util/PartiQLValueComparatorTest.kt | 24 +++++++++++-------- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt index 26facc7952..e27bbe426b 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt @@ -30,6 +30,8 @@ internal class RelSort( val lVal = spec.first.eval(l) val rVal = spec.first.eval(r) + // DESC_NULLS_FIRST(l, r) == ASC_NULLS_LAST(r, l) + // DESC_NULLS_LAST(l, r) == ASC_NULLS_FIRST(r, l) val cmpResult = when (spec.second) { Rel.Op.Sort.Order.ASC_NULLS_FIRST -> nullsFirstComparator.compare(lVal, rVal) Rel.Op.Sort.Order.ASC_NULLS_LAST -> nullsLastComparator.compare(lVal, rVal) diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/NumberExtensions.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/NumberExtensions.kt index f7f3f98e43..1479a1cd23 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/NumberExtensions.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/NumberExtensions.kt @@ -45,12 +45,13 @@ internal fun bigDecimalOf(num: Number, mc: MathContext = MATH_CONTEXT): BigDecim private val CONVERSION_MAP = mapOf>, Class>( setOf(Int::class.javaObjectType, Int::class.javaObjectType) to Int::class.javaObjectType, setOf(Int::class.javaObjectType, Long::class.javaObjectType) to Long::class.javaObjectType, - setOf(Int::class.javaObjectType, Float::class.javaObjectType) to Float::class.javaObjectType, + // Int w/ Float -> Double + setOf(Int::class.javaObjectType, Float::class.javaObjectType) to Double::class.javaObjectType, setOf(Int::class.javaObjectType, Double::class.javaObjectType) to Double::class.javaObjectType, setOf(Int::class.javaObjectType, BigDecimal::class.javaObjectType) to BigDecimal::class.javaObjectType, setOf(Float::class.javaObjectType, Float::class.javaObjectType) to Float::class.javaObjectType, - // Float w/ long -> Double + // Float w/ Long -> Double setOf(Float::class.javaObjectType, Long::class.javaObjectType) to Double::class.javaObjectType, setOf(Float::class.javaObjectType, Double::class.javaObjectType) to Double::class.javaObjectType, setOf(Float::class.javaObjectType, BigDecimal::class.javaObjectType) to BigDecimal::class.javaObjectType, diff --git a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt index cf60db5d64..bfbadeb3fc 100644 --- a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt +++ b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/PartiQLEngineDefaultTest.kt @@ -339,6 +339,14 @@ class PartiQLEngineDefaultTest { structValue("a" to int32Value(1), "b" to int32Value(2)) ) ), + SuccessTestCase( // use multiple sort specs + input = "SELECT * FROM <<{'a': NULL, 'b': 1}, {'a': 1, 'b': 2}, {'a': 1, 'b': 4}>> AS t ORDER BY t.a DESC NULLS FIRST, t.b DESC;", + expected = listValue( + structValue("a" to nullValue(), "b" to int32Value(1)), + structValue("a" to int32Value(1), "b" to int32Value(4)), + structValue("a" to int32Value(1), "b" to int32Value(2)) + ) + ), ) } public class SuccessTestCase @OptIn(PartiQLValueExperimental::class) constructor( diff --git a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/util/PartiQLValueComparatorTest.kt b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/util/PartiQLValueComparatorTest.kt index 66f8e77e14..dc819c0ecc 100644 --- a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/util/PartiQLValueComparatorTest.kt +++ b/partiql-eval/src/test/kotlin/org/partiql/eval/internal/util/PartiQLValueComparatorTest.kt @@ -93,6 +93,8 @@ class PartiQLValueComparatorTest { } } + // [EquivValues] in this list are sorted from ascending order per the less-than-order-by operator defined in spec + // section 12.2. Values within each [EquivValues] are equivalent. private val nullValues = listOf( EquivValues( nullValue(), // null @@ -106,12 +108,12 @@ class PartiQLValueComparatorTest { private val nonNullPartiQLValue = listOf( EquivValues( - boolValue(false), - boolValue(false, annotations = listOf("b")) + boolValue(false), // false + boolValue(false, annotations = listOf("b")) // `b::false` ), EquivValues( - boolValue(true, annotations = listOf("c")), - boolValue(true) + boolValue(true, annotations = listOf("c")), // `c::true` + boolValue(true) // true ), EquivValues( // make sure there are at least two nan @@ -119,13 +121,13 @@ class PartiQLValueComparatorTest { float64Value(Double.NaN), ), EquivValues( - // make sure there are at least two nan + // make sure there are at least two -inf float32Value(Float.NEGATIVE_INFINITY), float64Value(Double.NEGATIVE_INFINITY), ), EquivValues( - float32Value(-1e1000f), - float64Value(-1e1000) + float32Value(-1e1000f), // -inf + float64Value(-1e1000) // -inf ), EquivValues( float32Value(-5e-1f), @@ -169,6 +171,8 @@ class PartiQLValueComparatorTest { EquivValues( dateValue(date(year = 2021, month = 8, day = 22)) ), + // Set a [timeZone] for every [TimeValue] and [TimestampValue] since comparison between time types without + // a timezone results in an error. TODO: add a way to compare between time and timestamp types EquivValues( timeValue(time(hour = 12, minute = 12, second = 12, timeZone = TimeZone.UnknownTimeZone)), timeValue(time(hour = 12, minute = 12, second = 12, nano = 0, timeZone = TimeZone.UnknownTimeZone)), @@ -276,8 +280,8 @@ class PartiQLValueComparatorTest { sexpValue(emptyList(), annotations = listOf("a", "b", "c")) // `a::b::c::()` ), EquivValues( - sexpValue(float32Value(1f)), // "`a::b::c::(1e0)`" - sexpValue(float64Value(1.0), annotations = listOf("a", "b", "c")), // "`a::b::c::(1e0)`" + sexpValue(float32Value(1f)), // `a::b::c::(1e0)` + sexpValue(float64Value(1.0), annotations = listOf("a", "b", "c")), // `a::b::c::(1e0)` sexpValue(int32Value(1)), // `(1)` sexpValue(decimalValue(BigDecimal("1.0000000000000"))) // `(1.0000000000000)` ), @@ -362,7 +366,7 @@ class PartiQLValueComparatorTest { ), EquivValues( structValue( // { 'm': [2, 2], 'n': [2, 2]} - "m" to listValue(int32Value(1), int32Value(2)), + "m" to listValue(int32Value(2), int32Value(2)), "n" to listValue(int32Value(2), int32Value(2)) ) ), From cbbdc33ce3749c3d04d4a8e2778f8382d1d9e1ba Mon Sep 17 00:00:00 2001 From: Alan Cai Date: Tue, 23 Jan 2024 14:02:48 -0800 Subject: [PATCH 4/5] Move PartiQLValueComparator to partiql-types and make a public fn --- .../eval/internal/operator/rel/RelSort.kt | 11 +++-- .../partiql/value}/PartiQLValueComparator.kt | 46 +++++++++---------- .../partiql/value}/util/NumberExtensions.kt | 2 +- .../value}/PartiQLValueComparatorTest.kt | 33 ++----------- 4 files changed, 32 insertions(+), 60 deletions(-) rename {partiql-eval/src/main/kotlin/org/partiql/eval/internal/util => partiql-types/src/main/kotlin/org/partiql/value}/PartiQLValueComparator.kt (90%) rename {partiql-eval/src/main/kotlin/org/partiql/eval/internal => partiql-types/src/main/kotlin/org/partiql/value}/util/NumberExtensions.kt (99%) rename {partiql-eval/src/test/kotlin/org/partiql/eval/internal/util => partiql-types/src/test/kotlin/org/partiql/value}/PartiQLValueComparatorTest.kt (92%) diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt index e27bbe426b..f363f06648 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt @@ -2,10 +2,12 @@ package org.partiql.eval.internal.operator.rel import org.partiql.eval.internal.Record import org.partiql.eval.internal.operator.Operator -import org.partiql.eval.internal.util.PartiQLValueComparator import org.partiql.plan.Rel +import org.partiql.value.NullOrder +import org.partiql.value.PartiQLValueComparator import org.partiql.value.PartiQLValueExperimental +@OptIn(PartiQLValueExperimental::class) internal class RelSort( val input: Operator.Relation, val specs: List> @@ -14,8 +16,8 @@ internal class RelSort( private var records: MutableList = mutableListOf() private var init: Boolean = false - private val nullsFirstComparator = PartiQLValueComparator(nullOrder = PartiQLValueComparator.NullOrder.FIRST) - private val nullsLastComparator = PartiQLValueComparator(nullOrder = PartiQLValueComparator.NullOrder.LAST) + private val nullsFirstComparator = PartiQLValueComparator.comparator(NullOrder.FIRST) + private val nullsLastComparator = PartiQLValueComparator.comparator(NullOrder.LAST) override fun open() { input.open() @@ -23,8 +25,7 @@ internal class RelSort( records = mutableListOf() } - @OptIn(PartiQLValueExperimental::class) - val comparator = object : Comparator { + private val comparator = object : Comparator { override fun compare(l: Record, r: Record): Int { specs.forEach { spec -> val lVal = spec.first.eval(l) diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/PartiQLValueComparator.kt b/partiql-types/src/main/kotlin/org/partiql/value/PartiQLValueComparator.kt similarity index 90% rename from partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/PartiQLValueComparator.kt rename to partiql-types/src/main/kotlin/org/partiql/value/PartiQLValueComparator.kt index 17d8233f77..3c4acc65fc 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/PartiQLValueComparator.kt +++ b/partiql-types/src/main/kotlin/org/partiql/value/PartiQLValueComparator.kt @@ -1,22 +1,10 @@ -package org.partiql.eval.internal.util +package org.partiql.value -import org.partiql.value.BagValue -import org.partiql.value.BlobValue -import org.partiql.value.BoolValue -import org.partiql.value.ClobValue -import org.partiql.value.DateValue -import org.partiql.value.ListValue -import org.partiql.value.MissingValue -import org.partiql.value.NullValue -import org.partiql.value.NumericValue -import org.partiql.value.PartiQLValue -import org.partiql.value.PartiQLValueExperimental -import org.partiql.value.ScalarValue -import org.partiql.value.SexpValue -import org.partiql.value.StructValue -import org.partiql.value.TextValue -import org.partiql.value.TimeValue -import org.partiql.value.TimestampValue +import org.partiql.value.util.compareTo +import org.partiql.value.util.isNaN +import org.partiql.value.util.isNegInf +import org.partiql.value.util.isPosInf +import org.partiql.value.util.isZero /** * Provides a total, natural ordering over [PartiQLValue] as defined by section 12.2 of the PartiQL specification @@ -48,17 +36,25 @@ import org.partiql.value.TimestampValue * (as defined by this definition) members, as pairs of field name and the member value. * * [BagValue] values come finally (except with [NullOrder.LAST]), and their values * compare lexicographically based on the *sorted* child elements. - * - * @param nullOrder that places [NullValue], [MissingValue], and typed Ion null values first or last */ @OptIn(PartiQLValueExperimental::class) -internal class PartiQLValueComparator(private val nullOrder: NullOrder) : Comparator { - /** Whether null values come first or last. */ - enum class NullOrder { - FIRST, - LAST +public class PartiQLValueComparator { + public companion object { + /** + * @param nullOrder that places [NullValue], [MissingValue], and typed Ion null values first or last + */ + public fun comparator(nullOrder: NullOrder): Comparator = PartiQLValueComparatorInternal(nullOrder) } +} +/** Whether null values come first or last. */ +public enum class NullOrder { + FIRST, + LAST +} + +@OptIn(PartiQLValueExperimental::class) +internal class PartiQLValueComparatorInternal(private val nullOrder: NullOrder) : Comparator { private val EQUAL = 0 private val LESS = -1 private val GREATER = 1 diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/NumberExtensions.kt b/partiql-types/src/main/kotlin/org/partiql/value/util/NumberExtensions.kt similarity index 99% rename from partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/NumberExtensions.kt rename to partiql-types/src/main/kotlin/org/partiql/value/util/NumberExtensions.kt index 1479a1cd23..cab5a82d2a 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/util/NumberExtensions.kt +++ b/partiql-types/src/main/kotlin/org/partiql/value/util/NumberExtensions.kt @@ -1,4 +1,4 @@ -package org.partiql.eval.internal.util +package org.partiql.value.util /* * Copyright 2019 Amazon.com, Inc. or its affiliates. All rights reserved. diff --git a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/util/PartiQLValueComparatorTest.kt b/partiql-types/src/test/kotlin/org/partiql/value/PartiQLValueComparatorTest.kt similarity index 92% rename from partiql-eval/src/test/kotlin/org/partiql/eval/internal/util/PartiQLValueComparatorTest.kt rename to partiql-types/src/test/kotlin/org/partiql/value/PartiQLValueComparatorTest.kt index dc819c0ecc..fa2cfa564b 100644 --- a/partiql-eval/src/test/kotlin/org/partiql/eval/internal/util/PartiQLValueComparatorTest.kt +++ b/partiql-types/src/test/kotlin/org/partiql/value/PartiQLValueComparatorTest.kt @@ -1,36 +1,11 @@ -package org.partiql.eval.internal.util +package org.partiql.value import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test -import org.partiql.value.BagValue -import org.partiql.value.ListValue -import org.partiql.value.PartiQLValue -import org.partiql.value.PartiQLValueExperimental -import org.partiql.value.StructValue -import org.partiql.value.bagValue -import org.partiql.value.blobValue -import org.partiql.value.boolValue -import org.partiql.value.clobValue -import org.partiql.value.dateValue import org.partiql.value.datetime.DateTimeValue.date import org.partiql.value.datetime.DateTimeValue.time import org.partiql.value.datetime.DateTimeValue.timestamp import org.partiql.value.datetime.TimeZone -import org.partiql.value.decimalValue -import org.partiql.value.float32Value -import org.partiql.value.float64Value -import org.partiql.value.int32Value -import org.partiql.value.int64Value -import org.partiql.value.listValue -import org.partiql.value.missingValue -import org.partiql.value.nullValue -import org.partiql.value.sexpValue -import org.partiql.value.stringValue -import org.partiql.value.structValue -import org.partiql.value.symbolValue -import org.partiql.value.timeValue -import org.partiql.value.timestampValue -import org.partiql.value.toIon import java.math.BigDecimal import java.util.Base64 import java.util.Random @@ -39,8 +14,8 @@ import java.util.Random class PartiQLValueComparatorTest { class EquivValues(vararg val values: PartiQLValue) - private val nullsFirstComparator = PartiQLValueComparator(nullOrder = PartiQLValueComparator.NullOrder.FIRST) - private val nullsLastComparator = PartiQLValueComparator(nullOrder = PartiQLValueComparator.NullOrder.LAST) + private val nullsFirstComparator = PartiQLValueComparator.comparator(nullOrder = NullOrder.FIRST) + private val nullsLastComparator = PartiQLValueComparator.comparator(nullOrder = NullOrder.LAST) // TODO consider replacing linear congruential generator with something else (e.g. xorshift) // RNG for fuzz testing the sort orders, the seed is arbitrary but static for determinism @@ -53,7 +28,7 @@ class PartiQLValueComparatorTest { private fun base64Decode(s: String): ByteArray = Base64.getDecoder().decode(s) // Checks that [allValues], when shuffled and sorted using [comparator], follow same ordering as [allValues] - private fun checkAllEquivalent(allValues: List, comparator: PartiQLValueComparator) { + private fun checkAllEquivalent(allValues: List, comparator: Comparator) { val shuffledValues = allValues.shuffled(Random(SEED)) val sortedAfterShuffle = shuffledValues.sortedWith(comparator) assertEquals(allValues.size, sortedAfterShuffle.size) From 792dabeb4605e568e50880904ce9f66e61aed69a Mon Sep 17 00:00:00 2001 From: Alan Cai Date: Mon, 5 Feb 2024 12:13:37 -0800 Subject: [PATCH 5/5] Change RelSort to use Iterator; move comparator to PartiQLValue --- .../eval/internal/operator/rel/RelOffset.kt | 1 + .../eval/internal/operator/rel/RelSort.kt | 29 +++++---- .../kotlin/org/partiql/value/PartiQLValue.kt | 38 ++++++++++++ ...r.kt => PartiQLValueComparatorInternal.kt} | 61 +++---------------- .../value/PartiQLValueComparatorTest.kt | 4 +- 5 files changed, 64 insertions(+), 69 deletions(-) rename partiql-types/src/main/kotlin/org/partiql/value/{PartiQLValueComparator.kt => PartiQLValueComparatorInternal.kt} (69%) diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelOffset.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelOffset.kt index 3801aa2002..97f2f3cea8 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelOffset.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelOffset.kt @@ -23,6 +23,7 @@ internal class RelOffset( input.next() ?: return null seen += 1 } + init = true } return input.next() } diff --git a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt index f363f06648..9bbf1bdfee 100644 --- a/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt +++ b/partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelSort.kt @@ -3,26 +3,26 @@ package org.partiql.eval.internal.operator.rel import org.partiql.eval.internal.Record import org.partiql.eval.internal.operator.Operator import org.partiql.plan.Rel -import org.partiql.value.NullOrder -import org.partiql.value.PartiQLValueComparator +import org.partiql.value.PartiQLValue import org.partiql.value.PartiQLValueExperimental +import java.util.Collections @OptIn(PartiQLValueExperimental::class) internal class RelSort( - val input: Operator.Relation, - val specs: List> + private val input: Operator.Relation, + private val specs: List> ) : Operator.Relation { - private var records: MutableList = mutableListOf() + private var records: Iterator = Collections.emptyIterator() private var init: Boolean = false - private val nullsFirstComparator = PartiQLValueComparator.comparator(NullOrder.FIRST) - private val nullsLastComparator = PartiQLValueComparator.comparator(NullOrder.LAST) + private val nullsFirstComparator = PartiQLValue.comparator(nullsFirst = true) + private val nullsLastComparator = PartiQLValue.comparator(nullsFirst = false) override fun open() { input.open() init = false - records = mutableListOf() + records = Collections.emptyIterator() } private val comparator = object : Comparator { @@ -49,15 +49,18 @@ internal class RelSort( override fun next(): Record? { if (!init) { + val sortedRecords = mutableListOf() while (true) { val row = input.next() ?: break - records.add(row) + sortedRecords.add(row) } - records.sortWith(comparator) + sortedRecords.sortWith(comparator) + records = sortedRecords.iterator() + init = true } - return when (records.isEmpty()) { - true -> null - else -> records.removeAt(0) + return when (records.hasNext()) { + true -> records.next() + false -> null } } diff --git a/partiql-types/src/main/kotlin/org/partiql/value/PartiQLValue.kt b/partiql-types/src/main/kotlin/org/partiql/value/PartiQLValue.kt index 10a9a8fa56..5df8b2dc41 100644 --- a/partiql-types/src/main/kotlin/org/partiql/value/PartiQLValue.kt +++ b/partiql-types/src/main/kotlin/org/partiql/value/PartiQLValue.kt @@ -48,6 +48,44 @@ public sealed interface PartiQLValue { public fun withoutAnnotations(): PartiQLValue public fun accept(visitor: PartiQLValueVisitor, ctx: C): R + + public companion object { + /** + * Provides a total, natural ordering over [PartiQLValue] as defined by section 12.2 of the PartiQL specification + * (https://partiql.org/assets/PartiQL-Specification.pdf#subsection.12.2). PartiQL treats Ion typed nulls as `NULL` + * for the purposes of comparisons and Ion annotations are not considered for comparison purposes. + * + * The ordering rules are as follows: + * + * * [NullValue] and [MissingValue] are always first or last and compare equally. In other words, + * comparison cannot distinguish between `NULL` or `MISSING`. + * * The [BoolValue] values follow with `false` coming before `true`. + * * The [NumericValue] types come next ordered by their numerical value irrespective + * of precision or specific type. + * For `FLOAT` special values, `nan` comes before `-inf`, which comes before all normal + * numeric values, which is followed by `+inf`. + * * [DateValue] values follow and are compared by the date from earliest to latest. + * * [TimeValue] values follow and are compared by the time of the day (point of time in a day of 24 hours) + * from earliest to latest. Note that time without time zone is not directly comparable with time with time zone. + * * [TimestampValue] values follow and are compared by the point of time irrespective of precision and + * local UTC offset. + * * The [TextValue] types come next ordered by their lexicographical ordering by + * Unicode scalar irrespective of their specific type. + * * The [BlobValue] and [ClobValue] types follow and are ordered by their lexicographical ordering + * by octet. + * * [ListValue] comes next, and their values compare lexicographically based on their + * child elements recursively based on this definition. + * * [SexpValue] follows and compares within its type similar to `LIST`. + * * [StructValue] values follow and compare lexicographically based on the *sorted* + * (as defined by this definition) members, as pairs of field name and the member value. + * * [BagValue] values come finally (except with [nullsFirst] == true), and their values + * compare lexicographically based on the *sorted* child elements. + * + * @param nullsFirst whether [NullValue], [MissingValue], and typed Ion null values come first + */ + @JvmStatic + public fun comparator(nullsFirst: Boolean): Comparator = PartiQLValueComparatorInternal(nullsFirst) + } } @PartiQLValueExperimental diff --git a/partiql-types/src/main/kotlin/org/partiql/value/PartiQLValueComparator.kt b/partiql-types/src/main/kotlin/org/partiql/value/PartiQLValueComparatorInternal.kt similarity index 69% rename from partiql-types/src/main/kotlin/org/partiql/value/PartiQLValueComparator.kt rename to partiql-types/src/main/kotlin/org/partiql/value/PartiQLValueComparatorInternal.kt index 3c4acc65fc..fc770fc652 100644 --- a/partiql-types/src/main/kotlin/org/partiql/value/PartiQLValueComparator.kt +++ b/partiql-types/src/main/kotlin/org/partiql/value/PartiQLValueComparatorInternal.kt @@ -6,55 +6,8 @@ import org.partiql.value.util.isNegInf import org.partiql.value.util.isPosInf import org.partiql.value.util.isZero -/** - * Provides a total, natural ordering over [PartiQLValue] as defined by section 12.2 of the PartiQL specification - * (https://partiql.org/assets/PartiQL-Specification.pdf#subsection.12.2). PartiQL treats Ion typed nulls as `NULL` - * for the purposes of comparisons and Ion annotations are not considered for comparison purposes. - * - * The ordering rules are as follows: - * - * * [NullValue] and [MissingValue] are always first or last and compare equally. In other words, - * comparison cannot distinguish between `NULL` or `MISSING`. - * * The [BoolValue] values follow with `false` coming before `true`. - * * The [NumericValue] types come next ordered by their numerical value irrespective - * of precision or specific type. - * For `FLOAT` special values, `nan` comes before `-inf`, which comes before all normal - * numeric values, which is followed by `+inf`. - * * [DateValue] values follow and are compared by the date from earliest to latest. - * * [TimeValue] values follow and are compared by the time of the day (point of time in a day of 24 hours) - * from earliest to latest. Note that time without time zone is not directly comparable with time with time zone. - * * [TimestampValue] values follow and are compared by the point of time irrespective of precision and - * local UTC offset. - * * The [TextValue] types come next ordered by their lexicographical ordering by - * Unicode scalar irrespective of their specific type. - * * The [BlobValue] and [ClobValue] types follow and are ordered by their lexicographical ordering - * by octet. - * * [ListValue] comes next, and their values compare lexicographically based on their - * child elements recursively based on this definition. - * * [SexpValue] follows and compares within its type similar to `LIST`. - * * [StructValue] values follow and compare lexicographically based on the *sorted* - * (as defined by this definition) members, as pairs of field name and the member value. - * * [BagValue] values come finally (except with [NullOrder.LAST]), and their values - * compare lexicographically based on the *sorted* child elements. - */ @OptIn(PartiQLValueExperimental::class) -public class PartiQLValueComparator { - public companion object { - /** - * @param nullOrder that places [NullValue], [MissingValue], and typed Ion null values first or last - */ - public fun comparator(nullOrder: NullOrder): Comparator = PartiQLValueComparatorInternal(nullOrder) - } -} - -/** Whether null values come first or last. */ -public enum class NullOrder { - FIRST, - LAST -} - -@OptIn(PartiQLValueExperimental::class) -internal class PartiQLValueComparatorInternal(private val nullOrder: NullOrder) : Comparator { +internal class PartiQLValueComparatorInternal(private val nullsFirst: Boolean) : Comparator { private val EQUAL = 0 private val LESS = -1 private val GREATER = 1 @@ -72,7 +25,7 @@ internal class PartiQLValueComparatorInternal(private val nullOrder: NullOrder) } } - private fun compareInternal(l: PartiQLValue, r: PartiQLValue, nullsFirst: NullOrder): Int { + private fun compareInternal(l: PartiQLValue, r: PartiQLValue, nullsFirst: Boolean): Int { if (l.withoutAnnotations() == r.withoutAnnotations()) { return EQUAL } @@ -80,12 +33,12 @@ internal class PartiQLValueComparatorInternal(private val nullOrder: NullOrder) when { l.isNullOrMissing() && r.isNullOrMissing() -> return EQUAL l.isNullOrMissing() -> return when (nullsFirst) { - NullOrder.FIRST -> LESS - NullOrder.LAST -> GREATER + true -> LESS + false -> GREATER } r.isNullOrMissing() -> return when (nullsFirst) { - NullOrder.FIRST -> GREATER - NullOrder.LAST -> LESS + true -> GREATER + false -> LESS } } @@ -263,6 +216,6 @@ internal class PartiQLValueComparatorInternal(private val nullOrder: NullOrder) } override fun compare(l: PartiQLValue, r: PartiQLValue): Int { - return compareInternal(l, r, nullOrder) + return compareInternal(l, r, nullsFirst) } } diff --git a/partiql-types/src/test/kotlin/org/partiql/value/PartiQLValueComparatorTest.kt b/partiql-types/src/test/kotlin/org/partiql/value/PartiQLValueComparatorTest.kt index fa2cfa564b..b0a23be177 100644 --- a/partiql-types/src/test/kotlin/org/partiql/value/PartiQLValueComparatorTest.kt +++ b/partiql-types/src/test/kotlin/org/partiql/value/PartiQLValueComparatorTest.kt @@ -14,8 +14,8 @@ import java.util.Random class PartiQLValueComparatorTest { class EquivValues(vararg val values: PartiQLValue) - private val nullsFirstComparator = PartiQLValueComparator.comparator(nullOrder = NullOrder.FIRST) - private val nullsLastComparator = PartiQLValueComparator.comparator(nullOrder = NullOrder.LAST) + private val nullsFirstComparator = PartiQLValue.comparator(nullsFirst = true) + private val nullsLastComparator = PartiQLValue.comparator(nullsFirst = false) // TODO consider replacing linear congruential generator with something else (e.g. xorshift) // RNG for fuzz testing the sort orders, the seed is arbitrary but static for determinism