diff --git a/lang/resources/org/partiql/type-domains/partiql.ion b/lang/resources/org/partiql/type-domains/partiql.ion index 7638162967..09d96fa34d 100644 --- a/lang/resources/org/partiql/type-domains/partiql.ion +++ b/lang/resources/org/partiql/type-domains/partiql.ion @@ -216,8 +216,8 @@ // ORDER BY ... (product order_by sort_specs::(* sort_spec 1)) - // [ASC | DESC] ? - (product sort_spec expr::expr ordering_spec::(? ordering_spec)) + // [ASC | DESC] [NULLS FIRST | NULLS LAST] + (product sort_spec expr::expr ordering_spec::(? ordering_spec) nulls_spec::(? nulls_spec)) // Desired ordering spec: ASC or DESC (sum ordering_spec @@ -225,6 +225,12 @@ (desc) ) + // Desired null/missing ordering spec: NULLS FIRST or NULLS LAST + (sum nulls_spec + (nulls_first) + (nulls_last) + ) + // Indicates if variable lookup should be case-sensitive or not. (sum case_sensitivity (case_sensitive) (case_insensitive)) diff --git a/lang/src/org/partiql/lang/ast/ExprNodeToStatement.kt b/lang/src/org/partiql/lang/ast/ExprNodeToStatement.kt index 71d8f5e026..119295ac33 100644 --- a/lang/src/org/partiql/lang/ast/ExprNodeToStatement.kt +++ b/lang/src/org/partiql/lang/ast/ExprNodeToStatement.kt @@ -225,16 +225,24 @@ private fun OrderBy.toAstOrderBySpec(): PartiqlAst.OrderBy { val thiz = this return PartiqlAst.build { orderBy( - thiz.sortSpecItems.map { sortSpec(it.expr.toAstExpr(), it.orderingSpec.toAstOrderSpec()) } + thiz.sortSpecItems.map { sortSpec(it.expr.toAstExpr(), it.orderingSpec?.toAstOrderSpec(), it.nullsSpec?.toAstNullsSpec()) } ) } } -private fun OrderingSpec?.toAstOrderSpec(): PartiqlAst.OrderingSpec = +private fun OrderingSpec.toAstOrderSpec(): PartiqlAst.OrderingSpec = PartiqlAst.build { when (this@toAstOrderSpec) { + OrderingSpec.ASC -> asc() OrderingSpec.DESC -> desc() - else -> asc() + } + } + +private fun NullsSpec.toAstNullsSpec(): PartiqlAst.NullsSpec = + PartiqlAst.build { + when (this@toAstNullsSpec) { + NullsSpec.FIRST -> nullsFirst() + NullsSpec.LAST -> nullsLast() } } diff --git a/lang/src/org/partiql/lang/ast/StatementToExprNode.kt b/lang/src/org/partiql/lang/ast/StatementToExprNode.kt index 361552a76f..0fd6bcc832 100644 --- a/lang/src/org/partiql/lang/ast/StatementToExprNode.kt +++ b/lang/src/org/partiql/lang/ast/StatementToExprNode.kt @@ -295,15 +295,22 @@ private class StatementTransformer(val ion: IonSystem) { sortSpecItems = this.sortSpecs.map { SortSpec( it.expr.toExprNode(), - it.orderingSpec.toOrderSpec() + it.orderingSpec?.toOrderSpec(), + it.nullsSpec?.toNullsSpec() ) } ) - private fun PartiqlAst.OrderingSpec?.toOrderSpec(): OrderingSpec = + private fun PartiqlAst.OrderingSpec.toOrderSpec(): OrderingSpec = when (this) { + is PartiqlAst.OrderingSpec.Asc -> OrderingSpec.ASC is PartiqlAst.OrderingSpec.Desc -> OrderingSpec.DESC - else -> OrderingSpec.ASC + } + + private fun PartiqlAst.NullsSpec.toNullsSpec(): NullsSpec = + when (this) { + is PartiqlAst.NullsSpec.NullsFirst -> NullsSpec.FIRST + is PartiqlAst.NullsSpec.NullsLast -> NullsSpec.LAST } private fun PartiqlAst.GroupBy.toGroupBy(): GroupBy = diff --git a/lang/src/org/partiql/lang/ast/ast.kt b/lang/src/org/partiql/lang/ast/ast.kt index ece71514f5..4f5013efc8 100644 --- a/lang/src/org/partiql/lang/ast/ast.kt +++ b/lang/src/org/partiql/lang/ast/ast.kt @@ -847,7 +847,8 @@ data class OrderBy( data class SortSpec( val expr: ExprNode, - val orderingSpec: OrderingSpec + val orderingSpec: OrderingSpec?, + val nullsSpec: NullsSpec? ) : AstNode() { override val children: List = listOf(expr) } @@ -1055,6 +1056,13 @@ enum class OrderingSpec { DESC } +/** Nulls specification */ +enum class NullsSpec { + /** Represents whether null or missing values are placed before non-null values */ + FIRST, + LAST +} + /** * AST Node corresponding to the DATE literal * diff --git a/lang/src/org/partiql/lang/ast/passes/AstRewriterBase.kt b/lang/src/org/partiql/lang/ast/passes/AstRewriterBase.kt index 24f737b351..043659a43c 100644 --- a/lang/src/org/partiql/lang/ast/passes/AstRewriterBase.kt +++ b/lang/src/org/partiql/lang/ast/passes/AstRewriterBase.kt @@ -428,7 +428,8 @@ open class AstRewriterBase : AstRewriter { open fun rewriteSortSpec(sortSpec: SortSpec): SortSpec = SortSpec( rewriteExprNode(sortSpec.expr), - sortSpec.orderingSpec + sortSpec.orderingSpec, + sortSpec.nullsSpec ) open fun rewriteDataType(dataType: DataType) = dataType diff --git a/lang/src/org/partiql/lang/errors/ErrorCode.kt b/lang/src/org/partiql/lang/errors/ErrorCode.kt index 7ca7bb78fe..256be64e65 100644 --- a/lang/src/org/partiql/lang/errors/ErrorCode.kt +++ b/lang/src/org/partiql/lang/errors/ErrorCode.kt @@ -557,6 +557,22 @@ enum class ErrorCode( "No such function: ${errorContext?.get(Property.FUNCTION_NAME)?.stringValue() ?: UNKNOWN} " }, + EVALUATOR_ORDER_BY_NULL_COMPARATOR( + ErrorCategory.EVALUATOR, + LOCATION, + "" + ) { + override fun getErrorMessage(errorContext: PropertyValueMap?): String = "" + }, + + EVALUATOR_ENVIRONMENT_CANNOT_BE_RESOLVED( + ErrorCategory.EVALUATOR, + LOCATION, + "" + ) { + override fun getErrorMessage(errorContext: PropertyValueMap?): String = "" + }, + SEMANTIC_DUPLICATE_ALIASES_IN_SELECT_LIST_ITEM( ErrorCategory.SEMANTIC, LOCATION, diff --git a/lang/src/org/partiql/lang/eval/EvaluatingCompiler.kt b/lang/src/org/partiql/lang/eval/EvaluatingCompiler.kt index 4419c32b5c..a97ba5cbcc 100644 --- a/lang/src/org/partiql/lang/eval/EvaluatingCompiler.kt +++ b/lang/src/org/partiql/lang/eval/EvaluatingCompiler.kt @@ -68,6 +68,7 @@ import org.partiql.lang.util.compareTo import org.partiql.lang.util.div import org.partiql.lang.util.drop import org.partiql.lang.util.foldLeftProduct +import org.partiql.lang.util.interruptibleFold import org.partiql.lang.util.isZero import org.partiql.lang.util.minus import org.partiql.lang.util.plus @@ -83,6 +84,7 @@ import java.math.BigDecimal import java.util.LinkedList import java.util.Stack import java.util.TreeSet +import kotlin.Comparator /** * A basic compiler that converts an instance of [PartiqlAst] to an [Expression]. @@ -170,6 +172,9 @@ internal class EvaluatingCompiler( */ private data class FromSourceBindingNamePair(val bindingName: BindingName, val nameExprValue: ExprValue) + /** Represents an instance of a compiled `ORDER BY` expression, orderingSpec and nulls type. */ + private class CompiledOrderByItem(val comparator: NaturalExprValueComparators, val thunk: ThunkEnv) + /** * Base class for [ExprAggregator] instances which accumulate values and perform a final computation. */ @@ -1643,14 +1648,6 @@ internal class EvaluatingCompiler( } private fun compileSelect(selectExpr: PartiqlAst.Expr.Select, metas: MetaContainer): ThunkEnv { - selectExpr.order?.let { - err( - "ORDER BY is not supported in evaluator yet", - ErrorCode.EVALUATOR_FEATURE_NOT_SUPPORTED_YET, - errorContextFrom(metas).also { it[Property.FEATURE_NAME] = "ORDER BY" }, - internal = false - ) - } // Get all the FROM source aliases and LET bindings for binding error checks val fold = object : PartiqlAst.VisitorFold>() { @@ -1679,6 +1676,9 @@ internal class EvaluatingCompiler( val letSourceThunks = selectExpr.fromLet?.let { compileLetSources(it) } val sourceThunks = compileQueryWithoutProjection(selectExpr, fromSourceThunks, letSourceThunks) + val orderByThunk = selectExpr.order?.let { compileOrderByExpression(selectExpr.order.sortSpecs) } + val orderByLocationMeta = selectExpr.order?.metas?.sourceLocation + val offsetThunk = selectExpr.offset?.let { compileAstExpr(it) } val offsetLocationMeta = selectExpr.offset?.metas?.sourceLocation @@ -1709,7 +1709,12 @@ internal class EvaluatingCompiler( // Grouping is not needed -- simply project the results from the FROM clause directly. thunkFactory.thunkEnv(metas) { env -> - val projectedRows = sourceThunks(env).map { (joinedValues, projectEnv) -> + val orderedRows = when (orderByThunk) { + null -> sourceThunks(env) + else -> evalOrderBy(sourceThunks(env), orderByThunk, orderByLocationMeta) + } + + val projectedRows = orderedRows.map { (joinedValues, projectEnv) -> selectProjectionThunk(projectEnv, joinedValues) } @@ -1719,13 +1724,17 @@ internal class EvaluatingCompiler( is PartiqlAst.SetQuantifier.All -> projectedRows }.let { rowsWithOffsetAndLimit(it, env) } - valueFactory.newBag( - quantifiedRows.map { - // TODO make this expose the ordinal for ordered sequences - // make sure we don't expose the underlying value's name out of a SELECT - it.unnamedValue() - } - ) + // if order by is specified, return list otherwise bag + when (orderByThunk) { + null -> valueFactory.newBag( + quantifiedRows.map { + // TODO make this expose the ordinal for ordered sequences + // make sure we don't expose the underlying value's name out of a SELECT + it.unnamedValue() + } + ) + else -> valueFactory.newList(quantifiedRows.map { it.unnamedValue() }) + } } else -> { // Grouping is needed @@ -1766,8 +1775,13 @@ internal class EvaluatingCompiler( // Create a closure that groups all the rows in the FROM source into a single group. thunkFactory.thunkEnv(metas) { env -> // Evaluate the FROM clause + val orderedRows = when (orderByThunk) { + null -> sourceThunks(env) + else -> evalOrderBy(sourceThunks(env), orderByThunk, orderByLocationMeta) + } + val fromProductions: Sequence = - rowsWithOffsetAndLimit(sourceThunks(env), env) + rowsWithOffsetAndLimit(orderedRows, env) val registers = createRegisterBank() // note: the group key can be anything here because we only ever have a single @@ -1788,7 +1802,11 @@ internal class EvaluatingCompiler( listOf(syntheticGroup.key) ) - valueFactory.newBag(listOf(groupResult).asSequence()) + // if order by is specified, return list otherwise bag + when (orderByThunk) { + null -> valueFactory.newBag(listOf(groupResult).asSequence()) + else -> valueFactory.newList(listOf(groupResult).asSequence()) + } } } else -> { @@ -1852,13 +1870,22 @@ internal class EvaluatingCompiler( } } + val groupByEnvValuePairs = env.groups.mapNotNull { g -> getGroupEnv(env, g.value) to g.value }.asSequence() + val orderedGroupEnvPairs = when (orderByThunk) { + null -> groupByEnvValuePairs + else -> evalOrderBy(groupByEnvValuePairs, orderByThunk, orderByLocationMeta) + } + // generate the final group by projection - val projectedRows = env.groups.mapNotNull { g -> - val groupByEnv = getGroupEnv(env, g.value) - filterHavingAndProject(groupByEnv, g.value) + val projectedRows = orderedGroupEnvPairs.mapNotNull { (groupByEnv, groupValue) -> + filterHavingAndProject(groupByEnv, groupValue) }.asSequence().let { rowsWithOffsetAndLimit(it, env) } - valueFactory.newBag(projectedRows) + // if order by is specified, return list otherwise bag + when (orderByThunk) { + null -> valueFactory.newBag(projectedRows) + else -> valueFactory.newList(projectedRows) + } } } } @@ -1998,6 +2025,92 @@ internal class EvaluatingCompiler( GroupKeyExprValue(valueFactory.ion, keyValues.asSequence(), uniqueNames) } + private fun compileOrderByExpression(sortSpecs: List): List = + sortSpecs.map { + it.orderingSpec + ?: errNoContext( + "SortSpec.orderingSpec was not specified", + errorCode = ErrorCode.INTERNAL_ERROR, + internal = true + ) + + it.nullsSpec + ?: errNoContext( + "SortSpec.nullsSpec was not specified", + errorCode = ErrorCode.INTERNAL_ERROR, + internal = true + ) + + val comparator = when (it.orderingSpec) { + is PartiqlAst.OrderingSpec.Asc -> + when (it.nullsSpec) { + is PartiqlAst.NullsSpec.NullsFirst -> NaturalExprValueComparators.NULLS_FIRST_ASC + is PartiqlAst.NullsSpec.NullsLast -> NaturalExprValueComparators.NULLS_LAST_ASC + } + is PartiqlAst.OrderingSpec.Desc -> + when (it.nullsSpec) { + is PartiqlAst.NullsSpec.NullsFirst -> NaturalExprValueComparators.NULLS_FIRST_DESC + is PartiqlAst.NullsSpec.NullsLast -> NaturalExprValueComparators.NULLS_LAST_DESC + } + } + + CompiledOrderByItem(comparator, compileAstExpr(it.expr)) + } + + private fun evalOrderBy( + rows: Sequence, + orderByItems: List, + offsetLocationMeta: SourceLocationMeta? + ): Sequence { + val initialComparator: Comparator? = null + val resultComparator = orderByItems.interruptibleFold(initialComparator) { intermediateComparator, orderByItem -> + if (intermediateComparator == null) { + return@interruptibleFold compareBy(orderByItem.comparator) { row -> + val env = resolveEnvironment(row, offsetLocationMeta) + orderByItem.thunk(env) + } + } + + return@interruptibleFold intermediateComparator.thenBy(orderByItem.comparator) { row -> + val env = resolveEnvironment(row, offsetLocationMeta) + orderByItem.thunk(env) + } + } ?: err( + "Order BY comparator cannot be null", + ErrorCode.EVALUATOR_ORDER_BY_NULL_COMPARATOR, + null, + internal = true + ) + + return rows.sortedWith(resultComparator) + } + + private fun resolveEnvironment(envWrapper: T, offsetLocationMeta: SourceLocationMeta?): Environment { + return when (envWrapper) { + is FromProduction -> envWrapper.env + is Pair<*, *> -> { + if (envWrapper.first is Environment) { + envWrapper.first as Environment + } else if (envWrapper.second is Environment) { + envWrapper.second as Environment + } else { + err( + "Environment cannot be resolved from pair", + ErrorCode.EVALUATOR_ENVIRONMENT_CANNOT_BE_RESOLVED, + errorContextFrom(offsetLocationMeta), + internal = true + ) + } + } + else -> err( + "Environment cannot be resolved", + ErrorCode.EVALUATOR_ENVIRONMENT_CANNOT_BE_RESOLVED, + errorContextFrom(offsetLocationMeta), + internal = true + ) + } + } + /** * Returns a closure which creates an [Environment] for the specified [Group]. * If a GROUP AS name was specified, also nests that [Environment] in another that diff --git a/lang/src/org/partiql/lang/eval/ExprValueExtensions.kt b/lang/src/org/partiql/lang/eval/ExprValueExtensions.kt index 0dc75aa129..70ede55a81 100644 --- a/lang/src/org/partiql/lang/eval/ExprValueExtensions.kt +++ b/lang/src/org/partiql/lang/eval/ExprValueExtensions.kt @@ -164,7 +164,7 @@ fun ExprValue.rangeOver(): Iterable = when { fun ExprValue.stringify(): String = ConfigurableExprValueFormatter.standard.format(this) -val DEFAULT_COMPARATOR = NaturalExprValueComparators.NULLS_FIRST +val DEFAULT_COMPARATOR = NaturalExprValueComparators.NULLS_FIRST_ASC /** Provides the default equality function. */ fun ExprValue.exprEquals(other: ExprValue): Boolean = DEFAULT_COMPARATOR.compare(this, other) == 0 diff --git a/lang/src/org/partiql/lang/eval/NaturalExprValueComparators.kt b/lang/src/org/partiql/lang/eval/NaturalExprValueComparators.kt index 4828544e89..50f7a54d85 100644 --- a/lang/src/org/partiql/lang/eval/NaturalExprValueComparators.kt +++ b/lang/src/org/partiql/lang/eval/NaturalExprValueComparators.kt @@ -54,10 +54,21 @@ import org.partiql.lang.util.isZero * (as defined by this definition) members, as pairs of field name and the member value. * * `BAG` values come finally (except with [NullOrder.NULLS_LAST]), and their values * compare lexicographically based on the *sorted* child elements. + * + * @param order that compares left and right values by [Order.ASC] (ascending) or [Order.DESC] (descending) order + * @param nullOrder that places `NULL`/`MISSING` values first or last */ -enum class NaturalExprValueComparators(private val nullOrder: NullOrder) : Comparator { - NULLS_FIRST(NullOrder.FIRST), - NULLS_LAST(NullOrder.LAST); +enum class NaturalExprValueComparators(private val order: Order, private val nullOrder: NullOrder) : Comparator { + NULLS_FIRST_ASC(Order.ASC, NullOrder.FIRST), + NULLS_FIRST_DESC(Order.DESC, NullOrder.FIRST), + NULLS_LAST_ASC(Order.ASC, NullOrder.LAST), + NULLS_LAST_DESC(Order.DESC, NullOrder.LAST); + + /** Compare items by ascending or descending order */ + private enum class Order { + ASC, + DESC + } /** Whether or not null values come first or last. */ private enum class NullOrder { @@ -104,8 +115,10 @@ enum class NaturalExprValueComparators(private val nullOrder: NullOrder) : Compa val rIter = right.iterator() while (lIter.hasNext() && rIter.hasNext()) { - val lChild = lIter.next() - val rChild = rIter.next() + val (lChild, rChild) = when (order) { + Order.ASC -> lIter.next() to rIter.next() + Order.DESC -> rIter.next() to lIter.next() + } val cmp = comparator.compare(lChild, rChild) if (cmp != 0) { return cmp @@ -126,11 +139,15 @@ enum class NaturalExprValueComparators(private val nullOrder: NullOrder) : Compa ): Int { val pairCmp = object : Comparator> { override fun compare(o1: Pair, o2: Pair): Int { - val cmp = entityCmp.compare(o1.first, o2.first) + val (leftPair, rightPair) = when (order) { + Order.ASC -> o1 to o2 + Order.DESC -> o2 to o1 + } + val cmp = entityCmp.compare(leftPair.first, rightPair.first) if (cmp != 0) { return cmp } - return o2.second - o1.second + return rightPair.second - leftPair.second } } @@ -157,7 +174,7 @@ enum class NaturalExprValueComparators(private val nullOrder: NullOrder) : Compa } } - override fun compare(left: ExprValue, right: ExprValue): Int { + private fun compareInternal(left: ExprValue, right: ExprValue, nullOrder: NullOrder): Int { if (left === right) return EQUAL val lType = left.type @@ -294,4 +311,22 @@ enum class NaturalExprValueComparators(private val nullOrder: NullOrder) : Compa throw IllegalStateException("Could not compare: $left and $right") } + + // can think of `DESC` as the converse/reverse of `ASC` + // - ASC with NULLS FIRST == DESC with NULLS LAST (reverse) + // - ASC with NULLS LAST == DESC with NULLS FIRST (reverse) + // for `DESC`, return the converse result by multiplying by -1 + // need to also flip the NullOrder in the `DESC` case + override fun compare(left: ExprValue, right: ExprValue): Int { + return when (order) { + Order.ASC -> compareInternal(left, right, nullOrder) + Order.DESC -> compareInternal( + left, right, + when (nullOrder) { + NullOrder.FIRST -> NullOrder.LAST + NullOrder.LAST -> NullOrder.FIRST + } + ) * -1 + } + } } diff --git a/lang/src/org/partiql/lang/syntax/SqlLexer.kt b/lang/src/org/partiql/lang/syntax/SqlLexer.kt index 71881932f4..daac0f42ca 100644 --- a/lang/src/org/partiql/lang/syntax/SqlLexer.kt +++ b/lang/src/org/partiql/lang/syntax/SqlLexer.kt @@ -531,6 +531,18 @@ class SqlLexer(private val ion: IonSystem) : Lexer { tokenType = TokenType.DESC ion.newSymbol(lower) } + lower == "nulls" -> { + tokenType = TokenType.NULLS + ion.newSymbol(lower) + } + lower == "first" -> { + tokenType = TokenType.FIRST + ion.newSymbol(lower) + } + lower == "last" -> { + tokenType = TokenType.LAST + ion.newSymbol(lower) + } lower in BOOLEAN_KEYWORDS -> { // literal boolean tokenType = TokenType.LITERAL diff --git a/lang/src/org/partiql/lang/syntax/SqlParser.kt b/lang/src/org/partiql/lang/syntax/SqlParser.kt index 4db4e75dab..0b39d9a112 100644 --- a/lang/src/org/partiql/lang/syntax/SqlParser.kt +++ b/lang/src/org/partiql/lang/syntax/SqlParser.kt @@ -132,6 +132,7 @@ class SqlParser( ORDER_BY, SORT_SPEC, ORDERING_SPEC, + NULLS_SPEC, GROUP, GROUP_PARTIAL, HAVING, @@ -577,8 +578,20 @@ class SqlParser( orderBy( it.children[0].children.map { when (it.children.size) { - 1 -> sortSpec(it.children[0].toAstExpr(), asc()) - 2 -> sortSpec(it.children[0].toAstExpr(), it.children[1].toOrderingSpec()) + 1 -> sortSpec(it.children[0].toAstExpr(), asc(), nullsLast()) + 2 -> when (it.children[1].type) { + ParseType.ORDERING_SPEC -> { + val orderingSpec = it.children[1].toOrderingSpec() + val defaultNullsSpec = when (orderingSpec) { + is PartiqlAst.OrderingSpec.Asc -> nullsLast() + is PartiqlAst.OrderingSpec.Desc -> nullsFirst() + } + sortSpec(it.children[0].toAstExpr(), orderingSpec, defaultNullsSpec) + } + ParseType.NULLS_SPEC -> sortSpec(it.children[0].toAstExpr(), asc(), it.children[1].toNullsSpec()) + else -> errMalformedParseTree("Invalid ordering expressions syntax") + } + 3 -> sortSpec(it.children[0].toAstExpr(), it.children[1].toOrderingSpec(), it.children[2].toNullsSpec()) else -> errMalformedParseTree("Invalid ordering expressions syntax") } } @@ -1140,6 +1153,19 @@ class SqlParser( } } + private fun ParseNode.toNullsSpec(): PartiqlAst.NullsSpec { + if (type != ParseType.NULLS_SPEC) { + errMalformedParseTree("Expected ParseType.NULLS_SPEC instead of $type") + } + return PartiqlAst.build { + when (token?.type) { + TokenType.FIRST -> nullsFirst() + TokenType.LAST -> nullsLast() + else -> errMalformedParseTree("Invalid nulls spec parsing") + } + } + } + private fun ParseNode.toSymbolicName(): SymbolPrimitive { if (token == null) { errMalformedParseTree("Expected ParseNode to have a token") @@ -2210,17 +2236,6 @@ class SqlParser( parseOptionalSingleExpressionClause(ParseType.WHERE) - if (rem.head?.keywordText == "order") { - rem = rem.tail.tailExpectedToken(TokenType.BY) - - val orderByChildren = listOf(rem.parseOrderByArgList()) - rem = orderByChildren.first().remaining - - children.add( - ParseNode(type = ParseType.ORDER_BY, token = null, children = orderByChildren, remaining = rem) - ) - } - if (rem.head?.keywordText == "group") { rem = rem.tail val type = when (rem.head?.keywordText) { @@ -2276,6 +2291,17 @@ class SqlParser( parseOptionalSingleExpressionClause(ParseType.HAVING) + if (rem.head?.keywordText == "order") { + rem = rem.tail.tailExpectedToken(TokenType.BY) + + val orderByChildren = listOf(rem.parseOrderByArgList()) + rem = orderByChildren.first().remaining + + children.add( + ParseNode(type = ParseType.ORDER_BY, token = null, children = orderByChildren, remaining = rem) + ) + } + parseOptionalSingleExpressionClause(ParseType.LIMIT) parseOptionalSingleExpressionClause(ParseType.OFFSET) @@ -2837,13 +2863,12 @@ class SqlParser( var rem = this var child = rem.parseExpression() - var sortSpecKey = listOf(child) + var children = listOf(child) rem = child.remaining when (rem.head?.type) { TokenType.ASC, TokenType.DESC -> { - sortSpecKey = listOf( - child, + children = children + listOf( ParseNode( type = ParseType.ORDERING_SPEC, token = rem.head, @@ -2854,7 +2879,26 @@ class SqlParser( rem = rem.tail } } - ParseNode(type = ParseType.SORT_SPEC, token = null, children = sortSpecKey, remaining = rem) + when (rem.head?.type) { + TokenType.NULLS -> { + rem = rem.tail + when (rem.head?.type) { + TokenType.FIRST, TokenType.LAST -> { + children = children + listOf( + ParseNode( + type = ParseType.NULLS_SPEC, + token = rem.head, + children = listOf(), + remaining = rem.tail + ) + ) + } + else -> rem.head.err("Expected FIRST OR LAST after NULLS", ErrorCode.PARSE_UNEXPECTED_TOKEN) + } + rem = rem.tail + } + } + ParseNode(type = ParseType.SORT_SPEC, token = null, children = children, remaining = rem) } } diff --git a/lang/src/org/partiql/lang/syntax/TokenType.kt b/lang/src/org/partiql/lang/syntax/TokenType.kt index b0620d6507..0e0047c9aa 100644 --- a/lang/src/org/partiql/lang/syntax/TokenType.kt +++ b/lang/src/org/partiql/lang/syntax/TokenType.kt @@ -46,6 +46,9 @@ enum class TokenType { NULL, ASC, DESC, + NULLS, + FIRST, + LAST, // function specific TRIM_SPECIFICATION, DATETIME_PART, diff --git a/lang/test/org/partiql/lang/errors/ParserErrorsTest.kt b/lang/test/org/partiql/lang/errors/ParserErrorsTest.kt index 597eb5f903..47f6df7735 100644 --- a/lang/test/org/partiql/lang/errors/ParserErrorsTest.kt +++ b/lang/test/org/partiql/lang/errors/ParserErrorsTest.kt @@ -1134,6 +1134,90 @@ class ParserErrorsTest : SqlParserTestBase() { ) } + @Test + fun orderByMissingNullsType() { + checkInputThrowingParserException( + "SELECT a FROM tb ORDER BY a ASC NULLS", + ErrorCode.PARSE_UNEXPECTED_TOKEN, + mapOf( + Property.LINE_NUMBER to 1L, + Property.COLUMN_NUMBER to 38L, + Property.TOKEN_TYPE to TokenType.EOF, + Property.TOKEN_VALUE to ion.newSymbol("EOF") + ) + ) + } + + @Test + fun orderByMissingNullsKeywordWithFirstNullsType() { + checkInputThrowingParserException( + "SELECT a FROM tb ORDER BY a ASC FIRST", + ErrorCode.PARSE_UNEXPECTED_TOKEN, + mapOf( + Property.LINE_NUMBER to 1L, + Property.COLUMN_NUMBER to 33L, + Property.TOKEN_TYPE to TokenType.FIRST, + Property.TOKEN_VALUE to ion.newSymbol("first") + ) + ) + } + + @Test + fun orderByMissingNullsKeywordWithLastNullsType() { + checkInputThrowingParserException( + "SELECT a FROM tb ORDER BY a ASC LAST", + ErrorCode.PARSE_UNEXPECTED_TOKEN, + mapOf( + Property.LINE_NUMBER to 1L, + Property.COLUMN_NUMBER to 33L, + Property.TOKEN_TYPE to TokenType.LAST, + Property.TOKEN_VALUE to ion.newSymbol("last") + ) + ) + } + + @Test + fun nullsBeforeOrderBy() { + checkInputThrowingParserException( + "SELECT a FROM tb NULLS LAST ORDER BY a ASC", + ErrorCode.PARSE_UNEXPECTED_TOKEN, + mapOf( + Property.LINE_NUMBER to 1L, + Property.COLUMN_NUMBER to 18L, + Property.TOKEN_TYPE to TokenType.NULLS, + Property.TOKEN_VALUE to ion.newSymbol("nulls") + ) + ) + } + + @Test + fun orderByUnexpectedNullsKeywordAsAttribute() { + checkInputThrowingParserException( + "SELECT a FROM tb ORDER BY a NULLS SELECT", + ErrorCode.PARSE_UNEXPECTED_TOKEN, + mapOf( + Property.LINE_NUMBER to 1L, + Property.COLUMN_NUMBER to 35L, + Property.TOKEN_TYPE to TokenType.KEYWORD, + Property.TOKEN_VALUE to ion.newSymbol("select") + ) + ) + } + + @Test + fun orderByUnexpectedKeyword() { + checkInputThrowingParserException( + "SELECT a FROM tb ORDER BY a NULLS FIRST SELECT", + ErrorCode.PARSE_UNEXPECTED_TOKEN, + mapOf( + Property.LINE_NUMBER to 1L, + Property.COLUMN_NUMBER to 41L, + Property.TOKEN_TYPE to TokenType.KEYWORD, + Property.TOKEN_VALUE to ion.newSymbol("select") + ) + ) + } + @Test fun offsetBeforeLimit() { checkInputThrowingParserException( @@ -1162,6 +1246,20 @@ class ParserErrorsTest : SqlParserTestBase() { ) } + @Test + fun limitOffsetBeforeOrderByWithNulls() { + checkInputThrowingParserException( + "SELECT a FROM tb LIMIT 10 OFFSET 5 ORDER BY b ASC NULLS FIRST", + ErrorCode.PARSE_UNEXPECTED_TOKEN, + mapOf( + Property.LINE_NUMBER to 1L, + Property.COLUMN_NUMBER to 36L, + Property.TOKEN_TYPE to TokenType.KEYWORD, + Property.TOKEN_VALUE to ion.newSymbol("order") + ) + ) + } + @Test fun offsetMissingArgument() { checkInputThrowingParserException( diff --git a/lang/test/org/partiql/lang/eval/EvaluatingCompilerExceptionsTest.kt b/lang/test/org/partiql/lang/eval/EvaluatingCompilerExceptionsTest.kt index 08d5c408da..31a9aa15b0 100644 --- a/lang/test/org/partiql/lang/eval/EvaluatingCompilerExceptionsTest.kt +++ b/lang/test/org/partiql/lang/eval/EvaluatingCompilerExceptionsTest.kt @@ -446,18 +446,4 @@ class EvaluatingCompilerExceptionsTest : EvaluatorTestBase() { ), expectedPermissiveModeResult = "MISSING" ) - - // TODO: ORDER BY node is missing metas https://github.com/partiql/partiql-lang-kotlin/issues/516 hence the - // incorrect source location in the reported error - @Test - fun orderByThrowsCorrectException() = - checkInputThrowingEvaluationException( - input = "SELECT 1 FROM <<>> ORDER BY x", - errorCode = ErrorCode.EVALUATOR_FEATURE_NOT_SUPPORTED_YET, - expectErrorContextValues = mapOf( - Property.LINE_NUMBER to 1L, - Property.COLUMN_NUMBER to 1L, - Property.FEATURE_NAME to "ORDER BY" - ) - ) } diff --git a/lang/test/org/partiql/lang/eval/EvaluatingCompilerOrderByTests.kt b/lang/test/org/partiql/lang/eval/EvaluatingCompilerOrderByTests.kt new file mode 100644 index 0000000000..35be48fa53 --- /dev/null +++ b/lang/test/org/partiql/lang/eval/EvaluatingCompilerOrderByTests.kt @@ -0,0 +1,278 @@ +package org.partiql.lang.eval + +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.ArgumentsSource +import org.partiql.lang.util.ArgumentsProviderBase + +class EvaluatingCompilerOrderByTests : EvaluatorTestBase() { + private val session = mapOf( + "simple_1" to "[{col1: 1, col2: 10}, {col1: 1, col2: 5}, {col1: 1, col2: 7}, {col1: 5, col2: 7}, {col1: 3, col2: 12}]", + "suppliers" to """[ + { supplierId: 10, supplierName: "Umbrella" }, + { supplierId: 11, supplierName: "Initech" } + ]""", + "products" to """[ + { productId: 1, supplierId: 10, categoryId: 20, price: 5.0, numInStock: 1 }, + { productId: 2, supplierId: 10, categoryId: 20, price: 10.0, numInStock: 10 }, + { productId: 3, supplierId: 10, categoryId: 21, price: 15.0, numInStock: 100 }, + { productId: 4, supplierId: 11, categoryId: 21, price: 5.0, numInStock: 1000 }, + { productId: 5, supplierId: 11, categoryId: 21, price: 15.0, numInStock: 10000 } + ]""", + "products_sparse" to """[ + { productId: 1, categoryId: 20, regionId: 100, supplierId_nulls: 10, supplierId_missings: 10, supplierId_mixed: 10, price_nulls: 1.0, price_missings: 1.0, price_mixed: 1.0 }, + { productId: 2, categoryId: 20, regionId: 100, supplierId_nulls: 10, supplierId_missings: 10, supplierId_mixed: 10, price_nulls: 2.0, price_missings: 2.0, price_mixed: 2.0 }, + { productId: 3, categoryId: 20, regionId: 200, supplierId_nulls: 10, supplierId_missings: 10, supplierId_mixed: 10, price_nulls: 3.0, price_missings: 3.0, price_mixed: 3.0 }, + { productId: 5, categoryId: 21, regionId: 100, supplierId_nulls: null, price_nulls: null }, + { productId: 4, categoryId: 20, regionId: 100, supplierId_nulls: null, supplierId_mixed: null, price_nulls: null, price_mixed: null }, + { productId: 6, categoryId: 21, regionId: 100, supplierId_nulls: 11, supplierId_missings: 11, supplierId_mixed: 11, price_nulls: 4.0, price_missings: 4.0, price_mixed: 4.0 }, + { productId: 7, categoryId: 21, regionId: 200, supplierId_nulls: 11, supplierId_missings: 11, supplierId_mixed: 11, price_nulls: 5.0, price_missings: 5.0, price_mixed: 5.0 }, + { productId: 8, categoryId: 21, regionId: 200, supplierId_nulls: null, supplierId_mixed: null, price_nulls: null, price_mixed: null }, + { productId: 9, categoryId: 21, regionId: 200, supplierId_nulls: null, price_nulls: null, }, + { productId: 10, categoryId: 21, regionId: 200, supplierId_nulls: null, supplierId_mixed: null, price_nulls: null, } + ]""", + "orders" to """[ + { customerId: 123, sellerId: 1, productId: 11111, cost: 1 }, + { customerId: 123, sellerId: 2, productId: 22222, cost: 2 }, + { customerId: 123, sellerId: 1, productId: 33333, cost: 3 }, + { customerId: 456, sellerId: 2, productId: 44444, cost: 4 }, + { customerId: 456, sellerId: 1, productId: 55555, cost: 5 }, + { customerId: 456, sellerId: 2, productId: 66666, cost: 6 }, + { customerId: 789, sellerId: 1, productId: 77777, cost: 7 }, + { customerId: 789, sellerId: 2, productId: 88888, cost: 8 }, + { customerId: 789, sellerId: 1, productId: 99999, cost: 9 }, + { customerId: 100, sellerId: 2, productId: 10000, cost: 10 } + ]""" + ).toSession() + + class ArgsProviderValid : ArgumentsProviderBase() { + private val differentDataTypes = """ + [ + { 'data_value': {} }, + { 'data_value': 5 }, + { 'data_value': `2017-01-01T00:00-00:00` }, + { 'data_value': [] }, + { 'data_value': TIME '12:12:12.1' }, + { 'data_value': 'a' }, + { 'data_value': null }, + { 'data_value': false }, + { 'data_value': `{{YWFhYWFhYWFhYWFhYf8=}}` }, + { 'data_value': DATE '2021-08-22' }, + { 'data_value': <<>> }, + { 'data_value': `{{"aaaaaaaaaaaaa\xFF"}}` } + ] + """.trimIndent() + + override fun getParameters(): List = listOf( + + // SIMPLE CASES + + // should order by col1 asc + EvaluatorTestCase( + "SELECT col1 FROM simple_1 ORDER BY col1", + "[{'col1': 1}, {'col1': 1}, {'col1': 1}, {'col1': 3}, {'col1': 5}]" + ), + // should order by col1 desc + EvaluatorTestCase( + "SELECT col1 FROM simple_1 ORDER BY col1 DESC", + "[{'col1': 5}, {'col1': 3}, {'col1': 1}, {'col1': 1}, {'col1': 1}]" + ), + // should order by col1 and then col2 asc + EvaluatorTestCase( + "SELECT * FROM simple_1 ORDER BY col1, col2", + "[{'col1': 1, 'col2': 5}, {'col1': 1, 'col2': 7}, {'col1': 1, 'col2': 10}, {'col1': 3, 'col2': 12}, {'col1': 5, 'col2': 7}]" + ), + // should order by price desc and productId asc + EvaluatorTestCase( + "SELECT productId, price FROM products ORDER BY price DESC, productId ASC", + "[{'productId': 3, 'price': 15.0}, {'productId': 5, 'price': 15.0}, {'productId': 2, 'price': 10.0}, {'productId': 1, 'price': 5.0}, {'productId': 4, 'price': 5.0}]" + ), + // should order by supplierId_nulls nulls last + EvaluatorTestCase( + "SELECT productId, supplierId_nulls FROM products_sparse ORDER BY supplierId_nulls NULLS LAST, productId", + "[{'productId': 1, 'supplierId_nulls': 10}, {'productId': 2, 'supplierId_nulls': 10}, {'productId': 3, 'supplierId_nulls': 10}, {'productId': 6, 'supplierId_nulls': 11}, {'productId': 7, 'supplierId_nulls': 11}, {'productId': 4, 'supplierId_nulls': NULL}, {'productId': 5, 'supplierId_nulls': NULL}, {'productId': 8, 'supplierId_nulls': NULL}, {'productId': 9, 'supplierId_nulls': NULL}, {'productId': 10, 'supplierId_nulls': NULL}]" + ), + // should order by supplierId_nulls nulls first + EvaluatorTestCase( + "SELECT productId, supplierId_nulls FROM products_sparse ORDER BY supplierId_nulls NULLS FIRST, productId", + "[{'productId': 4, 'supplierId_nulls': NULL}, {'productId': 5, 'supplierId_nulls': NULL}, {'productId': 8, 'supplierId_nulls': NULL}, {'productId': 9, 'supplierId_nulls': NULL}, {'productId': 10, 'supplierId_nulls': NULL}, {'productId': 1, 'supplierId_nulls': 10}, {'productId': 2, 'supplierId_nulls': 10}, {'productId': 3, 'supplierId_nulls': 10}, {'productId': 6, 'supplierId_nulls': 11}, {'productId': 7, 'supplierId_nulls': 11}]" + ), + // should order by nulls last as default for supplierId_nulls asc + EvaluatorTestCase( + "SELECT productId, supplierId_nulls FROM products_sparse ORDER BY supplierId_nulls ASC, productId", + "[{'productId': 1, 'supplierId_nulls': 10}, {'productId': 2, 'supplierId_nulls': 10}, {'productId': 3, 'supplierId_nulls': 10}, {'productId': 6, 'supplierId_nulls': 11}, {'productId': 7, 'supplierId_nulls': 11}, {'productId': 4, 'supplierId_nulls': NULL}, {'productId': 5, 'supplierId_nulls': NULL}, {'productId': 8, 'supplierId_nulls': NULL}, {'productId': 9, 'supplierId_nulls': NULL}, {'productId': 10, 'supplierId_nulls': NULL}]" + ), + // should order by nulls first as default for supplierId_nulls desc + EvaluatorTestCase( + "SELECT productId, supplierId_nulls FROM products_sparse ORDER BY supplierId_nulls DESC, productId", + "[{'productId': 4, 'supplierId_nulls': NULL}, {'productId': 5, 'supplierId_nulls': NULL}, {'productId': 8, 'supplierId_nulls': NULL}, {'productId': 9, 'supplierId_nulls': NULL}, {'productId': 10, 'supplierId_nulls': NULL}, {'productId': 6, 'supplierId_nulls': 11}, {'productId': 7, 'supplierId_nulls': 11}, {'productId': 1, 'supplierId_nulls': 10}, {'productId': 2, 'supplierId_nulls': 10}, {'productId': 3, 'supplierId_nulls': 10}]" + ), + // should group and order by asc sellerId + EvaluatorTestCase( + "SELECT sellerId FROM orders GROUP BY sellerId ORDER BY sellerId ASC", + "[{'sellerId': 1}, {'sellerId': 2}]" + ), + // should group and order by desc sellerId + EvaluatorTestCase( + "SELECT sellerId FROM orders GROUP BY sellerId ORDER BY sellerId DESC", + "[{'sellerId': 2}, {'sellerId': 1}]" + ), + // should group and order by DESC (NULLS FIRST as default) + EvaluatorTestCase( + "SELECT supplierId_nulls FROM products_sparse GROUP BY supplierId_nulls ORDER BY supplierId_nulls DESC", + " [{'supplierId_nulls': NULL}, {'supplierId_nulls': 11}, {'supplierId_nulls': 10}]" + ), + // should group and order by ASC (NULLS LAST as default) + EvaluatorTestCase( + "SELECT supplierId_nulls FROM products_sparse GROUP BY supplierId_nulls ORDER BY supplierId_nulls ASC", + "[{'supplierId_nulls': 10}, {'supplierId_nulls': 11}, {'supplierId_nulls': NULL}]" + ), + // should group and place nulls first (asc as default) + EvaluatorTestCase( + "SELECT supplierId_nulls FROM products_sparse GROUP BY supplierId_nulls ORDER BY supplierId_nulls NULLS FIRST", + "[{'supplierId_nulls': NULL}, {'supplierId_nulls': 10}, {'supplierId_nulls': 11}]" + ), + // should group and place nulls last (asc as default) + EvaluatorTestCase( + "SELECT supplierId_nulls FROM products_sparse GROUP BY supplierId_nulls ORDER BY supplierId_nulls NULLS LAST", + "[{'supplierId_nulls': 10}, {'supplierId_nulls': 11}, {'supplierId_nulls': NULL}]" + ), + // should group and order by asc and place nulls first + EvaluatorTestCase( + "SELECT supplierId_nulls FROM products_sparse GROUP BY supplierId_nulls ORDER BY supplierId_nulls ASC NULLS FIRST", + "[{'supplierId_nulls': NULL}, {'supplierId_nulls': 10}, {'supplierId_nulls': 11}]" + ), + + // DIFFERENT DATA TYPES + // should order different data types by following order bool, numbers, date, time, timestamp, text, LOB Types, lists, struct, bag + // handling nulls/missing can be change by ordering spec(if nulls spec is not specified, NULLS FIRST is default for asc, NULLS LAST default for desc) or nulls spec + + // should order data types by the specifications (NULLS LAST default for asc) + EvaluatorTestCase( + "SELECT * FROM $differentDataTypes ORDER BY data_value", + """[{'data_value': false}, {'data_value': 5}, {'data_value': DATE '2021-08-22'}, {'data_value': TIME '12:12:12.1'}, {'data_value': `2017-01-01T00:00-00:00`}, {'data_value': 'a'}, {'data_value': `{{YWFhYWFhYWFhYWFhYf8=}}`}, {'data_value': `{{"aaaaaaaaaaaaa\xff"}}`}, {'data_value': []}, {'data_value': {}}, {'data_value': <<>>}, {'data_value': NULL}]""" + ), + // should order data types by the specifications (NULLS FIRST default for desc) + EvaluatorTestCase( + "SELECT * FROM $differentDataTypes ORDER BY data_value DESC", + """[{'data_value': NULL}, {'data_value': <<>>}, {'data_value': {}}, {'data_value': []}, {'data_value': `{{YWFhYWFhYWFhYWFhYf8=}}`}, {'data_value': `{{"aaaaaaaaaaaaa\xff"}}`}, {'data_value': 'a'}, {'data_value': `2017-01-01T00:00-00:00`}, {'data_value': TIME '12:12:12.1'}, {'data_value': DATE '2021-08-22'}, {'data_value': 5}, {'data_value': false}]""" + ), + // should order data types by the specifications (nulls should be first due to nulls spec) + EvaluatorTestCase( + "SELECT * FROM $differentDataTypes ORDER BY data_value NULLS FIRST", + """[{'data_value': NULL}, {'data_value': false}, {'data_value': 5}, {'data_value': DATE '2021-08-22'}, {'data_value': TIME '12:12:12.1'}, {'data_value': `2017-01-01T00:00-00:00`}, {'data_value': 'a'}, {'data_value': `{{YWFhYWFhYWFhYWFhYf8=}}`}, {'data_value': `{{"aaaaaaaaaaaaa\xff"}}`}, {'data_value': []}, {'data_value': {}}, {'data_value': <<>>}]""" + ), + // should order data types by the specifications (nulls should be last due to nulls spec) + EvaluatorTestCase( + "SELECT * FROM $differentDataTypes ORDER BY data_value NULLS LAST", + """[{'data_value': false}, {'data_value': 5}, {'data_value': DATE '2021-08-22'}, {'data_value': TIME '12:12:12.1'}, {'data_value': `2017-01-01T00:00-00:00`}, {'data_value': 'a'}, {'data_value': `{{YWFhYWFhYWFhYWFhYf8=}}`}, {'data_value': `{{"aaaaaaaaaaaaa\xff"}}`}, {'data_value': []}, {'data_value': {}}, {'data_value': <<>>}, {'data_value': NULL}]""" + ), + + // EDGE CASES + + // false before true (ASC) + EvaluatorTestCase( + "SELECT * FROM [{ 'a': false }, { 'a': true }, { 'a': true }, { 'a': false }] ORDER BY a", + "[{'a': false}, {'a': false}, {'a': true}, {'a': true}]" + ), + // true before false (DESC) + EvaluatorTestCase( + "SELECT * FROM [{ 'a': false }, { 'a': true }, { 'a': true }, { 'a': false }] ORDER BY a DESC", + "[{'a': true}, {'a': true}, {'a': false}, {'a': false}]" + ), + // nan before -inf, then numeric values then +inf (ASC) + EvaluatorTestCase( + "SELECT * FROM [{ 'a': 5 }, { 'a': -5e-1 }, { 'a': `-inf` }, { 'a': `nan` }, { 'a': 7 }, { 'a': `+inf` }, { 'a': 9 }] ORDER BY a", + "[{'a': `nan`}, {'a': `-inf`}, {'a': -0.5}, {'a': 5}, {'a': 7}, {'a': 9}, {'a': `+inf`}]" + ), + // +inf before numeric values then -inf then nan (DESC) + EvaluatorTestCase( + "SELECT * FROM [{ 'a': 5 }, { 'a': -5e-1 }, { 'a': `-inf` }, { 'a': `nan` }, { 'a': 7 }, { 'a': `+inf` }, { 'a': 9 }] ORDER BY a DESC", + "[{'a': `+inf`}, {'a': 9}, {'a': 7}, {'a': 5}, {'a': -0.5}, {'a': `-inf`}, {'a': `nan`}]" + ), + // text types compared by lexicographical ordering of Unicode scalar (ASC) + EvaluatorTestCase( + """SELECT * FROM [{ 'a': `'\uD83D\uDCA9'`}, { 'a': 'Z'}, { 'a': '9' }, { 'a': 'A'}, { 'a': `"\U0001F4A9"`}, { 'a': 'a'}, { 'a': 'z'}, { 'a': '0' }] ORDER BY a""", + """[{'a': '0'}, {'a': '9'}, {'a': 'A'}, {'a': 'Z'}, {'a': 'a'}, {'a': 'z'}, {'a': `"\U0001F4A9"`}, {'a': `'\uD83D\uDCA9'`}]""" + ), + // text types compared by lexicographical ordering of Unicode scalar (DESC) + EvaluatorTestCase( + """SELECT * FROM [{ 'a': `'\uD83D\uDCA9'`}, { 'a': 'Z'}, { 'a': '9' }, { 'a': 'A'}, { 'a': `"\U0001F4A9"`}, { 'a': 'a'}, { 'a': 'z'}, { 'a': '0' }] ORDER BY a DESC""", + """[{'a': `'\uD83D\uDCA9'`}, {'a': `"\U0001F4A9"`}, {'a': 'z'}, {'a': 'a'}, {'a': 'Z'}, {'a': 'A'}, {'a': '9'}, {'a': '0'}]""" + ), + // LOB types follow their lexicographical ordering by octet (ASC) + EvaluatorTestCase( + """SELECT * FROM [{'a': `{{"Z"}}`}, {'a': `{{"a"}}`}, {'a': `{{"A"}}`}, {'a': `{{"z"}}`}] ORDER BY a""", + """[{'a': `{{"A"}}`}, {'a': `{{"Z"}}`}, {'a': `{{"a"}}`}, {'a': `{{"z"}}`}]""" + ), + // LOB types should ordered (DESC) + EvaluatorTestCase( + """SELECT * FROM [{'a': `{{"Z"}}`}, {'a': `{{"a"}}`}, {'a': `{{"A"}}`}, {'a': `{{"z"}}`}] ORDER BY a DESC""", + """[{'a': `{{"z"}}`}, {'a': `{{"a"}}`}, {'a': `{{"Z"}}`}, {'a': `{{"A"}}`}]""" + ), + // shorter array comes first (ASC) + EvaluatorTestCase( + "SELECT * FROM [ {'a': [1, 2, 3, 4]}, {'a': [1, 2]}, {'a': [1, 2, 3]}, {'a': []}] ORDER BY a", + "[{'a': []}, {'a': [1, 2]}, {'a': [1, 2, 3]}, {'a': [1, 2, 3, 4]}]" + ), + // longer array comes first (DESC) + EvaluatorTestCase( + "SELECT * FROM [ {'a': [1, 2, 3, 4]}, {'a': [1, 2]}, {'a': [1, 2, 3]}, {'a': []}] ORDER BY a DESC", + "[{'a': [1, 2, 3, 4]}, {'a': [1, 2, 3]}, {'a': [1, 2]}, {'a': []}]" + ), + // lists compared lexicographically based on comparison of elements (ASC) + EvaluatorTestCase( + "SELECT * FROM [ {'a': ['b', 'a']}, {'a': ['a', 'b']}, {'a': ['b', 'c']}, {'a': ['a', 'c']}] ORDER BY a", + "[{'a': ['a', 'b']}, {'a': ['a', 'c']}, {'a': ['b', 'a']}, {'a': ['b', 'c']}]" + ), + // lists compared lexicographically based on comparison of elements (DESC) + EvaluatorTestCase( + "SELECT * FROM [ {'a': ['b', 'a']}, {'a': ['a', 'b']}, {'a': ['b', 'c']}, {'a': ['a', 'c']}] ORDER BY a DESC", + "[{'a': ['b', 'c']}, {'a': ['b', 'a']}, {'a': ['a', 'c']}, {'a': ['a', 'b']}]" + ), + // lists items should be ordered by data types (ASC) (nulls last as default for asc) + EvaluatorTestCase( + """SELECT * FROM [{'a': ['a']}, {'a': [1]}, {'a': [true]}, {'a': [null]}, {'a': [{}]}, {'a': [<<>>]}, {'a': [`{{}}`]}, {'a': [[]]} ] ORDER BY a""", + "[{'a': [true]}, {'a': [1]}, {'a': ['a']}, {'a': [`{{}}`]}, {'a': [[]]}, {'a': [{}]}, {'a': [<<>>]}, {'a': [NULL]}]" + ), + // lists items should be ordered by data types (DESC) (nulls first as default for desc) + EvaluatorTestCase( + """SELECT * FROM [{'a': ['a']}, {'a': [1]}, {'a': [true]}, {'a': [null]}, {'a': [{}]}, {'a': [<<>>]}, {'a': [`{{}}`]}, {'a': [[]]} ] ORDER BY a DESC""", + "[{'a': [NULL]}, {'a': [<<>>]}, {'a': [{}]}, {'a': [[]]}, {'a': [`{{}}`]}, {'a': ['a']}, {'a': [1]}, {'a': [true]}]" + ), + // structs compared lexicographically first by key then by value (ASC) + EvaluatorTestCase( + "SELECT * FROM [{'a': {'b': 'a'}}, {'a': {'a': 'b'}}, {'a': {'b': 'c'}}, {'a': {'a': 'c'}}] ORDER BY a", + "[{'a': {'a': 'b'}}, {'a': {'a': 'c'}}, {'a': {'b': 'a'}}, {'a': {'b': 'c'}}]" + ), + // structs compared lexicographically first by key then by value (DESC) + EvaluatorTestCase( + "SELECT * FROM [{'a': {'b': 'a'}}, {'a': {'a': 'b'}}, {'a': {'b': 'c'}}, {'a': {'a': 'c'}}] ORDER BY a DESC", + "[{'a': {'b': 'c'}}, {'a': {'b': 'a'}}, {'a': {'a': 'c'}}, {'a': {'a': 'b'}}]" + ), + // structs should be ordered by data types (ASC) (nulls last as default for asc) + EvaluatorTestCase( + "SELECT * FROM [{'a': {'a': 5}}, {'a': {'a': 'b'}}, {'a': {'a': true}}, {'a': {'a': []}}, {'a': {'a': {}}}, {'a': {'a': <<>>}}, {'a': {'a': `{{}}`}}, {'a': {'a': null}}] ORDER BY a", + "[{'a': {'a': true}}, {'a': {'a': 5}}, {'a': {'a': 'b'}}, {'a': {'a': `{{}}`}}, {'a': {'a': []}}, {'a': {'a': {}}}, {'a': {'a': <<>>}}, {'a': {'a': NULL}}]" + ), + // structs should be ordered by data types (DESC) (nulls first as default for desc) + EvaluatorTestCase( + "SELECT * FROM [{'a': {'a': 5}}, {'a': {'a': 'b'}}, {'a': {'a': true}}, {'a': {'a': []}}, {'a': {'a': {}}}, {'a': {'a': <<>>}}, {'a': {'a': `{{}}`}}, {'a': {'a': null}}] ORDER BY a DESC", + "[{'a': {'a': NULL}}, {'a': {'a': <<>>}}, {'a': {'a': {}}}, {'a': {'a': []}}, {'a': {'a': `{{}}`}}, {'a': {'a': 'b'}}, {'a': {'a': 5}}, {'a': {'a': true}}]" + ), + // bags compared as sorted lists (ASC) + EvaluatorTestCase( + "SELECT * FROM [{'a': <<5>>}, {'a': <<1>>}, {'a': <<10>>}] ORDER BY a", + "[{'a': <<1>>}, {'a': <<5>>}, {'a': <<10>>}]" + ), + // bags compared as sorted lists (DESC) + EvaluatorTestCase( + "SELECT * FROM [{'a': <<5>>}, {'a': <<1>>}, {'a': <<10>>}] ORDER BY a DESC", + "[{'a': <<10>>}, {'a': <<5>>}, {'a': <<1>>}]" + ), + ) + } + + @ParameterizedTest + @ArgumentsSource(ArgsProviderValid::class) + fun validTests(tc: EvaluatorTestCase) = runTestCase(tc, session) +} diff --git a/lang/test/org/partiql/lang/eval/NaturalExprValueComparatorsTest.kt b/lang/test/org/partiql/lang/eval/NaturalExprValueComparatorsTest.kt index a8a423f7e2..6fde1cb743 100644 --- a/lang/test/org/partiql/lang/eval/NaturalExprValueComparatorsTest.kt +++ b/lang/test/org/partiql/lang/eval/NaturalExprValueComparatorsTest.kt @@ -22,7 +22,7 @@ import java.util.Collections import java.util.Random class NaturalExprValueComparatorsTest : EvaluatorTestBase() { - // the lists below represent the expected ordering of values + // the lists below represent the expected ordering of values for asc/desc ordering // grouped by lists of equivalent values. private val nullExprs = listOf( @@ -327,6 +327,9 @@ class NaturalExprValueComparatorsTest : EvaluatorTestBase() { fun List>.moveHeadToTail(): List> = drop(1).plusElement(this[0]) + fun List>.moveTailToHead(): List> = + listOf(this.last()).plus(this).dropLast(1) + fun shuffleCase( description: String, comparator: Comparator, @@ -347,8 +350,10 @@ class NaturalExprValueComparatorsTest : EvaluatorTestBase() { return (1..iterations).flatMap { listOf( - shuffleCase("BASIC VALUES (NULLS FIRST)", NaturalExprValueComparators.NULLS_FIRST, basicExprs), - shuffleCase("BASIC VALUES (NULLS LAST)", NaturalExprValueComparators.NULLS_LAST, basicExprs.moveHeadToTail()) + shuffleCase("BASIC VALUES (NULLS FIRST ASC)", NaturalExprValueComparators.NULLS_FIRST_ASC, basicExprs), + shuffleCase("BASIC VALUES (NULLS LAST ASC)", NaturalExprValueComparators.NULLS_LAST_ASC, basicExprs.moveHeadToTail()), + shuffleCase("BASIC VALUES (NULLS FIRST DESC)", NaturalExprValueComparators.NULLS_FIRST_DESC, basicExprs.reversed().moveTailToHead()), + shuffleCase("BASIC VALUES (NULLS LAST DESC)", NaturalExprValueComparators.NULLS_LAST_DESC, basicExprs.reversed()), ) } } diff --git a/lang/test/org/partiql/lang/syntax/SqlParserTest.kt b/lang/test/org/partiql/lang/syntax/SqlParserTest.kt index d019ed27f8..abb04d4782 100644 --- a/lang/test/org/partiql/lang/syntax/SqlParserTest.kt +++ b/lang/test/org/partiql/lang/syntax/SqlParserTest.kt @@ -1696,7 +1696,8 @@ class SqlParserTest : SqlParserTestBase() { (order_by (sort_spec (id rk1 (case_insensitive) (unqualified)) - (asc))))) + (asc) + (nulls_last))))) """ ) @@ -1721,13 +1722,16 @@ class SqlParserTest : SqlParserTestBase() { (order_by (sort_spec (id rk1 (case_insensitive) (unqualified)) - (asc)) + (asc) + (nulls_last)) (sort_spec (id rk2 (case_insensitive) (unqualified)) - (asc)) + (asc) + (nulls_last)) (sort_spec (id rk3 (case_insensitive) (unqualified)) - (asc))))) + (asc) + (nulls_last))))) """ ) @@ -1752,7 +1756,8 @@ class SqlParserTest : SqlParserTestBase() { (order_by (sort_spec (id rk1 (case_insensitive) (unqualified)) - (desc))))) + (desc) + (nulls_first))))) """ ) @@ -1777,12 +1782,155 @@ class SqlParserTest : SqlParserTestBase() { (order_by (sort_spec (id rk1 (case_insensitive) (unqualified)) - (asc)) + (asc) + (nulls_last)) (sort_spec (id rk2 (case_insensitive) (unqualified)) - (desc))))) + (desc) + (nulls_first))))) """ ) + + @Test + fun orderBySingleIdWithoutOrderingAndNullsSpecShouldProduceAscNullsLastAsDefault() = assertExpression("SELECT x FROM tb ORDER BY rk1") { + select( + project = projectX, + from = scan(id("tb")), + order = orderBy( + listOf( + sortSpec(id("rk1"), asc(), nullsLast()) + ) + ) + ) + } + + @Test + fun orderByMultipleIdWithoutOrderingAndNullsSpecShouldProduceAscNullsLastAsDefault() = assertExpression("SELECT x FROM tb ORDER BY rk1, rk2, rk3, rk4") { + select( + project = projectX, + from = scan(id("tb")), + order = orderBy( + listOf( + sortSpec(id("rk1"), asc(), nullsLast()), + sortSpec(id("rk2"), asc(), nullsLast()), + sortSpec(id("rk3"), asc(), nullsLast()), + sortSpec(id("rk4"), asc(), nullsLast()) + ) + ) + ) + } + + @Test + fun orderByWithAscShouldProduceNullsLastAsDefault() = assertExpression("SELECT x FROM tb ORDER BY rk1 asc") { + select( + project = projectX, + from = scan(id("tb")), + order = orderBy( + listOf( + sortSpec(id("rk1"), asc(), nullsLast()) + ) + ) + ) + } + + @Test + fun orderByWithDescShouldProduceNullsFirstAsDefault() = assertExpression("SELECT x FROM tb ORDER BY rk1 desc") { + select( + project = projectX, + from = scan(id("tb")), + order = orderBy( + listOf( + sortSpec(id("rk1"), desc(), nullsFirst()) + ) + ) + ) + } + + @Test + fun orderByWithAscAndDescShouldProduceDefaultNullsSpec() = assertExpression("SELECT x FROM tb ORDER BY rk1 desc, rk2 asc, rk3 asc, rk4 desc") { + select( + project = projectX, + from = scan(id("tb")), + order = orderBy( + listOf( + sortSpec(id("rk1"), desc(), nullsFirst()), + sortSpec(id("rk2"), asc(), nullsLast()), + sortSpec(id("rk3"), asc(), nullsLast()), + sortSpec(id("rk4"), desc(), nullsFirst()) + ) + ) + ) + } + + @Test + fun orderByAscMustBeDefaultIfOrderingSpecIsNotSpecifiedWithNullsFirst() = assertExpression("SELECT x FROM tb ORDER BY rk1 NULLS FIRST") { + select( + project = projectX, + from = scan(id("tb")), + order = orderBy( + listOf( + sortSpec(id("rk1"), asc(), nullsFirst()) + ) + ) + ) + } + + @Test + fun orderByAscMustBeDefaultIfOrderingSpecIsNotSpecifiedWithNullsLast() = assertExpression("SELECT x FROM tb ORDER BY rk1 NULLS LAST") { + select( + project = projectX, + from = scan(id("tb")), + order = orderBy( + listOf( + sortSpec(id("rk1"), asc(), nullsLast()) + ) + ) + ) + } + + @Test + fun orderByAscWithNullsSpec() = assertExpression("SELECT x FROM tb ORDER BY rk1 asc NULLS FIRST, rk2 asc NULLS LAST") { + select( + project = projectX, + from = scan(id("tb")), + order = orderBy( + listOf( + sortSpec(id("rk1"), asc(), nullsFirst()), + sortSpec(id("rk2"), asc(), nullsLast()) + ) + ) + ) + } + + @Test + fun orderByDescWithNullsSpec() = assertExpression("SELECT x FROM tb ORDER BY rk1 desc NULLS FIRST, rk2 desc NULLS LAST") { + select( + project = projectX, + from = scan(id("tb")), + order = orderBy( + listOf( + sortSpec(id("rk1"), desc(), nullsFirst()), + sortSpec(id("rk2"), desc(), nullsLast()) + ) + ) + ) + } + + @Test + fun orderByWithOrderingAndNullsSpec() = assertExpression("SELECT x FROM tb ORDER BY rk1 desc NULLS FIRST, rk2 asc NULLS LAST, rk3 desc NULLS LAST, rk4 asc NULLS FIRST") { + select( + project = projectX, + from = scan(id("tb")), + order = orderBy( + listOf( + sortSpec(id("rk1"), desc(), nullsFirst()), + sortSpec(id("rk2"), asc(), nullsLast()), + sortSpec(id("rk3"), desc(), nullsLast()), + sortSpec(id("rk4"), asc(), nullsFirst()) + ) + ) + ) + } // **************************************** // GROUP BY and GROUP PARTIAL BY // **************************************** @@ -4023,7 +4171,7 @@ class SqlParserTest : SqlParserTestBase() { select( project = buildProject("x"), from = scan(id("a")), - order = PartiqlAst.OrderBy(listOf(PartiqlAst.SortSpec(id("y"), PartiqlAst.OrderingSpec.Desc()))), + order = PartiqlAst.OrderBy(listOf(PartiqlAst.SortSpec(id("y"), PartiqlAst.OrderingSpec.Desc(), PartiqlAst.NullsSpec.NullsFirst()))), limit = buildLit("10"), offset = buildLit("5") )