From 88388979f45c12efd3f396d3a88e105a47581c6d Mon Sep 17 00:00:00 2001 From: Alan Cai Date: Fri, 20 Oct 2023 16:54:47 -0700 Subject: [PATCH] A few more tests; add subsumption tests --- .../partiql/lang/eval/EvaluatingCompiler.kt | 20 +- .../eval/EvaluatingCompilerExcludeTests.kt | 282 ++++++++++++++++++ 2 files changed, 298 insertions(+), 4 deletions(-) diff --git a/partiql-lang/src/main/kotlin/org/partiql/lang/eval/EvaluatingCompiler.kt b/partiql-lang/src/main/kotlin/org/partiql/lang/eval/EvaluatingCompiler.kt index 38a96c8a1c..66623e3a3b 100644 --- a/partiql-lang/src/main/kotlin/org/partiql/lang/eval/EvaluatingCompiler.kt +++ b/partiql-lang/src/main/kotlin/org/partiql/lang/eval/EvaluatingCompiler.kt @@ -2209,11 +2209,11 @@ internal open class EvaluatingCompiler( /** * Represents an instance of a compiled `EXCLUDE` expression. Notably, this expr will have redundant steps removed. */ - private data class CompiledExcludeExpr(val root: PartiqlAst.Identifier, val exclusions: RemoveAndOtherSteps) + internal data class CompiledExcludeExpr(val root: PartiqlAst.Identifier, val exclusions: RemoveAndOtherSteps) /** * Represents all the exclusions at the current level and other nested levels. */ - private data class RemoveAndOtherSteps(val remove: Set, val steps: Map) { + internal data class RemoveAndOtherSteps(val remove: Set, val steps: Map) { companion object { fun empty(): RemoveAndOtherSteps { return RemoveAndOtherSteps(emptySet(), emptyMap()) @@ -2224,7 +2224,7 @@ internal open class EvaluatingCompiler( /** * Creates a list of compiled exclude expressions. */ - private fun compileExcludeClause(excludeClause: PartiqlAst.ExcludeOp): List { + internal fun compileExcludeClause(excludeClause: PartiqlAst.ExcludeOp): List { val excludeExprs = excludeClause.exprs fun addToCompiledExcludeExprs(curCompiledExpr: RemoveAndOtherSteps, steps: List): RemoveAndOtherSteps { // subsumption cases @@ -2232,35 +2232,47 @@ internal open class EvaluatingCompiler( // when steps.size > 1: look at other steps val first = steps.first() var entryRemove = curCompiledExpr.remove.toMutableSet() - val entrySteps = curCompiledExpr.steps.toMutableMap() + var entrySteps = curCompiledExpr.steps.toMutableMap() if (steps.size == 1) { when (first) { is PartiqlAst.ExcludeStep.ExcludeTupleAttr -> { if (entryRemove.contains(PartiqlAst.build { excludeTupleWildcard() })) { // contains wildcard; do not add; a.b and a.* -> a[*] } else { + // add to entries to remove entryRemove.add(first) + // remove from other steps + entrySteps.remove(first) } } is PartiqlAst.ExcludeStep.ExcludeTupleWildcard -> { if (entryRemove.any { it is PartiqlAst.ExcludeStep.ExcludeCollectionWildcard || it is PartiqlAst.ExcludeStep.ExcludeCollectionIndex }) { // todo mistyping; and other mistyping } else { + // entries to remove just tuple wildcard entryRemove = mutableSetOf(first) + // todo: perhaps remove just the tuple attrs and tuple wildcard + entrySteps = mutableMapOf() } } is PartiqlAst.ExcludeStep.ExcludeCollectionIndex -> { if (entryRemove.contains(PartiqlAst.build { excludeCollectionWildcard() })) { // contains wildcard; do not add; a[*] and a[*] -> a[*] } else { + // add to entries to remove entryRemove.add(first) + // remove from other steps + entrySteps.remove(first) } } is PartiqlAst.ExcludeStep.ExcludeCollectionWildcard -> { if (entryRemove.any { it is PartiqlAst.ExcludeStep.ExcludeTupleWildcard || it is PartiqlAst.ExcludeStep.ExcludeTupleAttr }) { // todo mistyping; and other mistyping } else { + // entries to remove just collection wildcard entryRemove = mutableSetOf(first) + // todo: perhaps remove just the collection index and collection wildcard + entrySteps = mutableMapOf() } } } diff --git a/partiql-lang/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerExcludeTests.kt b/partiql-lang/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerExcludeTests.kt index 40ea677a64..3ab41e1362 100644 --- a/partiql-lang/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerExcludeTests.kt +++ b/partiql-lang/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerExcludeTests.kt @@ -1,12 +1,16 @@ package org.partiql.lang.eval +import com.amazon.ionelement.api.emptyMetaContainer import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.ArgumentsSource +import org.partiql.lang.domains.PartiqlAst import org.partiql.lang.eval.evaluatortestframework.CompilerPipelineFactory import org.partiql.lang.eval.evaluatortestframework.EvaluatorTestAdapter import org.partiql.lang.eval.evaluatortestframework.EvaluatorTestCase import org.partiql.lang.eval.evaluatortestframework.PipelineEvaluatorTestAdapter +import org.partiql.lang.syntax.PartiQLParserBuilder import org.partiql.lang.util.ArgumentsProviderBase +import org.partiql.pig.runtime.LongPrimitive class EvaluatingCompilerExcludeTests : EvaluatorTestBase() { @@ -783,6 +787,28 @@ class EvaluatingCompilerExcludeTests : EvaluatorTestBase() { >> """ ), + EvaluatorTestCase( // EXCLUDE with GROUP BY, HAVING, and aggregation + """ + SELECT SUM(t.b) AS total, g + EXCLUDE g[*].t.c -- EXCLUDE + FROM + [ + {'a': 1, 'b': 2, 'c': 3}, -- group `t.a` = 1 + {'a': 1, 'b': 22, 'c': 33}, -- group `t.a` = 1 + {'a': 2, 'b': 222, 'c': 333} -- row/group omitted due to `HAVING` clause + ] AS t + GROUP BY t.a GROUP AS g + HAVING COUNT(t.a) > 1 + """, + """ + << + { + 'total': 24, -- `total` from row 1's `a` (2) + row 2's `a` (22) = 24 + 'g': <<{'t': {'a': 1, 'b': 2}}, {'t': {'a': 1, 'b': 22}}>> -- `EXCLUDE`s `c` + } + >> + """ + ), ) } @@ -792,4 +818,260 @@ class EvaluatingCompilerExcludeTests : EvaluatorTestBase() { tc, EvaluationSession.standard() ) + + private fun testExcludeExprSubsumption(tc: SubsumptionTC) { + val parser = PartiQLParserBuilder.standard().build() + val parsedSFW = parser.parseAstStatement("SELECT * EXCLUDE ${tc.excludeExprStr} FROM t") + val exclude = (((parsedSFW as PartiqlAst.Statement.Query).expr) as PartiqlAst.Expr.Select).excludeClause!! + val eC = EvaluatingCompiler( + emptyList(), + emptyMap(), + emptyMap(), + ) + val actualExcludeExprs = eC.compileExcludeClause(exclude) + assertEquals(tc.expectedExcludeExprs, actualExcludeExprs) + } + + internal data class SubsumptionTC(val excludeExprStr: String, val expectedExcludeExprs: List) + + @ParameterizedTest + @ArgumentsSource(ExcludeSubsumptionTests::class) + internal fun subsumptionTests(tc: SubsumptionTC) = testExcludeExprSubsumption(tc) + + internal class ExcludeSubsumptionTests : ArgumentsProviderBase() { + private fun caseSensitiveId(id: String): PartiqlAst.Identifier { + return PartiqlAst.build { identifier(name = id, case = caseSensitive(emptyMetaContainer())) } + } + private fun caseInsensitiveId(id: String): PartiqlAst.Identifier { + return PartiqlAst.build { identifier(name = id, case = caseInsensitive(emptyMetaContainer())) } + } + private fun exTupleAttr(id: PartiqlAst.Identifier): PartiqlAst.ExcludeStep { + return PartiqlAst.ExcludeStep.ExcludeTupleAttr(id) + } + private fun exTupleWildcard(): PartiqlAst.ExcludeStep { + return PartiqlAst.ExcludeStep.ExcludeTupleWildcard() + } + private fun exCollIndex(i: Int): PartiqlAst.ExcludeStep { + return PartiqlAst.ExcludeStep.ExcludeCollectionIndex(index = LongPrimitive(i.toLong(), emptyMetaContainer())) + } + private fun exCollWildcard(): PartiqlAst.ExcludeStep { + return PartiqlAst.ExcludeStep.ExcludeCollectionWildcard() + } + + override fun getParameters(): List = listOf( + SubsumptionTC( + "s.a, t.a, t.b, s.b", + listOf( + EvaluatingCompiler.CompiledExcludeExpr( + root = caseInsensitiveId("s"), + exclusions = EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))), + steps = emptyMap(), + ) + ), + EvaluatingCompiler.CompiledExcludeExpr( + root = caseInsensitiveId("t"), + exclusions = EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))), + steps = emptyMap(), + ) + ) + ) + ), + SubsumptionTC( + "t.a, t.b", + listOf( + EvaluatingCompiler.CompiledExcludeExpr( + root = caseInsensitiveId("t"), + exclusions = EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))), + steps = emptyMap(), + ) + ) + ) + ), + SubsumptionTC( + "t.a, t.b, t.a, t.b, t.b", // duplicates subsumed + listOf( + EvaluatingCompiler.CompiledExcludeExpr( + root = caseInsensitiveId("t"), + exclusions = EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))), + steps = emptyMap(), + ) + ) + ) + ), + SubsumptionTC( + "t.a, t.b, t.*", // tuple wildcard subsumes tuple attr + listOf( + EvaluatingCompiler.CompiledExcludeExpr( + root = caseInsensitiveId("t"), + exclusions = EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exTupleWildcard()), + steps = emptyMap(), + ) + ) + ) + ), + SubsumptionTC( // removal at earlier step subsumes + """ + t.a, t.a.a1, -- t.a.a1 subsumed + t.b.b1.b2, t.b.b1, -- t.b.b1.b2 subsumed + t.c, t.c.c1[2].c3[*].* -- t.c.c1[2].c3[*].* subsumed + """, + listOf( + EvaluatingCompiler.CompiledExcludeExpr( + root = caseInsensitiveId("t"), + exclusions = EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("c"))), + steps = mapOf( + exTupleAttr(caseInsensitiveId("b")) to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exTupleAttr(caseInsensitiveId("b1"))), + steps = emptyMap() + ) + ), + ) + ) + ) + ), + SubsumptionTC( // exclude collection index + """ + t.a, t.a[1], + t.b[1], t.b + """, + listOf( + EvaluatingCompiler.CompiledExcludeExpr( + root = caseInsensitiveId("t"), + exclusions = EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))), + steps = emptyMap() + ) + ) + ) + ), + SubsumptionTC( // exclude collection index, collection wildcard + """ + t.a[*], t.a[1], + t.b[1], t.b[*], + t.c[*], t.c[1].c1, + t.d[1].d1, t.d[*] + """, + listOf( + EvaluatingCompiler.CompiledExcludeExpr( + root = caseInsensitiveId("t"), + exclusions = EvaluatingCompiler.RemoveAndOtherSteps( + remove = emptySet(), + steps = mapOf( + exTupleAttr(caseInsensitiveId("a")) to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exCollWildcard()), + steps = emptyMap() + ), + exTupleAttr(caseInsensitiveId("b")) to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exCollWildcard()), + steps = emptyMap() + ), + exTupleAttr(caseInsensitiveId("c")) to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exCollWildcard()), + steps = emptyMap() + ), + exTupleAttr(caseInsensitiveId("d")) to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exCollWildcard()), + steps = emptyMap() + ), + ) + ) + ) + ) + ), + SubsumptionTC( + """ + t.a[1].a1, t.a[1], + t.b[1], t.b[1].b1, + t.c[*], t.c[*].c1, + t.d[*].d1, t.d[*], + t.e[1], t.e[*].e1, -- keep both + t.f[*].f1, t.f[1], -- keep both + t.g[*], t.g[1].e1, + t.h[1].f1, t.h[*] + """, + listOf( + EvaluatingCompiler.CompiledExcludeExpr( + root = caseInsensitiveId("t"), + exclusions = EvaluatingCompiler.RemoveAndOtherSteps( + remove = emptySet(), + steps = mapOf( + exTupleAttr(caseInsensitiveId("a")) to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exCollIndex(1)), + steps = emptyMap() + ), + exTupleAttr(caseInsensitiveId("b")) to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exCollIndex(1)), + steps = emptyMap() + ), + exTupleAttr(caseInsensitiveId("c")) to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exCollWildcard()), + steps = emptyMap() + ), + exTupleAttr(caseInsensitiveId("d")) to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exCollWildcard()), + steps = emptyMap() + ), + exTupleAttr(caseInsensitiveId("e")) to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exCollIndex(1)), + steps = mapOf( + exCollWildcard() to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exTupleAttr(caseInsensitiveId("e1"))), + steps = emptyMap(), + ) + ) + ), + exTupleAttr(caseInsensitiveId("f")) to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exCollIndex(1)), + steps = mapOf( + exCollWildcard() to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exTupleAttr(caseInsensitiveId("f1"))), + steps = emptyMap(), + ) + ) + ), + exTupleAttr(caseInsensitiveId("g")) to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exCollWildcard()), + steps = emptyMap() + ), + exTupleAttr(caseInsensitiveId("h")) to EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exCollWildcard()), + steps = emptyMap() + ), + ) + ) + ) + ) + ), + SubsumptionTC( // case sensitive + """ + t.a, "t".a, -- "t".a in case-sensitive list + "t".b, t.b, -- "t".b in case-sensitive list + t."c", t.c, + t.d, t."d" + """, + listOf( + EvaluatingCompiler.CompiledExcludeExpr( + root = caseInsensitiveId("t"), + exclusions = EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b")), exTupleAttr(caseInsensitiveId("c")), exTupleAttr(caseInsensitiveId("d")), exTupleAttr(caseSensitiveId("c")), exTupleAttr(caseSensitiveId("d"))), + steps = emptyMap(), + ) + ), + EvaluatingCompiler.CompiledExcludeExpr( + root = caseSensitiveId("t"), + exclusions = EvaluatingCompiler.RemoveAndOtherSteps( + remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))), + steps = emptyMap(), + ) + ), + ) + ), + ) + } }