Skip to content

Commit

Permalink
A few more tests; add subsumption tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@alancai98
alancai98 committed Nov 16, 2023
1 parent bd239ab commit 8838897
Show file tree
Hide file tree
Showing 2 changed files with 298 additions and 4 deletions.
20 changes: 16 additions & 4 deletions partiql-lang/src/main/kotlin/org/partiql/lang/eval/EvaluatingCompiler.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2209,11 +2209,11 @@ internal open class EvaluatingCompiler(
/**
* Represents an instance of a compiled `EXCLUDE` expression. Notably, this expr will have redundant steps removed.
*/
private data class CompiledExcludeExpr(val root: PartiqlAst.Identifier, val exclusions: RemoveAndOtherSteps)
internal data class CompiledExcludeExpr(val root: PartiqlAst.Identifier, val exclusions: RemoveAndOtherSteps)
/**
* Represents all the exclusions at the current level and other nested levels.
*/
private data class RemoveAndOtherSteps(val remove: Set<PartiqlAst.ExcludeStep>, val steps: Map<PartiqlAst.ExcludeStep, RemoveAndOtherSteps>) {
internal data class RemoveAndOtherSteps(val remove: Set<PartiqlAst.ExcludeStep>, val steps: Map<PartiqlAst.ExcludeStep, RemoveAndOtherSteps>) {
companion object {
fun empty(): RemoveAndOtherSteps {
return RemoveAndOtherSteps(emptySet(), emptyMap())
Expand All @@ -2224,43 +2224,55 @@ internal open class EvaluatingCompiler(
/**
* Creates a list of compiled exclude expressions.
*/
private fun compileExcludeClause(excludeClause: PartiqlAst.ExcludeOp): List<CompiledExcludeExpr> {
internal fun compileExcludeClause(excludeClause: PartiqlAst.ExcludeOp): List<CompiledExcludeExpr> {
val excludeExprs = excludeClause.exprs
fun addToCompiledExcludeExprs(curCompiledExpr: RemoveAndOtherSteps, steps: List<PartiqlAst.ExcludeStep>): RemoveAndOtherSteps {
// subsumption cases
// when steps.size == 1: look at remove set
// when steps.size > 1: look at other steps
val first = steps.first()
var entryRemove = curCompiledExpr.remove.toMutableSet()
val entrySteps = curCompiledExpr.steps.toMutableMap()
var entrySteps = curCompiledExpr.steps.toMutableMap()
if (steps.size == 1) {
when (first) {
is PartiqlAst.ExcludeStep.ExcludeTupleAttr -> {
if (entryRemove.contains(PartiqlAst.build { excludeTupleWildcard() })) {
// contains wildcard; do not add; a.b and a.* -> a[*]
} else {
// add to entries to remove
entryRemove.add(first)
// remove from other steps
entrySteps.remove(first)
}
}
is PartiqlAst.ExcludeStep.ExcludeTupleWildcard -> {
if (entryRemove.any { it is PartiqlAst.ExcludeStep.ExcludeCollectionWildcard || it is PartiqlAst.ExcludeStep.ExcludeCollectionIndex }) {
// todo mistyping; and other mistyping
} else {
// entries to remove just tuple wildcard
entryRemove = mutableSetOf(first)
// todo: perhaps remove just the tuple attrs and tuple wildcard
entrySteps = mutableMapOf()
}
}
is PartiqlAst.ExcludeStep.ExcludeCollectionIndex -> {
if (entryRemove.contains(PartiqlAst.build { excludeCollectionWildcard() })) {
// contains wildcard; do not add; a[*] and a[*] -> a[*]
} else {
// add to entries to remove
entryRemove.add(first)
// remove from other steps
entrySteps.remove(first)
}
}
is PartiqlAst.ExcludeStep.ExcludeCollectionWildcard -> {
if (entryRemove.any { it is PartiqlAst.ExcludeStep.ExcludeTupleWildcard || it is PartiqlAst.ExcludeStep.ExcludeTupleAttr }) {
// todo mistyping; and other mistyping
} else {
// entries to remove just collection wildcard
entryRemove = mutableSetOf(first)
// todo: perhaps remove just the collection index and collection wildcard
entrySteps = mutableMapOf()
}
}
}
Expand Down
282 changes: 282 additions & 0 deletions partiql-lang/src/test/kotlin/org/partiql/lang/eval/EvaluatingCompilerExcludeTests.kt
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
package org.partiql.lang.eval

import com.amazon.ionelement.api.emptyMetaContainer
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.ArgumentsSource
import org.partiql.lang.domains.PartiqlAst
import org.partiql.lang.eval.evaluatortestframework.CompilerPipelineFactory
import org.partiql.lang.eval.evaluatortestframework.EvaluatorTestAdapter
import org.partiql.lang.eval.evaluatortestframework.EvaluatorTestCase
import org.partiql.lang.eval.evaluatortestframework.PipelineEvaluatorTestAdapter
import org.partiql.lang.syntax.PartiQLParserBuilder
import org.partiql.lang.util.ArgumentsProviderBase
import org.partiql.pig.runtime.LongPrimitive

class EvaluatingCompilerExcludeTests : EvaluatorTestBase() {

Expand Down Expand Up @@ -783,6 +787,28 @@ class EvaluatingCompilerExcludeTests : EvaluatorTestBase() {
>>
"""
),
EvaluatorTestCase( // EXCLUDE with GROUP BY, HAVING, and aggregation
"""
SELECT SUM(t.b) AS total, g
EXCLUDE g[*].t.c -- EXCLUDE
FROM
[
{'a': 1, 'b': 2, 'c': 3}, -- group `t.a` = 1
{'a': 1, 'b': 22, 'c': 33}, -- group `t.a` = 1
{'a': 2, 'b': 222, 'c': 333} -- row/group omitted due to `HAVING` clause
] AS t
GROUP BY t.a GROUP AS g
HAVING COUNT(t.a) > 1
""",
"""
<<
{
'total': 24, -- `total` from row 1's `a` (2) + row 2's `a` (22) = 24
'g': <<{'t': {'a': 1, 'b': 2}}, {'t': {'a': 1, 'b': 22}}>> -- `EXCLUDE`s `c`
}
>>
"""
),
)
}

Expand All @@ -792,4 +818,260 @@ class EvaluatingCompilerExcludeTests : EvaluatorTestBase() {
tc,
EvaluationSession.standard()
)

private fun testExcludeExprSubsumption(tc: SubsumptionTC) {
val parser = PartiQLParserBuilder.standard().build()
val parsedSFW = parser.parseAstStatement("SELECT * EXCLUDE ${tc.excludeExprStr} FROM t")
val exclude = (((parsedSFW as PartiqlAst.Statement.Query).expr) as PartiqlAst.Expr.Select).excludeClause!!
val eC = EvaluatingCompiler(
emptyList(),
emptyMap(),
emptyMap(),
)
val actualExcludeExprs = eC.compileExcludeClause(exclude)
assertEquals(tc.expectedExcludeExprs, actualExcludeExprs)
}

internal data class SubsumptionTC(val excludeExprStr: String, val expectedExcludeExprs: List<EvaluatingCompiler.CompiledExcludeExpr>)

@ParameterizedTest
@ArgumentsSource(ExcludeSubsumptionTests::class)
internal fun subsumptionTests(tc: SubsumptionTC) = testExcludeExprSubsumption(tc)

internal class ExcludeSubsumptionTests : ArgumentsProviderBase() {
private fun caseSensitiveId(id: String): PartiqlAst.Identifier {
return PartiqlAst.build { identifier(name = id, case = caseSensitive(emptyMetaContainer())) }
}
private fun caseInsensitiveId(id: String): PartiqlAst.Identifier {
return PartiqlAst.build { identifier(name = id, case = caseInsensitive(emptyMetaContainer())) }
}
private fun exTupleAttr(id: PartiqlAst.Identifier): PartiqlAst.ExcludeStep {
return PartiqlAst.ExcludeStep.ExcludeTupleAttr(id)
}
private fun exTupleWildcard(): PartiqlAst.ExcludeStep {
return PartiqlAst.ExcludeStep.ExcludeTupleWildcard()
}
private fun exCollIndex(i: Int): PartiqlAst.ExcludeStep {
return PartiqlAst.ExcludeStep.ExcludeCollectionIndex(index = LongPrimitive(i.toLong(), emptyMetaContainer()))
}
private fun exCollWildcard(): PartiqlAst.ExcludeStep {
return PartiqlAst.ExcludeStep.ExcludeCollectionWildcard()
}

override fun getParameters(): List<Any> = listOf(
SubsumptionTC(
"s.a, t.a, t.b, s.b",
listOf(
EvaluatingCompiler.CompiledExcludeExpr(
root = caseInsensitiveId("s"),
exclusions = EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))),
steps = emptyMap(),
)
),
EvaluatingCompiler.CompiledExcludeExpr(
root = caseInsensitiveId("t"),
exclusions = EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))),
steps = emptyMap(),
)
)
)
),
SubsumptionTC(
"t.a, t.b",
listOf(
EvaluatingCompiler.CompiledExcludeExpr(
root = caseInsensitiveId("t"),
exclusions = EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))),
steps = emptyMap(),
)
)
)
),
SubsumptionTC(
"t.a, t.b, t.a, t.b, t.b", // duplicates subsumed
listOf(
EvaluatingCompiler.CompiledExcludeExpr(
root = caseInsensitiveId("t"),
exclusions = EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))),
steps = emptyMap(),
)
)
)
),
SubsumptionTC(
"t.a, t.b, t.*", // tuple wildcard subsumes tuple attr
listOf(
EvaluatingCompiler.CompiledExcludeExpr(
root = caseInsensitiveId("t"),
exclusions = EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exTupleWildcard()),
steps = emptyMap(),
)
)
)
),
SubsumptionTC( // removal at earlier step subsumes
"""
t.a, t.a.a1, -- t.a.a1 subsumed
t.b.b1.b2, t.b.b1, -- t.b.b1.b2 subsumed
t.c, t.c.c1[2].c3[*].* -- t.c.c1[2].c3[*].* subsumed
""",
listOf(
EvaluatingCompiler.CompiledExcludeExpr(
root = caseInsensitiveId("t"),
exclusions = EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("c"))),
steps = mapOf(
exTupleAttr(caseInsensitiveId("b")) to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exTupleAttr(caseInsensitiveId("b1"))),
steps = emptyMap()
)
),
)
)
)
),
SubsumptionTC( // exclude collection index
"""
t.a, t.a[1],
t.b[1], t.b
""",
listOf(
EvaluatingCompiler.CompiledExcludeExpr(
root = caseInsensitiveId("t"),
exclusions = EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))),
steps = emptyMap()
)
)
)
),
SubsumptionTC( // exclude collection index, collection wildcard
"""
t.a[*], t.a[1],
t.b[1], t.b[*],
t.c[*], t.c[1].c1,
t.d[1].d1, t.d[*]
""",
listOf(
EvaluatingCompiler.CompiledExcludeExpr(
root = caseInsensitiveId("t"),
exclusions = EvaluatingCompiler.RemoveAndOtherSteps(
remove = emptySet(),
steps = mapOf(
exTupleAttr(caseInsensitiveId("a")) to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exCollWildcard()),
steps = emptyMap()
),
exTupleAttr(caseInsensitiveId("b")) to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exCollWildcard()),
steps = emptyMap()
),
exTupleAttr(caseInsensitiveId("c")) to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exCollWildcard()),
steps = emptyMap()
),
exTupleAttr(caseInsensitiveId("d")) to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exCollWildcard()),
steps = emptyMap()
),
)
)
)
)
),
SubsumptionTC(
"""
t.a[1].a1, t.a[1],
t.b[1], t.b[1].b1,
t.c[*], t.c[*].c1,
t.d[*].d1, t.d[*],
t.e[1], t.e[*].e1, -- keep both
t.f[*].f1, t.f[1], -- keep both
t.g[*], t.g[1].e1,
t.h[1].f1, t.h[*]
""",
listOf(
EvaluatingCompiler.CompiledExcludeExpr(
root = caseInsensitiveId("t"),
exclusions = EvaluatingCompiler.RemoveAndOtherSteps(
remove = emptySet(),
steps = mapOf(
exTupleAttr(caseInsensitiveId("a")) to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exCollIndex(1)),
steps = emptyMap()
),
exTupleAttr(caseInsensitiveId("b")) to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exCollIndex(1)),
steps = emptyMap()
),
exTupleAttr(caseInsensitiveId("c")) to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exCollWildcard()),
steps = emptyMap()
),
exTupleAttr(caseInsensitiveId("d")) to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exCollWildcard()),
steps = emptyMap()
),
exTupleAttr(caseInsensitiveId("e")) to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exCollIndex(1)),
steps = mapOf(
exCollWildcard() to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exTupleAttr(caseInsensitiveId("e1"))),
steps = emptyMap(),
)
)
),
exTupleAttr(caseInsensitiveId("f")) to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exCollIndex(1)),
steps = mapOf(
exCollWildcard() to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exTupleAttr(caseInsensitiveId("f1"))),
steps = emptyMap(),
)
)
),
exTupleAttr(caseInsensitiveId("g")) to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exCollWildcard()),
steps = emptyMap()
),
exTupleAttr(caseInsensitiveId("h")) to EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exCollWildcard()),
steps = emptyMap()
),
)
)
)
)
),
SubsumptionTC( // case sensitive
"""
t.a, "t".a, -- "t".a in case-sensitive list
"t".b, t.b, -- "t".b in case-sensitive list
t."c", t.c,
t.d, t."d"
""",
listOf(
EvaluatingCompiler.CompiledExcludeExpr(
root = caseInsensitiveId("t"),
exclusions = EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b")), exTupleAttr(caseInsensitiveId("c")), exTupleAttr(caseInsensitiveId("d")), exTupleAttr(caseSensitiveId("c")), exTupleAttr(caseSensitiveId("d"))),
steps = emptyMap(),
)
),
EvaluatingCompiler.CompiledExcludeExpr(
root = caseSensitiveId("t"),
exclusions = EvaluatingCompiler.RemoveAndOtherSteps(
remove = setOf(exTupleAttr(caseInsensitiveId("a")), exTupleAttr(caseInsensitiveId("b"))),
steps = emptyMap(),
)
),
)
),
)
}
}

0 comments on commit 8838897

Please sign in to comment.