From 41f75592a2cc112ac0c962626b30c73f80c3dc86 Mon Sep 17 00:00:00 2001 From: Young-Leo <122667095+Young-Leo@users.noreply.github.com> Date: Tue, 3 Dec 2024 22:10:35 +0800 Subject: [PATCH 1/4] introduce parser and analyzer of row pattern recognition --- .../plan/relational/analyzer/Analysis.java | 128 ++++ .../analyzer/ExpressionAnalyzer.java | 606 +++++++++++++++++- .../analyzer/PatternRecognitionAnalysis.java | 336 ++++++++++ .../analyzer/PatternRecognitionAnalyzer.java | 230 +++++++ .../analyzer/StatementAnalyzer.java | 236 ++++++- .../planner/ir/ExpressionTreeRewriter.java | 3 +- .../relational/sql/ast/AnchorPattern.java | 83 +++ .../plan/relational/sql/ast/AstVisitor.java | 80 +++ .../sql/ast/DefaultTraversalVisitor.java | 48 ++ .../plan/relational/sql/ast/EmptyPattern.java | 68 ++ .../relational/sql/ast/ExcludedPattern.java | 78 +++ .../plan/relational/sql/ast/FunctionCall.java | 62 +- .../relational/sql/ast/MeasureDefinition.java | 89 +++ .../sql/ast/OneOrMoreQuantifier.java | 31 + .../sql/ast/PatternAlternation.java | 80 +++ .../sql/ast/PatternConcatenation.java | 80 +++ .../sql/ast/PatternPermutation.java | 80 +++ .../relational/sql/ast/PatternQuantifier.java | 82 +++ .../sql/ast/PatternRecognitionRelation.java | 205 ++++++ .../relational/sql/ast/PatternVariable.java | 78 +++ .../relational/sql/ast/ProcessingMode.java | 77 +++ .../relational/sql/ast/QuantifiedPattern.java | 89 +++ .../relational/sql/ast/RangeQuantifier.java | 93 +++ .../plan/relational/sql/ast/RowPattern.java | 31 + .../plan/relational/sql/ast/SkipTo.java | 133 ++++ .../relational/sql/ast/SubsetDefinition.java | 90 +++ .../sql/ast/VariableDefinition.java | 89 +++ .../sql/ast/ZeroOrMoreQuantifier.java | 31 + .../sql/ast/ZeroOrOneQuantifier.java | 31 + .../relational/sql/parser/AstBuilder.java | 266 +++++++- .../sql/util/ExpressionFormatter.java | 25 + .../sql/util/RowPatternFormatter.java | 143 +++++ .../relational/sql/util/SqlFormatter.java | 139 +++- .../analyzer/RowPatternRecognitionTest.java | 458 +++++++++++++ .../relational/grammar/sql/RelationalSql.g4 | 89 ++- mvn | 0 36 files changed, 4433 insertions(+), 34 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/PatternRecognitionAnalysis.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/PatternRecognitionAnalyzer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AnchorPattern.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/EmptyPattern.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExcludedPattern.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/MeasureDefinition.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/OneOrMoreQuantifier.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternAlternation.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternConcatenation.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternPermutation.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternQuantifier.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternRecognitionRelation.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternVariable.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ProcessingMode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuantifiedPattern.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RangeQuantifier.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowPattern.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SkipTo.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SubsetDefinition.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/VariableDefinition.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ZeroOrMoreQuantifier.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ZeroOrOneQuantifier.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/RowPatternFormatter.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/RowPatternRecognitionTest.java create mode 100644 mvn diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 1f3873067b4d..7b930febf75b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -32,6 +32,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.memory.TableModelStatementMemorySourceContext; import org.apache.iotdb.db.queryengine.plan.execution.memory.TableModelStatementMemorySourceVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.TimePredicate; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.PatternInputAnalysis; import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; @@ -46,6 +47,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FieldReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Fill; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.InPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; @@ -57,10 +59,13 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuerySpecification; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RangeQuantifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Relation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowPattern; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowStatement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubsetDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; import org.apache.iotdb.db.queryengine.plan.statement.component.FillPolicy; @@ -95,6 +100,7 @@ import static com.google.common.base.MoreObjects.toStringHelper; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.ImmutableSet.toImmutableSet; @@ -135,6 +141,25 @@ public class Analysis implements IAnalysis { private final Map>> tableColumnReferences = new LinkedHashMap<>(); + // Record fields prefixed with labels in row pattern recognition context + private final Map, Optional> labels = new LinkedHashMap<>(); + private final Map, Range> ranges = new LinkedHashMap<>(); + private final Map, Set> undefinedLabels = new LinkedHashMap<>(); + + // Pattern function analysis (classifier, match_number, aggregations and prev/next/first/last) in + // the context of the given node + private final Map, List> patternInputsAnalysis = + new LinkedHashMap<>(); + + // FunctionCall nodes corresponding to any of the special pattern recognition functions + private final Set> patternRecognitionFunctionCalls = new LinkedHashSet<>(); + + // FunctionCall nodes corresponding to any of the navigation functions (prev/next/first/last) + private final Set> patternNavigationFunctions = new LinkedHashSet<>(); + + private final Map, String> resolvedLabels = new LinkedHashMap<>(); + private final Map, Set> subsets = new LinkedHashMap<>(); + private final Map, FillAnalysis> fill = new LinkedHashMap<>(); private final Map, Long> offset = new LinkedHashMap<>(); private final Map, OptionalLong> limit = new LinkedHashMap<>(); @@ -638,6 +663,91 @@ public Map>> getTableCol return tableColumnReferences; } + public void addLabels(Map, Optional> labels) { + this.labels.putAll(labels); + } + + public Optional getLabel(Expression expression) { + return labels.get(NodeRef.of(expression)); + } + + public void setRanges(Map, Range> quantifierRanges) { + ranges.putAll(quantifierRanges); + } + + public Range getRange(RangeQuantifier quantifier) { + Range range = ranges.get(NodeRef.of(quantifier)); + checkNotNull(range, "missing range for quantifier %s", quantifier); + return range; + } + + public void setUndefinedLabels(RowPattern pattern, Set labels) { + undefinedLabels.put(NodeRef.of(pattern), labels); + } + + public void setUndefinedLabels(Map, Set> labels) { + undefinedLabels.putAll(labels); + } + + public Set getUndefinedLabels(RowPattern pattern) { + Set labels = undefinedLabels.get(NodeRef.of(pattern)); + checkNotNull(labels, "missing undefined labels for %s", pattern); + return labels; + } + + public void addPatternRecognitionInputs( + Map, List> functions) { + patternInputsAnalysis.putAll(functions); + + functions.values().stream() + .flatMap(List::stream) + .map(PatternInputAnalysis::getExpression) + .filter(FunctionCall.class::isInstance) + .map(FunctionCall.class::cast) + .map(NodeRef::of) + .forEach(patternRecognitionFunctionCalls::add); + } + + public List getPatternInputsAnalysis(Expression expression) { + return patternInputsAnalysis.get(NodeRef.of(expression)); + } + + public void addPatternNavigationFunctions(Set> functions) { + patternNavigationFunctions.addAll(functions); + } + + public boolean isPatternRecognitionFunction(FunctionCall functionCall) { + return patternRecognitionFunctionCalls.contains(NodeRef.of(functionCall)); + } + + public boolean isPatternNavigationFunction(FunctionCall node) { + return patternNavigationFunctions.contains(NodeRef.of(node)); + } + + public void addResolvedLabel(Identifier label, String resolved) { + resolvedLabels.put(NodeRef.of(label), resolved); + } + + public void addResolvedLabels(Map, String> labels) { + resolvedLabels.putAll(labels); + } + + public String getResolvedLabel(Identifier identifier) { + return resolvedLabels.get(NodeRef.of(identifier)); + } + + public void addSubsetLabels(SubsetDefinition subset, Set labels) { + subsets.put(NodeRef.of(subset), labels); + } + + public void addSubsetLabels(Map, Set> subsets) { + this.subsets.putAll(subsets); + } + + public Set getSubsetLabels(SubsetDefinition subset) { + return subsets.get(NodeRef.of(subset)); + } + public RelationType getOutputDescriptor() { return getOutputDescriptor(root); } @@ -1157,6 +1267,24 @@ public Optional getSubqueryCoercion() { } } + public static class Range { + private final Optional atLeast; + private final Optional atMost; + + public Range(Optional atLeast, Optional atMost) { + this.atLeast = requireNonNull(atLeast, "atLeast is null"); + this.atMost = requireNonNull(atMost, "atMost is null"); + } + + public Optional getAtLeast() { + return atLeast; + } + + public Optional getAtMost() { + return atMost; + } + } + public static class FillAnalysis { protected final FillPolicy fillMethod; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index 404c88b2e1e8..a88735baf5cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -24,6 +24,13 @@ import org.apache.iotdb.db.queryengine.common.SessionInfo; import org.apache.iotdb.db.queryengine.execution.warnings.WarningCollector; import org.apache.iotdb.db.queryengine.plan.analyze.TypeProvider; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis.Range; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.ClassifierDescriptor; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.MatchNumberDescriptor; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.Navigation; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.NavigationMode; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.PatternInputAnalysis; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.ScalarInputDescriptor; import org.apache.iotdb.db.queryengine.plan.relational.function.BoundSignature; import org.apache.iotdb.db.queryengine.plan.relational.function.FunctionId; import org.apache.iotdb.db.queryengine.plan.relational.function.FunctionKind; @@ -68,14 +75,18 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullIfExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ProcessingMode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RangeQuantifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowPattern; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StackableAstVisitor; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubsetDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Trim; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WhenClause; @@ -93,6 +104,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -104,12 +116,16 @@ import java.util.function.BiFunction; import java.util.function.Function; +import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.Iterables.getOnlyElement; import static java.lang.String.format; import static java.util.Collections.unmodifiableMap; import static java.util.Collections.unmodifiableSet; +import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.ExpressionTreeUtils.extractExpressions; import static org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl.isCharType; import static org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl.isNumericType; import static org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl.isTwoTypeComparable; @@ -150,6 +166,21 @@ public class ExpressionAnalyzer { // Track referenced fields from source relation node private final Multimap, Field> referencedFields = HashMultimap.create(); + // Record fields prefixed with labels in row pattern recognition context + private final Map, Optional> labels = new HashMap<>(); + // Record functions specific to row pattern recognition context + private final Map, Range> ranges = new LinkedHashMap<>(); + private final Map, Set> undefinedLabels = new LinkedHashMap<>(); + private final Map, String> resolvedLabels = new LinkedHashMap<>(); + private final Map, Set> subsets = new LinkedHashMap<>(); + + // Pattern function analysis (classifier, match_number, aggregations and prev/next/first/last) in + // the context of the given node + private final Map, List> patternRecognitionInputs = + new LinkedHashMap<>(); + + private final Set> patternNavigationFunctions = new LinkedHashSet<>(); + private final MPPQueryContext context; private final SessionInfo session; @@ -296,11 +327,41 @@ public List getSourceFields() { return sourceFields; } + public Map, Optional> getLabels() { + return labels; + } + + public Map, Range> getRanges() { + return ranges; + } + + public Map, Set> getUndefinedLabels() { + return undefinedLabels; + } + + public Map, String> getResolvedLabels() { + return resolvedLabels; + } + + public Map, Set> getSubsetLabels() { + return subsets; + } + + public Map, List> getPatternRecognitionInputs() { + return patternRecognitionInputs; + } + + public Set> getPatternNavigationFunctions() { + return patternNavigationFunctions; + } + private class Visitor extends StackableAstVisitor { // Used to resolve FieldReferences (e.g. during local execution planning) private final Scope baseScope; private final WarningCollector warningCollector; + private final List patternRecognitionInputs = new ArrayList<>(); + public Visitor(Scope baseScope, WarningCollector warningCollector) { this.baseScope = requireNonNull(baseScope, "baseScope is null"); this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); @@ -409,6 +470,54 @@ protected Type visitDereferenceExpression( // If this Dereference looks like column reference, try match it to column first. if (qualifiedName != null) { + // In the context of row pattern matching, fields are optionally prefixed with labels. + // Labels are irrelevant during type analysis. + if (context.getContext().isPatternRecognition()) { + String label = label(qualifiedName.getOriginalParts().get(0)); + if (context.getContext().getPatternRecognitionContext().getLabels().contains(label)) { + // In the context of row pattern matching, the name of row pattern input table cannot be + // used to qualify column names. + // (it can only be accessed in PARTITION BY and ORDER BY clauses of MATCH_RECOGNIZE). + // Consequentially, if a dereference + // expression starts with a label, the next part must be a column. + // Only strict column references can be prefixed by label. Labeled references to row + // fields are not supported. + QualifiedName unlabeledName = + QualifiedName.of( + qualifiedName + .getOriginalParts() + .subList(1, qualifiedName.getOriginalParts().size())); + if (qualifiedName.getOriginalParts().size() > 2) { + throw new SemanticException( + String.format( + "Column %s prefixed with label %s cannot be resolved", unlabeledName, label)); + } + Optional resolvedField = + context.getContext().getScope().tryResolveField(node, unlabeledName); + if (!resolvedField.isPresent()) { + throw new SemanticException( + String.format( + "Column %s prefixed with label %s cannot be resolved", unlabeledName, label)); + } + // Correlation is not allowed in pattern recognition context. Visitor's context for + // pattern recognition has CorrelationSupport.DISALLOWED, + // and so the following call should fail if the field is from outer scope. + + labels.put(NodeRef.of(node), Optional.of(label)); + patternRecognitionInputs.add( + new PatternInputAnalysis( + node, + new ScalarInputDescriptor( + Optional.of(label), + context.getContext().getPatternRecognitionContext().getNavigation()))); + + return handleResolvedField(node, resolvedField.get(), context); + } + // In the context of row pattern matching, qualified column references are not allowed. + throw new SemanticException( + String.format("Column '%s' cannot be resolved", qualifiedName)); + } + Scope scope = context.getContext().getScope(); Optional resolvedField = scope.tryResolveField(node, qualifiedName); if (resolvedField.isPresent()) { @@ -773,6 +882,50 @@ protected Type visitFunctionCall( } }); + if (context.getContext().isPatternRecognition()) { + if (isPatternRecognitionFunction(node)) { + validatePatternRecognitionFunction(node); + + String name = node.getName().getSuffix().toUpperCase(ENGLISH); + switch (name) { + case "MATCH_NUMBER": + return setExpressionType(node, analyzeMatchNumber(node, context)); + case "CLASSIFIER": + return setExpressionType(node, analyzeClassifier(node, context)); + case "RPR_FIRST": + case "RPR_LAST": + return setExpressionType(node, analyzeLogicalNavigation(node, context, name)); + case "PREV": + case "NEXT": + return setExpressionType(node, analyzePhysicalNavigation(node, context, name)); + default: + throw new IllegalStateException("unexpected pattern recognition function " + name); + } + + } else if (isAggregation) { + if (node.isDistinct()) { + throw new SemanticException( + "Cannot use DISTINCT with aggregate function in pattern recognition context"); + } + } + } + + if (node.getProcessingMode().isPresent()) { + ProcessingMode processingMode = node.getProcessingMode().get(); + if (!context.getContext().isPatternRecognition()) { + throw new SemanticException( + String.format( + "%s semantics is not supported out of pattern recognition context", + processingMode.getMode())); + } + if (!isAggregation) { + throw new SemanticException( + String.format( + "%s semantics is supported only for FIRST(), LAST() and aggregation functions. Actual: %s", + processingMode.getMode(), node.getName())); + } + } + if (node.isDistinct() && !isAggregation) { throw new SemanticException("DISTINCT is not supported for non-aggregation functions"); } @@ -816,7 +969,7 @@ public List getCallArgumentTypes( // process the argument but do not include it in the list DereferenceExpression allRowsDereference = (DereferenceExpression) argument; String label = label((Identifier) allRowsDereference.getBase()); - if (!context.getContext().getLabels().contains(label)) { + if (!context.getContext().getPatternRecognitionContext().getLabels().contains(label)) { throw new SemanticException( String.format("%s is not a primary pattern variable or subset name", label)); } @@ -829,10 +982,317 @@ public List getCallArgumentTypes( return argumentTypesBuilder.build(); } + private Type analyzeMatchNumber( + FunctionCall node, StackableAstVisitorContext context) { + if (!node.getArguments().isEmpty()) { + throw new SemanticException("MATCH_NUMBER pattern recognition function takes no arguments"); + } + + patternRecognitionInputs.add(new PatternInputAnalysis(node, new MatchNumberDescriptor())); + + return INT64; + } + + private Type analyzeClassifier(FunctionCall node, StackableAstVisitorContext context) { + if (node.getArguments().size() > 1) { + throw new SemanticException( + "CLASSIFIER pattern recognition function takes no arguments or 1 argument"); + } + + Optional label = Optional.empty(); + if (node.getArguments().size() == 1) { + Node argument = node.getArguments().get(0); + if (!(argument instanceof Identifier)) { + throw new SemanticException( + String.format( + "CLASSIFIER function argument should be primary pattern variable or subset name. Actual: %s", + argument.getClass().getSimpleName())); + } + + Identifier identifier = (Identifier) argument; + label = Optional.of(label(identifier)); + if (!context + .getContext() + .getPatternRecognitionContext() + .getLabels() + .contains(label.get())) { + throw new SemanticException( + String.format( + "%s is not a primary pattern variable or subset name", identifier.getValue())); + } + } + + patternRecognitionInputs.add( + new PatternInputAnalysis( + node, + new ClassifierDescriptor( + label, context.getContext().getPatternRecognitionContext().getNavigation()))); + + return STRING; + } + + private Type analyzePhysicalNavigation( + FunctionCall node, StackableAstVisitorContext context, String name) { + validateNavigationFunctionArguments(node); + + // TODO: this should only be done at the root of a pattern recognition function call tree + checkNoNestedAggregations(node); + validateNavigationNesting(node); + + int offset = getNavigationOffset(node, 1); + if (name.equals("PREV")) { + offset = -offset; + } + + Navigation navigation = context.getContext().getPatternRecognitionContext().getNavigation(); + Type type = + process( + node.getArguments().get(0), + new StackableAstVisitorContext<>( + context + .getContext() + .withNavigation( + new Navigation( + navigation.getAnchor(), + navigation.getMode(), + navigation.getLogicalOffset(), + offset)))); + + // TODO: this should only be done at the root of a pattern recognition function call tree + // if (!validateLabelConsistency(node, 0).hasLabel()) { + // throw new SemanticException( + // String.format( + // "Pattern navigation function '%s' must contain at least one column reference + // or CLASSIFIER()", + // name)); + // } + + patternNavigationFunctions.add(NodeRef.of(node)); + + return type; + } + + private Type analyzeLogicalNavigation( + FunctionCall node, StackableAstVisitorContext context, String name) { + validateNavigationFunctionArguments(node); + + // TODO: this should only be done at the root of a pattern recognition function call tree + checkNoNestedAggregations(node); + validateNavigationNesting(node); + + PatternRecognitionAnalysis.NavigationAnchor anchor; + switch (name) { + case "RPR_FIRST": + anchor = PatternRecognitionAnalysis.NavigationAnchor.FIRST; + break; + case "RPR_LAST": + anchor = PatternRecognitionAnalysis.NavigationAnchor.LAST; + break; + default: + throw new IllegalStateException("Unexpected navigation anchor: " + name); + } + + Type type = + process( + node.getArguments().get(0), + new StackableAstVisitorContext<>( + context + .getContext() + .withNavigation( + new Navigation( + anchor, + mapProcessingMode(node.getProcessingMode()), + getNavigationOffset(node, 0), + context + .getContext() + .getPatternRecognitionContext() + .getNavigation() + .getPhysicalOffset())))); + + // TODO: this should only be done at the root of a pattern recognition function call tree + // if (!validateLabelConsistency(node, 0).hasLabel()) { + // throw new SemanticException( + // String.format( + // "Pattern navigation function '%s' must contain at least one column reference + // or CLASSIFIER()", + // name)); + // } + + patternNavigationFunctions.add(NodeRef.of(node)); + + return type; + } + + private NavigationMode mapProcessingMode(Optional processingMode) { + if (processingMode.isPresent()) { + ProcessingMode mode = processingMode.get(); + switch (mode.getMode()) { + case FINAL: + return NavigationMode.FINAL; + case RUNNING: + return NavigationMode.RUNNING; + default: + throw new IllegalArgumentException("Unexpected mode: " + mode.getMode()); + } + } else { + return NavigationMode.RUNNING; + } + } + + private int getNavigationOffset(FunctionCall node, int defaultOffset) { + int offset = defaultOffset; + if (node.getArguments().size() == 2) { + offset = (int) ((LongLiteral) node.getArguments().get(1)).getParsedValue(); + } + return offset; + } + + private void validatePatternRecognitionFunction(FunctionCall node) { + if (node.isDistinct()) { + throw new SemanticException( + String.format( + "Cannot use DISTINCT with %s pattern recognition function", node.getName())); + } + String name = node.getName().getSuffix(); + if (node.getProcessingMode().isPresent()) { + ProcessingMode processingMode = node.getProcessingMode().get(); + if (!name.equalsIgnoreCase("RPR_FIRST") && !name.equalsIgnoreCase("RPR_LAST")) { + throw new SemanticException( + String.format( + "%s semantics is not supported with %s pattern recognition function", + processingMode.getMode(), node.getName())); + } + } + } + + private void validateNavigationFunctionArguments(FunctionCall node) { + if (node.getArguments().size() != 1 && node.getArguments().size() != 2) { + throw new SemanticException( + String.format( + "%s pattern recognition function requires 1 or 2 arguments", node.getName())); + } + if (node.getArguments().size() == 2) { + // TODO the offset argument must be effectively constant, not necessarily a number. This + // could be extended with the use of ConstantAnalyzer. + if (!(node.getArguments().get(1) instanceof LongLiteral)) { + throw new SemanticException( + String.format( + "%s pattern recognition navigation function requires a number as the second argument", + node.getName())); + } + long offset = ((LongLiteral) node.getArguments().get(1)).getParsedValue(); + if (offset < 0) { + throw new SemanticException( + String.format( + "%s pattern recognition navigation function requires a non-negative number as the second argument (actual: %s)", + node.getName(), offset)); + } + if (offset > Integer.MAX_VALUE) { + throw new SemanticException( + String.format( + "The second argument of %s pattern recognition navigation function must not exceed %s (actual: %s)", + node.getName(), Integer.MAX_VALUE, offset)); + } + } + } + + private void validateNavigationNesting(FunctionCall node) { + checkArgument(isPatternNavigationFunction(node)); + String name = node.getName().getSuffix(); + + // It is allowed to nest FIRST and LAST functions within PREV and NEXT functions. Only + // immediate nesting is supported + List nestedNavigationFunctions = + extractExpressions(ImmutableList.of(node.getArguments().get(0)), FunctionCall.class) + .stream() + .filter(this::isPatternNavigationFunction) + .collect(toImmutableList()); + if (!nestedNavigationFunctions.isEmpty()) { + if (name.equalsIgnoreCase("RPR_FIRST") || name.equalsIgnoreCase("RPR_LAST")) { + throw new SemanticException( + String.format( + "Cannot nest %s pattern navigation function inside %s pattern navigation function", + nestedNavigationFunctions.get(0).getName(), name)); + } + if (nestedNavigationFunctions.size() > 1) { + throw new SemanticException( + String.format( + "Cannot nest multiple pattern navigation functions inside %s pattern navigation function", + name)); + } + FunctionCall nested = getOnlyElement(nestedNavigationFunctions); + String nestedName = nested.getName().getSuffix(); + if (nestedName.equalsIgnoreCase("PREV") || nestedName.equalsIgnoreCase("NEXT")) { + throw new SemanticException( + String.format( + "Cannot nest %s pattern navigation function inside %s pattern navigation function", + nestedName, name)); + } + if (nested != node.getArguments().get(0)) { + throw new SemanticException( + "Immediate nesting is required for pattern navigation functions"); + } + } + } + + private boolean isPatternNavigationFunction(FunctionCall node) { + if (!isPatternRecognitionFunction(node)) { + return false; + } + String name = node.getName().getSuffix().toUpperCase(ENGLISH); + return name.equals("RPR_FIRST") + || name.equals("RPR_LAST") + || name.equals("PREV") + || name.equals("NEXT"); + } + + private boolean isClassifierFunction(FunctionCall node) { + if (!isPatternRecognitionFunction(node)) { + return false; + } + return node.getName().getSuffix().toUpperCase(ENGLISH).equals("CLASSIFIER"); + } + + private boolean isMatchNumberFunction(FunctionCall node) { + if (!isPatternRecognitionFunction(node)) { + return false; + } + return node.getName().getSuffix().toUpperCase(ENGLISH).equals("MATCH_NUMBER"); + } + private String label(Identifier identifier) { return identifier.getCanonicalValue(); } + private void checkNoNestedAggregations(FunctionCall node) { + extractExpressions(node.getArguments(), FunctionCall.class).stream() + .filter( + function -> + metadata.isAggregationFunction( + session, function.getName().getSuffix(), accessControl)) + .findFirst() + .ifPresent( + aggregation -> { + throw new SemanticException( + String.format( + "Cannot nest %s aggregate function inside %s function", + aggregation.getName(), node.getName())); + }); + } + + private void checkNoNestedNavigations(FunctionCall node) { + extractExpressions(node.getArguments(), FunctionCall.class).stream() + .filter(this::isPatternNavigationFunction) + .findFirst() + .ifPresent( + navigation -> { + throw new SemanticException( + String.format( + "Cannot nest %s pattern navigation function inside %s function", + navigation.getName().getSuffix(), node.getName())); + }); + } + @Override protected Type visitCurrentDatabase( CurrentDatabase node, StackableAstVisitorContext context) { @@ -1293,55 +1753,64 @@ private static class Context { // arguments. // private final Map fieldToLambdaArgumentDeclaration; - // Primary row pattern variables and named unions (subsets) of variables - // necessary for the analysis of expressions in the context of row pattern recognition - private final Set labels; + private final Optional patternRecognitionContext; private final CorrelationSupport correlationSupport; private Context( Scope scope, List functionInputTypes, - Set labels, + Optional patternRecognitionContext, CorrelationSupport correlationSupport) { this.scope = requireNonNull(scope, "scope is null"); this.functionInputTypes = functionInputTypes; // this.fieldToLambdaArgumentDeclaration = fieldToLambdaArgumentDeclaration; - this.labels = labels; + this.patternRecognitionContext = + requireNonNull(patternRecognitionContext, "patternRecognitionContext is null"); this.correlationSupport = requireNonNull(correlationSupport, "correlationSupport is null"); } public static Context notInLambda(Scope scope, CorrelationSupport correlationSupport) { - return new Context(scope, null, null, correlationSupport); + return new Context(scope, null, Optional.empty(), correlationSupport); } public Context expectingLambda(List functionInputTypes) { return new Context( scope, requireNonNull(functionInputTypes, "functionInputTypes is null"), - labels, + Optional.empty(), correlationSupport); } public Context notExpectingLambda() { - return new Context(scope, null, labels, correlationSupport); + return new Context(scope, null, Optional.empty(), correlationSupport); } public static Context patternRecognition(Scope scope, Set labels) { return new Context( - scope, null, requireNonNull(labels, "labels is null"), CorrelationSupport.DISALLOWED); + scope, + null, + Optional.of(new PatternRecognitionContext(labels, Navigation.DEFAULT)), + CorrelationSupport.DISALLOWED); + } + + public Context withNavigation(Navigation navigation) { + PatternRecognitionContext patternRecognitionContext = + new PatternRecognitionContext(this.patternRecognitionContext.get().labels, navigation); + return new Context( + scope, functionInputTypes, Optional.of(patternRecognitionContext), correlationSupport); } public Context patternRecognition(Set labels) { return new Context( scope, functionInputTypes, - requireNonNull(labels, "labels is null"), + Optional.of(new PatternRecognitionContext(labels, Navigation.DEFAULT)), CorrelationSupport.DISALLOWED); } public Context notExpectingLabels() { - return new Context(scope, functionInputTypes, null, correlationSupport); + return new Context(scope, functionInputTypes, Optional.empty(), correlationSupport); } Scope getScope() { @@ -1356,18 +1825,120 @@ public boolean isExpectingLambda() { return functionInputTypes != null; } + public boolean isPatternRecognition() { + return patternRecognitionContext.isPresent(); + } + public List getFunctionInputTypes() { checkState(isExpectingLambda()); return functionInputTypes; } - public Set getLabels() { - return labels; + public PatternRecognitionContext getPatternRecognitionContext() { + return patternRecognitionContext.get(); } public CorrelationSupport getCorrelationSupport() { return correlationSupport; } + + public static class PatternRecognitionContext { + private final Set labels; + private final Navigation navigation; + + public PatternRecognitionContext(Set labels, Navigation navigation) { + this.labels = labels; + this.navigation = navigation; + } + + public Set getLabels() { + return labels; + } + + public Navigation getNavigation() { + return navigation; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + PatternRecognitionContext that = (PatternRecognitionContext) o; + + if (!labels.equals(that.labels)) return false; + return navigation.equals(that.navigation); + } + + @Override + public int hashCode() { + int result = labels.hashCode(); + result = 31 * result + navigation.hashCode(); + return result; + } + + @Override + public String toString() { + return "PatternRecognitionContext{" + + "labels=" + + labels + + ", navigation=" + + navigation + + '}'; + } + } + } + + public static boolean isPatternRecognitionFunction(FunctionCall node) { + QualifiedName qualifiedName = node.getName(); + if (qualifiedName.getParts().size() > 1) { + return false; + } + Identifier identifier = qualifiedName.getOriginalParts().get(0); + if (identifier.isDelimited()) { + return false; + } + String name = identifier.getValue().toUpperCase(ENGLISH); + return name.equals("RPR_FIRST") + || name.equals("RPR_LAST") + || name.equals("PREV") + || name.equals("NEXT") + || name.equals("CLASSIFIER") + || name.equals("MATCH_NUMBER"); + } + + public static ExpressionAnalysis analyzePatternRecognitionExpression( + Metadata metadata, + MPPQueryContext context, + SessionInfo session, + StatementAnalyzerFactory statementAnalyzerFactory, + AccessControl accessControl, + Scope scope, + Analysis analysis, + Expression expression, + WarningCollector warningCollector, + Set labels) { + ExpressionAnalyzer analyzer = + new ExpressionAnalyzer( + metadata, + context, + accessControl, + statementAnalyzerFactory, + analysis, + session, + TypeProvider.empty(), + warningCollector); + analyzer.analyze(expression, scope, labels); + + updateAnalysis(analysis, analyzer, session, accessControl); + + return new ExpressionAnalysis( + analyzer.getExpressionTypes(), + analyzer.getSubqueryInPredicates(), + analyzer.getSubqueries(), + analyzer.getExistsSubqueries(), + analyzer.getColumnReferences(), + analyzer.getQuantifiedComparisons()); } public static ExpressionAnalysis analyzeExpressions( @@ -1487,6 +2058,13 @@ private static void updateAnalysis( analysis.addTableColumnReferences( accessControl, session.getIdentity(), analyzer.getTableColumnReferences()); analysis.addPredicateCoercions(analyzer.getPredicateCoercions()); + analysis.addLabels(analyzer.getLabels()); + analysis.setRanges(analyzer.getRanges()); + analysis.setUndefinedLabels(analyzer.getUndefinedLabels()); + analysis.addResolvedLabels(analyzer.getResolvedLabels()); + analysis.addSubsetLabels(analyzer.getSubsetLabels()); + analysis.addPatternRecognitionInputs(analyzer.getPatternRecognitionInputs()); + analysis.addPatternNavigationFunctions(analyzer.getPatternNavigationFunctions()); } public static ExpressionAnalyzer createConstantAnalyzer( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/PatternRecognitionAnalysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/PatternRecognitionAnalysis.java new file mode 100644 index 000000000000..a778cadd3bc5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/PatternRecognitionAnalysis.java @@ -0,0 +1,336 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer; + +import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RangeQuantifier; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public class PatternRecognitionAnalysis { + private final Set allLabels; + private final Set undefinedLabels; + private final Map, Analysis.Range> ranges; + + public PatternRecognitionAnalysis( + Set allLabels, + Set undefinedLabels, + Map, Analysis.Range> ranges) { + this.allLabels = requireNonNull(allLabels, "allLabels is null"); + this.undefinedLabels = ImmutableSet.copyOf(undefinedLabels); + this.ranges = ImmutableMap.copyOf(ranges); + } + + public Set getAllLabels() { + return allLabels; + } + + public Set getUndefinedLabels() { + return undefinedLabels; + } + + public Map, Analysis.Range> getRanges() { + return ranges; + } + + @Override + public String toString() { + return "PatternRecognitionAnalysis{" + + "allLabels=" + + allLabels + + ", undefinedLabels=" + + undefinedLabels + + ", ranges=" + + ranges + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PatternRecognitionAnalysis that = (PatternRecognitionAnalysis) o; + return Objects.equals(allLabels, that.allLabels) + && Objects.equals(undefinedLabels, that.undefinedLabels) + && Objects.equals(ranges, that.ranges); + } + + @Override + public int hashCode() { + return Objects.hash(allLabels, undefinedLabels, ranges); + } + + public static class PatternInputAnalysis { + private final Expression expression; + private final Descriptor descriptor; + + public PatternInputAnalysis(Expression expression, Descriptor descriptor) { + this.expression = requireNonNull(expression, "expression is null"); + this.descriptor = requireNonNull(descriptor, "descriptor is null"); + } + + public Expression getExpression() { + return expression; + } + + public Descriptor getDescriptor() { + return descriptor; + } + + @Override + public String toString() { + return "PatternInputAnalysis{" + + "expression=" + + expression + + ", descriptor=" + + descriptor + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + PatternInputAnalysis that = (PatternInputAnalysis) o; + return Objects.equals(expression, that.expression) + && Objects.equals(descriptor, that.descriptor); + } + + @Override + public int hashCode() { + return Objects.hash(expression, descriptor); + } + } + + public enum NavigationMode { + RUNNING, + FINAL + } + + public interface Descriptor {} + + public static class AggregationDescriptor implements Descriptor { + private final ResolvedFunction function; + private final List arguments; + private final NavigationMode mode; + private final Set labels; + private final List matchNumberCalls; + private final List classifierCalls; + + public AggregationDescriptor( + ResolvedFunction function, + List arguments, + NavigationMode mode, + Set labels, + List matchNumberCalls, + List classifierCalls) { + this.function = requireNonNull(function, "function is null"); + this.arguments = requireNonNull(arguments, "arguments is null"); + this.mode = requireNonNull(mode, "mode is null"); + this.labels = requireNonNull(labels, "labels is null"); + this.matchNumberCalls = requireNonNull(matchNumberCalls, "matchNumberCalls is null"); + this.classifierCalls = requireNonNull(classifierCalls, "classifierCalls is null"); + } + + public ResolvedFunction getFunction() { + return function; + } + + public List getArguments() { + return arguments; + } + + public NavigationMode getMode() { + return mode; + } + + public Set getLabels() { + return labels; + } + + public List getMatchNumberCalls() { + return matchNumberCalls; + } + + public List getClassifierCalls() { + return classifierCalls; + } + + @Override + public String toString() { + return "AggregationDescriptor{" + + "function=" + + function + + ", arguments=" + + arguments + + ", mode=" + + mode + + ", labels=" + + labels + + ", matchNumberCalls=" + + matchNumberCalls + + ", classifierCalls=" + + classifierCalls + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AggregationDescriptor that = (AggregationDescriptor) o; + return Objects.equals(function, that.function) + && Objects.equals(arguments, that.arguments) + && mode == that.mode + && Objects.equals(labels, that.labels) + && Objects.equals(matchNumberCalls, that.matchNumberCalls) + && Objects.equals(classifierCalls, that.classifierCalls); + } + + @Override + public int hashCode() { + return Objects.hash(function, arguments, mode, labels, matchNumberCalls, classifierCalls); + } + } + + public static class ScalarInputDescriptor implements Descriptor { + private final Optional label; + private final Navigation navigation; + + public ScalarInputDescriptor(Optional label, Navigation navigation) { + this.label = requireNonNull(label, "label is null"); + this.navigation = requireNonNull(navigation, "navigation is null"); + } + + public Optional getLabel() { + return label; + } + + public Navigation getNavigation() { + return navigation; + } + + @Override + public String toString() { + return "ScalarInputDescriptor{" + "label=" + label + ", navigation=" + navigation + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ScalarInputDescriptor that = (ScalarInputDescriptor) o; + return Objects.equals(label, that.label) && Objects.equals(navigation, that.navigation); + } + + @Override + public int hashCode() { + return Objects.hash(label, navigation); + } + } + + public static class ClassifierDescriptor implements Descriptor { + private final Optional label; + private final Navigation navigation; + + public ClassifierDescriptor(Optional label, Navigation navigation) { + this.label = requireNonNull(label, "label is null"); + this.navigation = requireNonNull(navigation, "navigation is null"); + } + + public Optional getLabel() { + return label; + } + + public Navigation getNavigation() { + return navigation; + } + + @Override + public String toString() { + return "ClassifierDescriptor{" + "label=" + label + ", navigation=" + navigation + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ClassifierDescriptor that = (ClassifierDescriptor) o; + return Objects.equals(label, that.label) && Objects.equals(navigation, that.navigation); + } + + @Override + public int hashCode() { + return Objects.hash(label, navigation); + } + } + + public static class MatchNumberDescriptor implements Descriptor {} + + public enum NavigationAnchor { + FIRST, + LAST + } + + public static class Navigation { + public static final Navigation DEFAULT = + new Navigation(NavigationAnchor.LAST, NavigationMode.RUNNING, 0, 0); + + private final NavigationAnchor anchor; + private final NavigationMode mode; + private final int logicalOffset; + private final int physicalOffset; + + public Navigation( + NavigationAnchor anchor, NavigationMode mode, int logicalOffset, int physicalOffset) { + this.anchor = requireNonNull(anchor, "anchor is null"); + this.mode = requireNonNull(mode, "mode is null"); + this.logicalOffset = logicalOffset; + this.physicalOffset = physicalOffset; + } + + public NavigationAnchor getAnchor() { + return anchor; + } + + public NavigationMode getMode() { + return mode; + } + + public int getLogicalOffset() { + return logicalOffset; + } + + public int getPhysicalOffset() { + return physicalOffset; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/PatternRecognitionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/PatternRecognitionAnalyzer.java new file mode 100644 index 000000000000..58a3cbfc2758 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/PatternRecognitionAnalyzer.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer; + +import org.apache.iotdb.db.exception.sql.SemanticException; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis.Range; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExcludedPattern; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.FunctionCall; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Identifier; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.MeasureDefinition; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation.RowsPerMatch; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RangeQuantifier; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowPattern; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubsetDefinition; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.VariableDefinition; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; +import com.google.common.collect.Streams; + +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.ExpressionTreeUtils.extractExpressions; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ProcessingMode.Mode.FINAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.util.AstUtil.preOrder; + +public class PatternRecognitionAnalyzer { + private PatternRecognitionAnalyzer() {} + + public static PatternRecognitionAnalysis analyze( + List subsets, + List variableDefinitions, + List measures, + RowPattern pattern, + Optional skipTo) { + // extract label names (Identifiers) from PATTERN and SUBSET clauses. create labels respecting + // SQL identifier semantics + Set primaryLabels = + extractExpressions(ImmutableList.of(pattern), Identifier.class).stream() + .map(PatternRecognitionAnalyzer::label) + .collect(toImmutableSet()); + List unionLabels = + subsets.stream() + .map(SubsetDefinition::getName) + .map(PatternRecognitionAnalyzer::label) + .collect(toImmutableList()); + + // analyze SUBSET + Set unique = new HashSet<>(); + for (SubsetDefinition subset : subsets) { + String label = label(subset.getName()); + if (primaryLabels.contains(label)) { + throw new SemanticException( + String.format( + "union pattern variable name: %s is a duplicate of primary pattern variable name", + subset.getName())); + } + if (!unique.add(label)) { + throw new SemanticException( + String.format("union pattern variable name: %s is declared twice", subset.getName())); + } + for (Identifier element : subset.getIdentifiers()) { + // TODO can there be repetitions in the list of subset elements? (currently repetitions are + // supported) + if (!primaryLabels.contains(label(element))) { + throw new SemanticException( + String.format("subset element: %s is not a primary pattern variable", element)); + } + } + } + + // analyze DEFINE + unique = new HashSet<>(); + for (VariableDefinition definition : variableDefinitions) { + String label = label(definition.getName()); + if (!primaryLabels.contains(label)) { + throw new SemanticException( + String.format( + "defined variable: %s is not a primary pattern variable", definition.getName())); + } + if (!unique.add(label)) { + throw new SemanticException( + String.format("pattern variable with name: %s is defined twice", definition.getName())); + } + // DEFINE clause only supports RUNNING semantics which is default + Expression expression = definition.getExpression(); + extractExpressions(ImmutableList.of(expression), FunctionCall.class).stream() + .filter( + functionCall -> + functionCall + .getProcessingMode() + .map(mode -> mode.getMode() == FINAL) + .orElse(false)) + .findFirst() + .ifPresent( + functionCall -> { + throw new SemanticException( + String.format("FINAL semantics is not supported in DEFINE clause")); + }); + } + // record primary labels without definitions. they are implicitly associated with `true` + // condition + Set undefinedLabels = Sets.difference(primaryLabels, unique); + + // validate pattern quantifiers + ImmutableMap.Builder, Range> ranges = ImmutableMap.builder(); + preOrder(pattern) + .filter(RangeQuantifier.class::isInstance) + .map(RangeQuantifier.class::cast) + .forEach( + quantifier -> { + Optional atLeast = quantifier.getAtLeast().map(LongLiteral::getParsedValue); + atLeast.ifPresent( + value -> { + if (value < 0) { + throw new SemanticException( + "Pattern quantifier lower bound must be greater than or equal to 0"); + } + if (value > Integer.MAX_VALUE) { + throw new SemanticException( + "Pattern quantifier lower bound must not exceed " + Integer.MAX_VALUE); + } + }); + Optional atMost = quantifier.getAtMost().map(LongLiteral::getParsedValue); + atMost.ifPresent( + value -> { + if (value < 1) { + throw new SemanticException( + "Pattern quantifier upper bound must be greater than or equal to 1"); + } + if (value > Integer.MAX_VALUE) { + throw new SemanticException( + "Pattern quantifier upper bound must not exceed " + Integer.MAX_VALUE); + } + }); + if (atLeast.isPresent() && atMost.isPresent()) { + if (atLeast.get() > atMost.get()) { + throw new SemanticException( + "Pattern quantifier lower bound must not exceed upper bound"); + } + } + ranges.put( + NodeRef.of(quantifier), + new Range(atLeast.map(Math::toIntExact), atMost.map(Math::toIntExact))); + }); + + // validate AFTER MATCH SKIP + Set allLabels = + ImmutableSet.builder().addAll(primaryLabels).addAll(unionLabels).build(); + skipTo + .flatMap(SkipTo::getIdentifier) + .ifPresent( + identifier -> { + String label = label(identifier); + if (!allLabels.contains(label)) { + throw new SemanticException( + String.format("%s is not a primary or union pattern variable", identifier)); + } + }); + + // check no prohibited nesting: cannot nest one row pattern recognition within another + List expressions = + Streams.concat( + measures.stream().map(MeasureDefinition::getExpression), + variableDefinitions.stream().map(VariableDefinition::getExpression)) + .collect(toImmutableList()); + expressions.forEach( + expression -> + preOrder(expression) + .filter( + child -> + child instanceof PatternRecognitionRelation || child instanceof RowPattern) + .findFirst() + .ifPresent( + nested -> { + throw new SemanticException( + "nested row pattern recognition in row pattern recognition"); + })); + + return new PatternRecognitionAnalysis(allLabels, undefinedLabels, ranges.buildOrThrow()); + } + + public static void validatePatternExclusions( + Optional rowsPerMatch, RowPattern pattern) { + // exclusion syntax is not allowed in row pattern if ALL ROWS PER MATCH WITH UNMATCHED ROWS is + // specified + if (rowsPerMatch.isPresent() && (rowsPerMatch.get() == RowsPerMatch.ALL_WITH_UNMATCHED)) { + preOrder(pattern) + .filter(ExcludedPattern.class::isInstance) + .findFirst() + .ifPresent( + exclusion -> { + throw new SemanticException( + "Pattern exclusion syntax is not allowed when ALL ROWS PER MATCH WITH UNMATCHED ROWS is specified"); + }); + } + } + + private static String label(Identifier identifier) { + return identifier.getCanonicalValue(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java index 5ec2fda249ac..2d805c4f34a6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java @@ -96,10 +96,12 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Literal; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.MeasureDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NaturalJoin; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Offset; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PipeEnriched; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Property; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; @@ -124,11 +126,13 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ShowTopics; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubsetDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TableSubquery; @@ -137,6 +141,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.UpdateAssignment; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Values; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.VariableDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.With; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WithQuery; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WrappedInsertStatement; @@ -166,6 +171,7 @@ import java.util.Iterator; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; @@ -198,10 +204,12 @@ import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.Scope.BasisType.TABLE; import static org.apache.iotdb.db.queryengine.plan.relational.metadata.MetadataUtil.createQualifiedObjectName; import static org.apache.iotdb.db.queryengine.plan.relational.metadata.TableMetadataImpl.isTimestampType; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DereferenceExpression.getQualifiedName; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join.Type.FULL; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join.Type.INNER; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join.Type.LEFT; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Join.Type.RIGHT; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation.RowsPerMatch.ONE; import static org.apache.iotdb.db.queryengine.plan.relational.sql.util.AstUtil.preOrder; import static org.apache.iotdb.db.queryengine.plan.relational.utils.NodeUtils.getSortItemsFromOrderBy; import static org.apache.iotdb.db.storageengine.load.metrics.LoadTsFileCostMetricsSet.ANALYSIS; @@ -1553,7 +1561,7 @@ private Scope computeAndAssignOutputScope( if (expression instanceof Identifier) { name = QualifiedName.of(((Identifier) expression).getValue()); } else if (expression instanceof DereferenceExpression) { - name = DereferenceExpression.getQualifiedName((DereferenceExpression) expression); + name = getQualifiedName((DereferenceExpression) expression); } if (name != null) { @@ -1882,6 +1890,232 @@ private List analyzeTableOutputFields( // accessControlScope, filter)); // } + protected Scope visitPatternRecognitionRelation( + PatternRecognitionRelation relation, Optional scope) { + Scope inputScope = process(relation.getInput(), scope); + + // check that input table column names are not ambiguous + // Note: This check is not compliant with SQL identifier semantics. Quoted identifiers should + // have different comparison rules than unquoted identifiers. + // However, field names do not contain the information about quotation, and so every + // comparison is case-insensitive. For example, if there are fields named + // 'a' and 'A' (quoted), they should be considered non-ambiguous. However, their names will be + // compared case-insensitive and will cause failure as ambiguous. + Set inputNames = new HashSet<>(); + for (Field field : inputScope.getRelationType().getAllFields()) { + field + .getName() + .ifPresent( + name -> { + if (!inputNames.add(name.toUpperCase(ENGLISH))) { + throw new SemanticException( + String.format("ambiguous column: %s in row pattern input relation", name)); + } + }); + } + + // analyze PARTITION BY + for (Expression expression : relation.getPartitionBy()) { + // The PARTITION BY clause is a list of columns of the row pattern input table. + validateAndGetInputField(expression, inputScope); + Type type = analyzeExpression(expression, inputScope).getType(expression); + if (!type.isComparable()) { + throw new SemanticException( + String.format( + "%s is not comparable, and therefore cannot be used in PARTITION BY", type)); + } + } + + // analyze ORDER BY + for (SortItem sortItem : getSortItemsFromOrderBy(relation.getOrderBy())) { + // The ORDER BY clause is a list of columns of the row pattern input table. + Expression expression = sortItem.getSortKey(); + validateAndGetInputField(expression, inputScope); + Type type = analyzeExpression(expression, inputScope).getType(sortItem.getSortKey()); + if (!type.isOrderable()) { + throw new SemanticException( + String.format("%s is not orderable, and therefore cannot be used in ORDER BY", type)); + } + } + + // analyze pattern recognition clauses + PatternRecognitionAnalysis patternRecognitionAnalysis = + PatternRecognitionAnalyzer.analyze( + relation.getSubsets(), + relation.getVariableDefinitions(), + relation.getMeasures(), + relation.getPattern(), + relation.getAfterMatchSkipTo()); + + relation + .getAfterMatchSkipTo() + .flatMap(SkipTo::getIdentifier) + .ifPresent(label -> analysis.addResolvedLabel(label, label.getCanonicalValue())); + + for (SubsetDefinition subset : relation.getSubsets()) { + analysis.addResolvedLabel(subset.getName(), subset.getName().getCanonicalValue()); + analysis.addSubsetLabels( + subset, + subset.getIdentifiers().stream() + .map(Identifier::getCanonicalValue) + .collect(Collectors.toSet())); + } + + analysis.setUndefinedLabels( + relation.getPattern(), patternRecognitionAnalysis.getUndefinedLabels()); + analysis.setRanges(patternRecognitionAnalysis.getRanges()); + + PatternRecognitionAnalyzer.validatePatternExclusions( + relation.getRowsPerMatch(), relation.getPattern()); + + // Notes on potential name ambiguity between pattern labels and other identifiers: + // Labels are allowed in expressions of MEASURES and DEFINE clauses. In those expressions, + // qualifying column names with table name is not allowed. + // Theoretically, user might define pattern label "T" where input table name was "T". Then a + // dereference "T.column" would refer to: + // - input table's column, if it was in PARTITION BY or ORDER BY clause, + // - subset of rows matched with label "T", if it was in MEASURES or DEFINE clause. + // There could be a check to catch such non-intuitive situation and produce a warning. + // Similarly, it is possible to define pattern label with the same name as some input column. + // However, this causes no ambiguity, as labels can only + // appear as column name's prefix, and column names in pattern recognition context cannot be + // dereferenced. + + // analyze expressions in MEASURES and DEFINE (with set of all labels passed as context) + for (VariableDefinition variableDefinition : relation.getVariableDefinitions()) { + Expression expression = variableDefinition.getExpression(); + ExpressionAnalysis expressionAnalysis = + analyzePatternRecognitionExpression( + expression, inputScope, patternRecognitionAnalysis.getAllLabels()); + analysis.recordSubqueries(relation, expressionAnalysis); + analysis.addResolvedLabel( + variableDefinition.getName(), variableDefinition.getName().getCanonicalValue()); + Type type = expressionAnalysis.getType(expression); + if (!type.equals(BOOLEAN)) { + throw new SemanticException( + String.format("Expression defining a label must be boolean (actual type: %s)", type)); + } + } + ImmutableMap.Builder, Type> measureTypesBuilder = ImmutableMap.builder(); + for (MeasureDefinition measureDefinition : relation.getMeasures()) { + Expression expression = measureDefinition.getExpression(); + ExpressionAnalysis expressionAnalysis = + analyzePatternRecognitionExpression( + expression, inputScope, patternRecognitionAnalysis.getAllLabels()); + analysis.recordSubqueries(relation, expressionAnalysis); + analysis.addResolvedLabel( + measureDefinition.getName(), measureDefinition.getName().getCanonicalValue()); + measureTypesBuilder.put(NodeRef.of(expression), expressionAnalysis.getType(expression)); + } + Map, Type> measureTypes = measureTypesBuilder.buildOrThrow(); + + // create output scope + // ONE ROW PER MATCH: PARTITION BY columns, then MEASURES columns in order of declaration + // ALL ROWS PER MATCH: PARTITION BY columns, ORDER BY columns, MEASURES columns, then any + // remaining input table columns in order of declaration + // Note: row pattern input table name should not be exposed on output + PatternRecognitionRelation.RowsPerMatch rowsPerMatch = relation.getRowsPerMatch().orElse(ONE); + boolean oneRowPerMatch = rowsPerMatch == ONE; + + ImmutableSet.Builder inputFieldsOnOutputBuilder = ImmutableSet.builder(); + ImmutableList.Builder outputFieldsBuilder = ImmutableList.builder(); + + for (Expression expression : relation.getPartitionBy()) { + Field inputField = validateAndGetInputField(expression, inputScope); + outputFieldsBuilder.add(unqualifiedVisible(inputField)); + inputFieldsOnOutputBuilder.add(inputField); + } + if (!oneRowPerMatch) { + for (SortItem sortItem : getSortItemsFromOrderBy(relation.getOrderBy())) { + Field inputField = validateAndGetInputField(sortItem.getSortKey(), inputScope); + outputFieldsBuilder.add(unqualifiedVisible(inputField)); + inputFieldsOnOutputBuilder.add( + inputField); // might have duplicates (ORDER BY a ASC, a DESC) + } + } + for (MeasureDefinition measureDefinition : relation.getMeasures()) { + outputFieldsBuilder.add( + Field.newUnqualified( + measureDefinition.getName().getValue(), + measureTypes.get(NodeRef.of(measureDefinition.getExpression())), + TsTableColumnCategory.ATTRIBUTE)); + } + if (!oneRowPerMatch) { + Set inputFieldsOnOutput = inputFieldsOnOutputBuilder.build(); + for (Field inputField : inputScope.getRelationType().getAllFields()) { + if (!inputFieldsOnOutput.contains(inputField)) { + outputFieldsBuilder.add(unqualified(inputField)); + } + } + } + // pattern recognition output must have at least 1 column + List outputFields = outputFieldsBuilder.build(); + if (outputFields.isEmpty()) { + throw new SemanticException("pattern recognition output table has no columns"); + } + + return createAndAssignScope(relation, scope, outputFields); + } + + private Field validateAndGetInputField(Expression expression, Scope inputScope) { + QualifiedName qualifiedName; + if (expression instanceof Identifier) { + qualifiedName = QualifiedName.of(ImmutableList.of((Identifier) expression)); + } else if (expression instanceof DereferenceExpression) { + qualifiedName = getQualifiedName((DereferenceExpression) expression); + } else { + throw new SemanticException( + String.format("Expected column reference. Actual: %s", expression)); + } + Optional field = inputScope.tryResolveField(expression, qualifiedName); + if (!field.isPresent() || !field.get().isLocal()) { + throw new SemanticException( + String.format("Column %s is not present in the input relation", expression)); + } + + return field.get().getField(); + } + + private Field unqualifiedVisible(Field field) { + return new Field( + Optional.empty(), + field.getName(), + field.getType(), + field.getColumnCategory(), + false, + field.getOriginTable(), + field.getOriginColumnName(), + field.isAliased()); + } + + private Field unqualified(Field field) { + return new Field( + Optional.empty(), + field.getName(), + field.getType(), + field.getColumnCategory(), + field.isHidden(), + field.getOriginTable(), + field.getOriginColumnName(), + field.isAliased()); + } + + private ExpressionAnalysis analyzePatternRecognitionExpression( + Expression expression, Scope scope, Set labels) { + + return ExpressionAnalyzer.analyzePatternRecognitionExpression( + metadata, + queryContext, + sessionContext, + statementAnalyzerFactory, + accessControl, + scope, + analysis, + expression, + warningCollector, + labels); + } + @Override protected Scope visitValues(Values node, Optional scope) { checkState(!node.getRows().isEmpty()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java index a917d3d040f8..585829e25401 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/ir/ExpressionTreeRewriter.java @@ -447,7 +447,8 @@ public Expression visitFunctionCall(FunctionCall node, Context context) { List arguments = rewrite(node.getArguments(), context); if (!sameElements(node.getArguments(), arguments)) { - return new FunctionCall(node.getName(), arguments); + return new FunctionCall( + node.getName(), node.isDistinct(), node.getProcessingMode(), arguments); } return node; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AnchorPattern.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AnchorPattern.java new file mode 100644 index 000000000000..e8ced6283957 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AnchorPattern.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class AnchorPattern extends RowPattern { + public enum Type { + PARTITION_START, + PARTITION_END + } + + private final Type type; + + public AnchorPattern(NodeLocation location, Type type) { + super(location); + this.type = requireNonNull(type, "type is null"); + } + + public Type getType() { + return type; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitAnchorPattern(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + AnchorPattern o = (AnchorPattern) obj; + return Objects.equals(type, o.type); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } + + @Override + public String toString() { + return toStringHelper(this).add("type", type).toString(); + } + + @Override + public boolean shallowEquals(Node other) { + return sameClass(this, other); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java index 43bc9223b114..95e80577b1e3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AstVisitor.java @@ -169,6 +169,10 @@ protected R visitFunctionCall(FunctionCall node, C context) { return visitExpression(node, context); } + protected R visitProcessingMode(ProcessingMode node, C context) { + return visitNode(node, context); + } + protected R visitSimpleCaseExpression(SimpleCaseExpression node, C context) { return visitExpression(node, context); } @@ -616,4 +620,80 @@ protected R visitShowStatement(ShowStatement node, C context) { protected R visitKillQuery(KillQuery node, C context) { return visitStatement(node, context); } + + protected R visitMeasureDefinition(MeasureDefinition node, C context) { + return visitNode(node, context); + } + + protected R visitSkipTo(SkipTo node, C context) { + return visitNode(node, context); + } + + protected R visitSubsetDefinition(SubsetDefinition node, C context) { + return visitNode(node, context); + } + + protected R visitVariableDefinition(VariableDefinition node, C context) { + return visitNode(node, context); + } + + protected R visitPatternRecognitionRelation(PatternRecognitionRelation node, C context) { + return visitRelation(node, context); + } + + protected R visitRowPattern(RowPattern node, C context) { + return visitNode(node, context); + } + + protected R visitPatternAlternation(PatternAlternation node, C context) { + return visitRowPattern(node, context); + } + + protected R visitPatternConcatenation(PatternConcatenation node, C context) { + return visitRowPattern(node, context); + } + + protected R visitQuantifiedPattern(QuantifiedPattern node, C context) { + return visitRowPattern(node, context); + } + + protected R visitAnchorPattern(AnchorPattern node, C context) { + return visitRowPattern(node, context); + } + + protected R visitEmptyPattern(EmptyPattern node, C context) { + return visitRowPattern(node, context); + } + + protected R visitExcludedPattern(ExcludedPattern node, C context) { + return visitRowPattern(node, context); + } + + protected R visitPatternPermutation(PatternPermutation node, C context) { + return visitRowPattern(node, context); + } + + protected R visitPatternVariable(PatternVariable node, C context) { + return visitRowPattern(node, context); + } + + protected R visitPatternQuantifier(PatternQuantifier node, C context) { + return visitNode(node, context); + } + + protected R visitZeroOrMoreQuantifier(ZeroOrMoreQuantifier node, C context) { + return visitPatternQuantifier(node, context); + } + + protected R visitOneOrMoreQuantifier(OneOrMoreQuantifier node, C context) { + return visitPatternQuantifier(node, context); + } + + protected R visitZeroOrOneQuantifier(ZeroOrOneQuantifier node, C context) { + return visitPatternQuantifier(node, context); + } + + protected R visitRangeQuantifier(RangeQuantifier node, C context) { + return visitPatternQuantifier(node, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DefaultTraversalVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DefaultTraversalVisitor.java index cf5590d2ce4a..42e9a2c6ae6e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DefaultTraversalVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DefaultTraversalVisitor.java @@ -502,4 +502,52 @@ protected Void visitQuantifiedComparisonExpression( return null; } + + @Override + protected Void visitExcludedPattern(ExcludedPattern node, C context) { + process(node.getPattern(), context); + + return null; + } + + @Override + protected Void visitPatternAlternation(PatternAlternation node, C context) { + for (RowPattern rowPattern : node.getPatterns()) { + process(rowPattern, context); + } + + return null; + } + + @Override + protected Void visitPatternConcatenation(PatternConcatenation node, C context) { + for (RowPattern rowPattern : node.getPatterns()) { + process(rowPattern, context); + } + + return null; + } + + @Override + protected Void visitPatternPermutation(PatternPermutation node, C context) { + for (RowPattern rowPattern : node.getPatterns()) { + process(rowPattern, context); + } + + return null; + } + + @Override + protected Void visitPatternVariable(PatternVariable node, C context) { + process(node.getName(), context); + + return null; + } + + @Override + protected Void visitQuantifiedPattern(QuantifiedPattern node, C context) { + process(node.getPattern(), context); + + return null; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/EmptyPattern.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/EmptyPattern.java new file mode 100644 index 000000000000..85a316d71183 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/EmptyPattern.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +import static com.google.common.base.MoreObjects.toStringHelper; + +public class EmptyPattern extends RowPattern { + public EmptyPattern(NodeLocation location) { + super(location); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitEmptyPattern(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } + + @Override + public String toString() { + return toStringHelper(this).toString(); + } + + @Override + public boolean shallowEquals(Node other) { + return sameClass(this, other); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExcludedPattern.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExcludedPattern.java new file mode 100644 index 000000000000..9c69b3089f3c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ExcludedPattern.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class ExcludedPattern extends RowPattern { + private final RowPattern pattern; + + public ExcludedPattern(NodeLocation location, RowPattern pattern) { + super(location); + this.pattern = requireNonNull(pattern, "pattern is null"); + } + + public RowPattern getPattern() { + return pattern; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitExcludedPattern(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(pattern); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + ExcludedPattern o = (ExcludedPattern) obj; + return Objects.equals(pattern, o.pattern); + } + + @Override + public int hashCode() { + return Objects.hash(pattern); + } + + @Override + public String toString() { + return toStringHelper(this).add("pattern", pattern).toString(); + } + + @Override + public boolean shallowEquals(Node other) { + return sameClass(this, other); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java index 004e426ba201..74a834604a15 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java @@ -28,37 +28,51 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Optional; import static java.util.Objects.requireNonNull; public class FunctionCall extends Expression { private final QualifiedName name; private final boolean distinct; + private final Optional processingMode; private final List arguments; public FunctionCall(QualifiedName name, List arguments) { - super(null); - this.name = requireNonNull(name, "name is null"); - this.distinct = false; - this.arguments = requireNonNull(arguments, "arguments is null"); + this(null, name, false, Optional.empty(), arguments); } public FunctionCall(QualifiedName name, boolean distinct, List arguments) { - super(null); - this.name = requireNonNull(name, "name is null"); - this.distinct = distinct; - this.arguments = requireNonNull(arguments, "arguments is null"); + this(null, name, distinct, Optional.empty(), arguments); + } + + public FunctionCall( + QualifiedName name, Optional processingMode, List arguments) { + this(null, name, false, processingMode, arguments); + } + + public FunctionCall( + QualifiedName name, + boolean distinct, + Optional processingMode, + List arguments) { + this(null, name, distinct, processingMode, arguments); } public FunctionCall(NodeLocation location, QualifiedName name, List arguments) { - this(location, name, false, arguments); + this(location, name, false, Optional.empty(), arguments); } public FunctionCall( - NodeLocation location, QualifiedName name, boolean distinct, List arguments) { + NodeLocation location, + QualifiedName name, + boolean distinct, + Optional processingMode, + List arguments) { super(requireNonNull(location, "location is null")); this.name = requireNonNull(name, "name is null"); this.distinct = distinct; + this.processingMode = requireNonNull(processingMode, "processingMode is null"); this.arguments = requireNonNull(arguments, "arguments is null"); } @@ -70,6 +84,10 @@ public boolean isDistinct() { return distinct; } + public Optional getProcessingMode() { + return processingMode; + } + public List getArguments() { return arguments; } @@ -97,12 +115,13 @@ public boolean equals(Object obj) { FunctionCall o = (FunctionCall) obj; return Objects.equals(name, o.name) && Objects.equals(distinct, o.distinct) + && Objects.equals(processingMode, o.processingMode) && Objects.equals(arguments, o.arguments); } @Override public int hashCode() { - return Objects.hash(name, distinct, arguments); + return Objects.hash(name, distinct, processingMode, arguments); } @Override @@ -113,7 +132,9 @@ public boolean shallowEquals(Node other) { FunctionCall otherFunction = (FunctionCall) other; - return name.equals(otherFunction.name) && distinct == otherFunction.distinct; + return name.equals(otherFunction.name) + && distinct == otherFunction.distinct + && processingMode.equals(otherFunction.processingMode); } // =============== serialize ================= @@ -130,6 +151,14 @@ public void serialize(DataOutputStream stream) throws IOException { for (Expression argument : arguments) { Expression.serialize(argument, stream); } + + if (processingMode.isPresent()) { + ReadWriteIOUtils.write(true, stream); + ProcessingMode mode = processingMode.get(); + ReadWriteIOUtils.write(mode.getMode().name(), stream); + } else { + ReadWriteIOUtils.write(false, stream); + } } public FunctionCall(ByteBuffer byteBuffer) { @@ -141,5 +170,14 @@ public FunctionCall(ByteBuffer byteBuffer) { while (size-- > 0) { arguments.add(Expression.deserialize(byteBuffer)); } + + boolean hasProcessingMode = ReadWriteIOUtils.readBool(byteBuffer); + if (hasProcessingMode) { + String modeName = ReadWriteIOUtils.readString(byteBuffer); + ProcessingMode.Mode mode = ProcessingMode.Mode.valueOf(modeName); + this.processingMode = Optional.of(new ProcessingMode(null, mode)); + } else { + this.processingMode = Optional.empty(); + } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/MeasureDefinition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/MeasureDefinition.java new file mode 100644 index 000000000000..db6475307b30 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/MeasureDefinition.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class MeasureDefinition extends Node { + private final Expression expression; + private final Identifier name; + + public MeasureDefinition(NodeLocation location, Expression expression, Identifier name) { + super(location); + this.expression = requireNonNull(expression, "expression is null"); + this.name = requireNonNull(name, "name is null"); + } + + public Expression getExpression() { + return expression; + } + + public Identifier getName() { + return name; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitMeasureDefinition(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(expression); + } + + @Override + public String toString() { + return toStringHelper(this).add("expression", expression).add("name", name).toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + MeasureDefinition that = (MeasureDefinition) o; + return Objects.equals(expression, that.expression) && Objects.equals(name, that.name); + } + + @Override + public int hashCode() { + return Objects.hash(expression, name); + } + + @Override + public boolean shallowEquals(Node other) { + if (!sameClass(this, other)) { + return false; + } + + return Objects.equals(name, ((MeasureDefinition) other).name); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/OneOrMoreQuantifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/OneOrMoreQuantifier.java new file mode 100644 index 000000000000..ea82719f2bd2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/OneOrMoreQuantifier.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +public class OneOrMoreQuantifier extends PatternQuantifier { + public OneOrMoreQuantifier(NodeLocation location, boolean greedy) { + super(location, greedy); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitOneOrMoreQuantifier(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternAlternation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternAlternation.java new file mode 100644 index 000000000000..78e84fc6a2c6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternAlternation.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class PatternAlternation extends RowPattern { + private final List patterns; + + public PatternAlternation(NodeLocation location, List patterns) { + super(location); + this.patterns = requireNonNull(patterns, "patterns is null"); + checkArgument(!patterns.isEmpty(), "patterns list is empty"); + } + + public List getPatterns() { + return patterns; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitPatternAlternation(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.copyOf(patterns); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + PatternAlternation o = (PatternAlternation) obj; + return Objects.equals(patterns, o.patterns); + } + + @Override + public int hashCode() { + return Objects.hash(patterns); + } + + @Override + public String toString() { + return toStringHelper(this).add("patterns", patterns).toString(); + } + + @Override + public boolean shallowEquals(Node other) { + return sameClass(this, other); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternConcatenation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternConcatenation.java new file mode 100644 index 000000000000..2d0836d4c475 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternConcatenation.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class PatternConcatenation extends RowPattern { + private final List patterns; + + public PatternConcatenation(NodeLocation location, List patterns) { + super(location); + this.patterns = requireNonNull(patterns, "patterns is null"); + checkArgument(!patterns.isEmpty(), "patterns list is empty"); + } + + public List getPatterns() { + return patterns; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitPatternConcatenation(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.copyOf(patterns); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + PatternConcatenation o = (PatternConcatenation) obj; + return Objects.equals(patterns, o.patterns); + } + + @Override + public int hashCode() { + return Objects.hash(patterns); + } + + @Override + public String toString() { + return toStringHelper(this).add("patterns", patterns).toString(); + } + + @Override + public boolean shallowEquals(Node other) { + return sameClass(this, other); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternPermutation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternPermutation.java new file mode 100644 index 000000000000..577b1005ff0c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternPermutation.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class PatternPermutation extends RowPattern { + private final List patterns; + + public PatternPermutation(NodeLocation location, List patterns) { + super(location); + this.patterns = requireNonNull(patterns, "patterns is null"); + checkArgument(!patterns.isEmpty(), "patterns list is empty"); + } + + public List getPatterns() { + return patterns; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitPatternPermutation(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.copyOf(patterns); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + PatternPermutation o = (PatternPermutation) obj; + return Objects.equals(patterns, o.patterns); + } + + @Override + public int hashCode() { + return Objects.hash(patterns); + } + + @Override + public String toString() { + return toStringHelper(this).add("patterns", patterns).toString(); + } + + @Override + public boolean shallowEquals(Node other) { + return sameClass(this, other); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternQuantifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternQuantifier.java new file mode 100644 index 000000000000..e94249e93087 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternQuantifier.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; + +public abstract class PatternQuantifier extends Node { + private final boolean greedy; + + protected PatternQuantifier(NodeLocation location, boolean greedy) { + super(location); + this.greedy = greedy; + } + + public boolean isGreedy() { + return greedy; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitPatternQuantifier(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + PatternQuantifier o = (PatternQuantifier) obj; + return greedy == o.greedy; + } + + @Override + public int hashCode() { + return Objects.hash(greedy); + } + + @Override + public boolean shallowEquals(Node other) { + if (!sameClass(this, other)) { + return false; + } + + PatternQuantifier otherNode = (PatternQuantifier) other; + return greedy == otherNode.greedy; + } + + @Override + public String toString() { + return toStringHelper(this).add("greedy", greedy).toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternRecognitionRelation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternRecognitionRelation.java new file mode 100644 index 000000000000..479421245c5e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternRecognitionRelation.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class PatternRecognitionRelation extends Relation { + private final Relation input; + private final List partitionBy; + private final Optional orderBy; + private final List measures; + private final Optional rowsPerMatch; + private final Optional afterMatchSkipTo; + private final RowPattern pattern; + private final List subsets; + private final List variableDefinitions; + + public PatternRecognitionRelation( + NodeLocation location, + Relation input, + List partitionBy, + Optional orderBy, + List measures, + Optional rowsPerMatch, + Optional afterMatchSkipTo, + RowPattern pattern, + List subsets, + List variableDefinitions) { + super(location); + this.input = requireNonNull(input, "input is null"); + this.partitionBy = requireNonNull(partitionBy, "partitionBy is null"); + this.orderBy = requireNonNull(orderBy, "orderBy is null"); + this.measures = requireNonNull(measures, "measures is null"); + this.rowsPerMatch = requireNonNull(rowsPerMatch, "rowsPerMatch is null"); + this.afterMatchSkipTo = requireNonNull(afterMatchSkipTo, "afterMatchSkipTo is null"); + this.pattern = requireNonNull(pattern, "pattern is null"); + this.subsets = requireNonNull(subsets, "subsets is null"); + requireNonNull(variableDefinitions, "variableDefinitions is null"); + checkArgument(!variableDefinitions.isEmpty(), "variableDefinitions is empty"); + this.variableDefinitions = variableDefinitions; + } + + public Relation getInput() { + return input; + } + + public List getPartitionBy() { + return partitionBy; + } + + public Optional getOrderBy() { + return orderBy; + } + + public List getMeasures() { + return measures; + } + + public Optional getRowsPerMatch() { + return rowsPerMatch; + } + + public Optional getAfterMatchSkipTo() { + return afterMatchSkipTo; + } + + public RowPattern getPattern() { + return pattern; + } + + public List getSubsets() { + return subsets; + } + + public List getVariableDefinitions() { + return variableDefinitions; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitPatternRecognitionRelation(this, context); + } + + @Override + public List getChildren() { + ImmutableList.Builder builder = ImmutableList.builder(); + builder.add(input); + builder.addAll(partitionBy); + orderBy.ifPresent(builder::add); + builder.addAll(measures); + afterMatchSkipTo.ifPresent(builder::add); + builder.add(pattern).addAll(subsets).addAll(variableDefinitions); + + return builder.build(); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("input", input) + .add("partitionBy", partitionBy) + .add("orderBy", orderBy.orElse(null)) + .add("measures", measures) + .add("rowsPerMatch", rowsPerMatch.orElse(null)) + .add("afterMatchSkipTo", afterMatchSkipTo) + .add("pattern", pattern) + .add("subsets", subsets) + .add("variableDefinitions", variableDefinitions) + .omitNullValues() + .toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + PatternRecognitionRelation that = (PatternRecognitionRelation) o; + return Objects.equals(input, that.input) + && Objects.equals(partitionBy, that.partitionBy) + && Objects.equals(orderBy, that.orderBy) + && Objects.equals(measures, that.measures) + && Objects.equals(rowsPerMatch, that.rowsPerMatch) + && Objects.equals(afterMatchSkipTo, that.afterMatchSkipTo) + && Objects.equals(pattern, that.pattern) + && Objects.equals(subsets, that.subsets) + && Objects.equals(variableDefinitions, that.variableDefinitions); + } + + @Override + public int hashCode() { + return Objects.hash( + input, + partitionBy, + orderBy, + measures, + rowsPerMatch, + afterMatchSkipTo, + pattern, + subsets, + variableDefinitions); + } + + @Override + public boolean shallowEquals(Node other) { + if (!sameClass(this, other)) { + return false; + } + + return rowsPerMatch.equals(((PatternRecognitionRelation) other).rowsPerMatch); + } + + public enum RowsPerMatch { + // ONE option applies to the MATCH_RECOGNIZE clause. This is the default option. + // Output a single summary row for every match, including empty matches. + // In the case of an empty match, output the starting row of the match attempt. + ONE, + + // ALL_SHOW_EMPTY option applies to the MATCH_RECOGNIZE clause. + // Output all rows of every match, including empty matches. + // In the case of an empty match, output the starting row of the match attempt. + // Do not produce output for the rows matched within exclusion `{- ... -}`. + ALL_SHOW_EMPTY, + + // ALL_OMIT_EMPTY option applies to the MATCH_RECOGNIZE clause. + // Output all rows of every non-empty match. + // Do not produce output for the rows matched within exclusion `{- ... -}` + ALL_OMIT_EMPTY, + + // ALL_WITH_UNMATCHED option applies to the MATCH_RECOGNIZE clause. + // Output all rows of every match, including empty matches. + // Produce an additional output row for every unmatched row. + // Pattern exclusions are not allowed with this option. + ALL_WITH_UNMATCHED + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternVariable.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternVariable.java new file mode 100644 index 000000000000..ce9dcc4fe6c6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/PatternVariable.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class PatternVariable extends RowPattern { + private final Identifier name; + + public PatternVariable(NodeLocation location, Identifier name) { + super(location); + this.name = requireNonNull(name, "name is null"); + } + + public Identifier getName() { + return name; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitPatternVariable(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(name); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + PatternVariable o = (PatternVariable) obj; + return Objects.equals(name, o.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public String toString() { + return toStringHelper(this).add("name", name).toString(); + } + + @Override + public boolean shallowEquals(Node other) { + return sameClass(this, other); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ProcessingMode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ProcessingMode.java new file mode 100644 index 000000000000..780a5b9a1331 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ProcessingMode.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public final class ProcessingMode extends Node { + private final Mode mode; + + public ProcessingMode(NodeLocation location, Mode mode) { + super(location); + this.mode = requireNonNull(mode, "mode is null"); + } + + public Mode getMode() { + return mode; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitProcessingMode(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + return mode == ((ProcessingMode) obj).mode; + } + + @Override + public int hashCode() { + return Objects.hash(mode); + } + + @Override + public String toString() { + return toStringHelper(this).add("mode", mode).toString(); + } + + public enum Mode { + RUNNING, + FINAL + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuantifiedPattern.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuantifiedPattern.java new file mode 100644 index 000000000000..f4b6ae3944d1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/QuantifiedPattern.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class QuantifiedPattern extends RowPattern { + private final RowPattern pattern; + private final PatternQuantifier patternQuantifier; + + public QuantifiedPattern( + NodeLocation location, RowPattern pattern, PatternQuantifier patternQuantifier) { + super(location); + this.pattern = requireNonNull(pattern, "pattern is null"); + this.patternQuantifier = requireNonNull(patternQuantifier, "patternQuantifier is null"); + } + + public RowPattern getPattern() { + return pattern; + } + + public PatternQuantifier getPatternQuantifier() { + return patternQuantifier; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitQuantifiedPattern(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(pattern, patternQuantifier); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + QuantifiedPattern o = (QuantifiedPattern) obj; + return Objects.equals(pattern, o.pattern) + && Objects.equals(patternQuantifier, o.patternQuantifier); + } + + @Override + public int hashCode() { + return Objects.hash(pattern, patternQuantifier); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("pattern", pattern) + .add("patternQuantifier", patternQuantifier) + .toString(); + } + + @Override + public boolean shallowEquals(Node other) { + return sameClass(this, other); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RangeQuantifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RangeQuantifier.java new file mode 100644 index 000000000000..bf41ea2d8960 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RangeQuantifier.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class RangeQuantifier extends PatternQuantifier { + private final Optional atLeast; + private final Optional atMost; + + public RangeQuantifier( + NodeLocation location, + boolean greedy, + Optional atLeast, + Optional atMost) { + super(location, greedy); + this.atLeast = requireNonNull(atLeast, "atLeast is null"); + this.atMost = requireNonNull(atMost, "atMost is null"); + } + + public Optional getAtLeast() { + return atLeast; + } + + public Optional getAtMost() { + return atMost; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitRangeQuantifier(this, context); + } + + @Override + public List getChildren() { + ImmutableList.Builder children = ImmutableList.builder(); + atLeast.ifPresent(children::add); + atMost.ifPresent(children::add); + return children.build(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + RangeQuantifier o = (RangeQuantifier) obj; + return isGreedy() == o.isGreedy() + && Objects.equals(atLeast, o.atLeast) + && Objects.equals(atMost, o.atMost); + } + + @Override + public int hashCode() { + return Objects.hash(isGreedy(), atLeast, atMost); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("atLeast", atLeast) + .add("atMost", atMost) + .add("greedy", isGreedy()) + .toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowPattern.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowPattern.java new file mode 100644 index 000000000000..af5bf7678601 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/RowPattern.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +public abstract class RowPattern extends Node { + protected RowPattern(NodeLocation location) { + super(location); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitRowPattern(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SkipTo.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SkipTo.java new file mode 100644 index 000000000000..9a5b9e809368 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SkipTo.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo.Position.FIRST; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo.Position.LAST; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo.Position.NEXT; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo.Position.PAST_LAST; + +public class SkipTo extends Node { + private final Position position; + private final Optional identifier; + + public enum Position { + PAST_LAST, + NEXT, + FIRST, + LAST + } + + // default + public static SkipTo skipPastLastRow(NodeLocation location) { + return new SkipTo(location, PAST_LAST, Optional.empty()); + } + + public static SkipTo skipToNextRow(NodeLocation location) { + return new SkipTo(location, NEXT, Optional.empty()); + } + + public static SkipTo skipToFirst(NodeLocation location, Identifier identifier) { + return new SkipTo(location, FIRST, Optional.of(identifier)); + } + + public static SkipTo skipToLast(NodeLocation location, Identifier identifier) { + return new SkipTo(location, LAST, Optional.of(identifier)); + } + + private SkipTo(NodeLocation location, Position position, Optional identifier) { + super(location); + requireNonNull(position, "position is null"); + requireNonNull(identifier, "identifier is null"); + checkArgument( + identifier.isPresent() || (position == PAST_LAST || position == NEXT), + "missing identifier in SKIP TO %s", + position.name()); + checkArgument( + !identifier.isPresent() || (position == FIRST || position == LAST), + "unexpected identifier in SKIP TO %s", + position.name()); + this.position = position; + this.identifier = identifier; + } + + public Position getPosition() { + return position; + } + + public Optional getIdentifier() { + return identifier; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitSkipTo(this, context); + } + + @Override + public List getChildren() { + return identifier.map(ImmutableList::of).orElse(ImmutableList.of()); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("position", position) + .add("identifier", identifier.orElse(null)) + .omitNullValues() + .toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + SkipTo that = (SkipTo) o; + return Objects.equals(position, that.position) && Objects.equals(identifier, that.identifier); + } + + @Override + public int hashCode() { + return Objects.hash(position, identifier); + } + + @Override + public boolean shallowEquals(Node other) { + if (!sameClass(this, other)) { + return false; + } + + return position == ((SkipTo) other).position; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SubsetDefinition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SubsetDefinition.java new file mode 100644 index 000000000000..058eacdbde02 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/SubsetDefinition.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class SubsetDefinition extends Node { + private final Identifier name; + private final List identifiers; + + public SubsetDefinition(NodeLocation location, Identifier name, List identifiers) { + super(location); + this.name = requireNonNull(name, "name is null"); + requireNonNull(identifiers, "identifiers is null"); + checkArgument(!identifiers.isEmpty(), "identifiers is empty"); + this.identifiers = identifiers; + } + + public Identifier getName() { + return name; + } + + public List getIdentifiers() { + return identifiers; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitSubsetDefinition(this, context); + } + + @Override + public List getChildren() { + return identifiers; + } + + @Override + public String toString() { + return toStringHelper(this).add("name", name).add("identifiers", identifiers).toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + SubsetDefinition that = (SubsetDefinition) o; + return Objects.equals(name, that.name) && Objects.equals(identifiers, that.identifiers); + } + + @Override + public int hashCode() { + return Objects.hash(name, identifiers); + } + + @Override + public boolean shallowEquals(Node other) { + if (!sameClass(this, other)) { + return false; + } + + return Objects.equals(name, ((SubsetDefinition) other).name); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/VariableDefinition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/VariableDefinition.java new file mode 100644 index 000000000000..5af4d5f59133 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/VariableDefinition.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class VariableDefinition extends Node { + private final Identifier name; + private final Expression expression; + + public VariableDefinition(NodeLocation location, Identifier name, Expression expression) { + super(location); + this.name = requireNonNull(name, "name is null"); + this.expression = requireNonNull(expression, "expression is null"); + } + + public Identifier getName() { + return name; + } + + public Expression getExpression() { + return expression; + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitVariableDefinition(this, context); + } + + @Override + public List getChildren() { + return ImmutableList.of(expression); + } + + @Override + public String toString() { + return toStringHelper(this).add("name", name).add("expression", expression).toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + + VariableDefinition that = (VariableDefinition) o; + return Objects.equals(name, that.name) && Objects.equals(expression, that.expression); + } + + @Override + public int hashCode() { + return Objects.hash(name, expression); + } + + @Override + public boolean shallowEquals(Node other) { + if (!sameClass(this, other)) { + return false; + } + + return Objects.equals(name, ((VariableDefinition) other).name); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ZeroOrMoreQuantifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ZeroOrMoreQuantifier.java new file mode 100644 index 000000000000..c55091ba75e1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ZeroOrMoreQuantifier.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +public class ZeroOrMoreQuantifier extends PatternQuantifier { + public ZeroOrMoreQuantifier(NodeLocation location, boolean greedy) { + super(location, greedy); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitZeroOrMoreQuantifier(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ZeroOrOneQuantifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ZeroOrOneQuantifier.java new file mode 100644 index 000000000000..de4d6b829848 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ZeroOrOneQuantifier.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.ast; + +public class ZeroOrOneQuantifier extends PatternQuantifier { + public ZeroOrOneQuantifier(NodeLocation location, boolean greedy) { + super(location, greedy); + } + + @Override + public R accept(AstVisitor visitor, C context) { + return visitor.visitZeroOrOneQuantifier(this, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index c30aa8aaca29..eb39195141d6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -39,6 +39,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AllRows; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterDB; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AlterPipe; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AnchorPattern; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticBinaryExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ArithmeticUnaryExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BetweenPredicate; @@ -75,7 +76,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropPipePlugin; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.DropTopic; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.EmptyPattern; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Except; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExcludedPattern; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExistsPredicate; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Explain; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExplainAnalyze; @@ -106,6 +109,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LoadTsFile; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LogicalExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.LongLiteral; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.MeasureDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NaturalJoin; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NodeLocation; @@ -114,18 +118,30 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NullLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.NumericParameter; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Offset; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OneOrMoreQuantifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Parameter; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternAlternation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternConcatenation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternPermutation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternQuantifier; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation.RowsPerMatch; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternVariable; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ProcessingMode; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Property; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedComparisonExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedPattern; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QueryBody; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuerySpecification; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RangeQuantifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Relation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowPattern; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Select; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectItem; @@ -156,12 +172,14 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SingleColumn; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StartPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StopPipe; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubsetDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Table; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TableExpressionType; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.TableSubquery; @@ -172,9 +190,12 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.UpdateAssignment; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Use; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Values; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.VariableDefinition; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WhenClause; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.With; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.WithQuery; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ZeroOrMoreQuantifier; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ZeroOrOneQuantifier; import org.apache.iotdb.db.queryengine.plan.relational.sql.util.AstUtil; import org.apache.iotdb.db.queryengine.plan.statement.StatementType; import org.apache.iotdb.db.queryengine.plan.statement.crud.InsertRowStatement; @@ -231,10 +252,23 @@ import static org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinScalarFunction.DATE_BIN; import static org.apache.iotdb.db.queryengine.plan.execution.config.TableConfigTaskVisitor.DATABASE_NOT_SPECIFIED; import static org.apache.iotdb.db.queryengine.plan.parser.ASTVisitor.parseDateTimeFormat; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AnchorPattern.Type.PARTITION_END; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AnchorPattern.Type.PARTITION_START; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupingSets.Type.CUBE; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupingSets.Type.EXPLICIT; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.GroupingSets.Type.ROLLUP; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation.RowsPerMatch.ALL_OMIT_EMPTY; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation.RowsPerMatch.ALL_SHOW_EMPTY; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation.RowsPerMatch.ALL_WITH_UNMATCHED; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation.RowsPerMatch.ONE; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ProcessingMode.Mode.FINAL; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ProcessingMode.Mode.RUNNING; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName.mapIdentifier; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo.skipPastLastRow; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo.skipToFirst; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo.skipToLast; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo.skipToNextRow; +import static org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.TableBuiltinScalarFunction.DATE_BIN; import static org.apache.iotdb.db.utils.constant.SqlConstant.FIRST_AGGREGATION; import static org.apache.iotdb.db.utils.constant.SqlConstant.FIRST_BY_AGGREGATION; import static org.apache.iotdb.db.utils.constant.SqlConstant.LAST_AGGREGATION; @@ -271,6 +305,11 @@ public Node visitStandaloneType(RelationalSqlParser.StandaloneTypeContext contex return visit(context.type()); } + @Override + public Node visitStandaloneRowPattern(RelationalSqlParser.StandaloneRowPatternContext context) { + return visit(context.rowPattern()); + } + // ******************* statements ********************** @Override public Node visitUseDatabaseStatement(RelationalSqlParser.UseDatabaseStatementContext ctx) { @@ -1759,6 +1798,112 @@ public Node visitJoinRelation(RelationalSqlParser.JoinRelationContext ctx) { return new Join(getLocation(ctx), joinType, left, right, criteria); } + @Override + public Node visitPatternRecognition(RelationalSqlParser.PatternRecognitionContext context) { + Relation child = (Relation) visit(context.aliasedRelation()); + + if (context.MATCH_RECOGNIZE() == null) { + return child; + } + + Optional orderBy = Optional.empty(); + if (context.ORDER() != null) { + orderBy = + Optional.of( + new OrderBy(getLocation(context.ORDER()), visit(context.sortItem(), SortItem.class))); + } + + PatternRecognitionRelation relation = + new PatternRecognitionRelation( + getLocation(context), + child, + visit(context.partition, Expression.class), + orderBy, + visit(context.measureDefinition(), MeasureDefinition.class), + getRowsPerMatch(context.rowsPerMatch()), + visitIfPresent(context.skipTo(), SkipTo.class), + (RowPattern) visit(context.rowPattern()), + visit(context.subsetDefinition(), SubsetDefinition.class), + visit(context.variableDefinition(), VariableDefinition.class)); + + if (context.identifier() == null) { + return relation; + } + + List aliases = null; + if (context.columnAliases() != null) { + aliases = visit(context.columnAliases().identifier(), Identifier.class); + } + + return new AliasedRelation( + getLocation(context), relation, (Identifier) visit(context.identifier()), aliases); + } + + @Override + public Node visitMeasureDefinition(RelationalSqlParser.MeasureDefinitionContext context) { + return new MeasureDefinition( + getLocation(context), + (Expression) visit(context.expression()), + (Identifier) visit(context.identifier())); + } + + private Optional getRowsPerMatch(RelationalSqlParser.RowsPerMatchContext context) { + if (context == null) { + return Optional.empty(); + } + + if (context.ONE() != null) { + return Optional.of(ONE); + } + + if (context.emptyMatchHandling() == null) { + return Optional.of(ALL_SHOW_EMPTY); + } + + if (context.emptyMatchHandling().SHOW() != null) { + return Optional.of(ALL_SHOW_EMPTY); + } + + if (context.emptyMatchHandling().OMIT() != null) { + return Optional.of(ALL_OMIT_EMPTY); + } + + return Optional.of(ALL_WITH_UNMATCHED); + } + + @Override + public Node visitSkipTo(RelationalSqlParser.SkipToContext context) { + if (context.PAST() != null) { + return skipPastLastRow(getLocation(context)); + } + + if (context.NEXT() != null) { + return skipToNextRow(getLocation(context)); + } + + if (context.FIRST() != null) { + return skipToFirst(getLocation(context), (Identifier) visit(context.identifier())); + } + + return skipToLast(getLocation(context), (Identifier) visit(context.identifier())); + } + + @Override + public Node visitSubsetDefinition(RelationalSqlParser.SubsetDefinitionContext context) { + return new SubsetDefinition( + getLocation(context), + (Identifier) visit(context.name), + visit(context.union, Identifier.class)); + } + + @Override + public Node visitVariableDefinition(RelationalSqlParser.VariableDefinitionContext context) { + return new VariableDefinition( + getLocation(context), + (Identifier) visit(context.identifier()), + (Expression) visit(context.expression())); + } + @Override public Node visitAliasedRelation(RelationalSqlParser.AliasedRelationContext ctx) { Relation child = (Relation) visit(ctx.relationPrimary()); @@ -2117,12 +2262,15 @@ public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { boolean distinct = isDistinct(ctx.setQuantifier()); + RelationalSqlParser.ProcessingModeContext processingMode = ctx.processingMode(); + if (name.toString().equalsIgnoreCase("if")) { check( ctx.expression().size() == 2 || ctx.expression().size() == 3, "Invalid number of arguments for 'if' function", ctx); check(!distinct, "DISTINCT not valid for 'if' function", ctx); + check(processingMode == null, "Running or final semantics not valid for 'if' function", ctx); Expression elseExpression = null; if (ctx.expression().size() == 3) { @@ -2139,6 +2287,10 @@ public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { if (name.toString().equalsIgnoreCase("nullif")) { check(ctx.expression().size() == 2, "Invalid number of arguments for 'nullif' function", ctx); check(!distinct, "DISTINCT not valid for 'nullif' function", ctx); + check( + processingMode == null, + "Running or final semantics not valid for 'nullif' function", + ctx); return new NullIfExpression( getLocation(ctx), @@ -2152,10 +2304,23 @@ public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { "The 'coalesce' function must have at least two arguments", ctx); check(!distinct, "DISTINCT not valid for 'coalesce' function", ctx); + check( + processingMode == null, + "Running or final semantics not valid for 'coalesce' function", + ctx); return new CoalesceExpression(getLocation(ctx), visit(ctx.expression(), Expression.class)); } + Optional mode = Optional.empty(); + if (processingMode != null) { + if (processingMode.RUNNING() != null) { + mode = Optional.of(new ProcessingMode(getLocation(processingMode), RUNNING)); + } else if (processingMode.FINAL() != null) { + mode = Optional.of(new ProcessingMode(getLocation(processingMode), FINAL)); + } + } + List arguments = visit(ctx.expression(), Expression.class); if (ctx.label != null) { arguments = @@ -2197,7 +2362,7 @@ public Node visitFunctionCall(RelationalSqlParser.FunctionCallContext ctx) { } } - return new FunctionCall(getLocation(ctx), name, distinct, arguments); + return new FunctionCall(getLocation(ctx), name, distinct, mode, arguments); } @Override @@ -2278,6 +2443,105 @@ private long parseTimeValue(RelationalSqlParser.TimeValueContext ctx, long curre } } + @Override + public Node visitPatternAlternation(RelationalSqlParser.PatternAlternationContext context) { + List parts = visit(context.rowPattern(), RowPattern.class); + return new PatternAlternation(getLocation(context), parts); + } + + @Override + public Node visitPatternConcatenation(RelationalSqlParser.PatternConcatenationContext context) { + List parts = visit(context.rowPattern(), RowPattern.class); + return new PatternConcatenation(getLocation(context), parts); + } + + @Override + public Node visitQuantifiedPrimary(RelationalSqlParser.QuantifiedPrimaryContext context) { + RowPattern primary = (RowPattern) visit(context.patternPrimary()); + if (context.patternQuantifier() != null) { + return new QuantifiedPattern( + getLocation(context), primary, (PatternQuantifier) visit(context.patternQuantifier())); + } + return primary; + } + + @Override + public Node visitPatternVariable(RelationalSqlParser.PatternVariableContext context) { + return new PatternVariable(getLocation(context), (Identifier) visit(context.identifier())); + } + + @Override + public Node visitEmptyPattern(RelationalSqlParser.EmptyPatternContext context) { + return new EmptyPattern(getLocation(context)); + } + + @Override + public Node visitPatternPermutation(RelationalSqlParser.PatternPermutationContext context) { + return new PatternPermutation( + getLocation(context), visit(context.rowPattern(), RowPattern.class)); + } + + @Override + public Node visitGroupedPattern(RelationalSqlParser.GroupedPatternContext context) { + // skip parentheses + return visit(context.rowPattern()); + } + + @Override + public Node visitPartitionStartAnchor(RelationalSqlParser.PartitionStartAnchorContext context) { + return new AnchorPattern(getLocation(context), PARTITION_START); + } + + @Override + public Node visitPartitionEndAnchor(RelationalSqlParser.PartitionEndAnchorContext context) { + return new AnchorPattern(getLocation(context), PARTITION_END); + } + + @Override + public Node visitExcludedPattern(RelationalSqlParser.ExcludedPatternContext context) { + return new ExcludedPattern(getLocation(context), (RowPattern) visit(context.rowPattern())); + } + + @Override + public Node visitZeroOrMoreQuantifier(RelationalSqlParser.ZeroOrMoreQuantifierContext context) { + boolean greedy = context.reluctant == null; + return new ZeroOrMoreQuantifier(getLocation(context), greedy); + } + + @Override + public Node visitOneOrMoreQuantifier(RelationalSqlParser.OneOrMoreQuantifierContext context) { + boolean greedy = context.reluctant == null; + return new OneOrMoreQuantifier(getLocation(context), greedy); + } + + @Override + public Node visitZeroOrOneQuantifier(RelationalSqlParser.ZeroOrOneQuantifierContext context) { + boolean greedy = context.reluctant == null; + return new ZeroOrOneQuantifier(getLocation(context), greedy); + } + + @Override + public Node visitRangeQuantifier(RelationalSqlParser.RangeQuantifierContext context) { + boolean greedy = context.reluctant == null; + + Optional atLeast = Optional.empty(); + Optional atMost = Optional.empty(); + if (context.exactly != null) { + atLeast = + Optional.of(new LongLiteral(getLocation(context.exactly), context.exactly.getText())); + atMost = + Optional.of(new LongLiteral(getLocation(context.exactly), context.exactly.getText())); + } + if (context.atLeast != null) { + atLeast = + Optional.of(new LongLiteral(getLocation(context.atLeast), context.atLeast.getText())); + } + if (context.atMost != null) { + atMost = Optional.of(new LongLiteral(getLocation(context.atMost), context.atMost.getText())); + } + return new RangeQuantifier(getLocation(context), greedy, atLeast, atMost); + } + // ************** literals ************** @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java index d6222564ec7e..0f25f993267e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java @@ -67,6 +67,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; @@ -85,6 +86,7 @@ import java.util.Optional; import java.util.function.Function; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Iterables.getOnlyElement; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; @@ -290,6 +292,10 @@ protected String visitFunctionCall(FunctionCall node, Void context) { StringBuilder builder = new StringBuilder(); + if (node.getProcessingMode().isPresent()) { + builder.append(node.getProcessingMode().get().getMode()).append(" "); + } + String arguments = joinExpressions(node.getArguments()); if (node.getArguments().isEmpty() && "count".equalsIgnoreCase(node.getName().getSuffix())) { arguments = "*"; @@ -606,6 +612,25 @@ public static String formatSortItems(List sortItems) { return sortItems.stream().map(sortItemFormatterFunction()).collect(joining(", ")); } + public static String formatSkipTo(SkipTo skipTo) { + switch (skipTo.getPosition()) { + case PAST_LAST: + return "AFTER MATCH SKIP PAST LAST ROW"; + case NEXT: + return "AFTER MATCH SKIP TO NEXT ROW"; + case LAST: + checkState( + skipTo.getIdentifier().isPresent(), "missing identifier in AFTER MATCH SKIP TO LAST"); + return "AFTER MATCH SKIP TO LAST " + formatExpression(skipTo.getIdentifier().get()); + case FIRST: + checkState( + skipTo.getIdentifier().isPresent(), "missing identifier in AFTER MATCH SKIP TO FIRST"); + return "AFTER MATCH SKIP TO FIRST " + formatExpression(skipTo.getIdentifier().get()); + default: + throw new IllegalArgumentException("Invalid input: " + skipTo.getPosition()); + } + } + static String formatGroupBy(List groupingElements) { return groupingElements.stream() .map( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/RowPatternFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/RowPatternFormatter.java new file mode 100644 index 000000000000..e9eb828b2941 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/RowPatternFormatter.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.sql.util; + +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AnchorPattern; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.EmptyPattern; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExcludedPattern; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OneOrMoreQuantifier; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternAlternation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternConcatenation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternPermutation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternVariable; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedPattern; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RangeQuantifier; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowPattern; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ZeroOrMoreQuantifier; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ZeroOrOneQuantifier; + +import static java.util.stream.Collectors.joining; + +public final class RowPatternFormatter { + private RowPatternFormatter() {} + + public static String formatPattern(RowPattern pattern) { + return new Formatter().process(pattern, null); + } + + public static class Formatter extends AstVisitor { + @Override + protected String visitNode(Node node, Void context) { + throw new UnsupportedOperationException(); + } + + @Override + protected String visitRowPattern(RowPattern node, Void context) { + throw new UnsupportedOperationException( + String.format( + "not yet implemented: %s.visit%s", + getClass().getName(), node.getClass().getSimpleName())); + } + + @Override + protected String visitPatternAlternation(PatternAlternation node, Void context) { + return node.getPatterns().stream() + .map(child -> process(child, context)) + .collect(joining(" | ", "(", ")")); + } + + @Override + protected String visitPatternConcatenation(PatternConcatenation node, Void context) { + return node.getPatterns().stream() + .map(child -> process(child, context)) + .collect(joining(" ", "(", ")")); + } + + @Override + protected String visitQuantifiedPattern(QuantifiedPattern node, Void context) { + return "(" + + process(node.getPattern(), context) + + process(node.getPatternQuantifier(), context) + + ")"; + } + + @Override + protected String visitPatternVariable(PatternVariable node, Void context) { + return ExpressionFormatter.formatExpression(node.getName()); + } + + @Override + protected String visitEmptyPattern(EmptyPattern node, Void context) { + return "()"; + } + + @Override + protected String visitPatternPermutation(PatternPermutation node, Void context) { + return node.getPatterns().stream() + .map(child -> process(child, context)) + .collect(joining(", ", "PERMUTE(", ")")); + } + + @Override + protected String visitAnchorPattern(AnchorPattern node, Void context) { + switch (node.getType()) { + case PARTITION_START: + return "^"; + case PARTITION_END: + return "$"; + default: + throw new IllegalArgumentException("Invalid input: " + node.getType()); + } + } + + @Override + protected String visitExcludedPattern(ExcludedPattern node, Void context) { + return "{-" + process(node.getPattern(), context) + "-}"; + } + + @Override + protected String visitZeroOrMoreQuantifier(ZeroOrMoreQuantifier node, Void context) { + String greedy = node.isGreedy() ? "" : "?"; + return "*" + greedy; + } + + @Override + protected String visitOneOrMoreQuantifier(OneOrMoreQuantifier node, Void context) { + String greedy = node.isGreedy() ? "" : "?"; + return "+" + greedy; + } + + @Override + protected String visitZeroOrOneQuantifier(ZeroOrOneQuantifier node, Void context) { + String greedy = node.isGreedy() ? "" : "?"; + return "?" + greedy; + } + + @Override + protected String visitRangeQuantifier(RangeQuantifier node, Void context) { + String greedy = node.isGreedy() ? "" : "?"; + String atLeast = node.getAtLeast().map(ExpressionFormatter::formatExpression).orElse(""); + String atMost = node.getAtMost().map(ExpressionFormatter::formatExpression).orElse(""); + return "{" + atLeast + "," + atMost + "}" + greedy; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java index 98c5a048809a..f092aa288548 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/SqlFormatter.java @@ -58,6 +58,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Node; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Offset; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OrderBy; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternRecognitionRelation; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Property; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QualifiedName; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Query; @@ -66,6 +67,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameColumn; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RenameTable; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Row; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowPattern; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Select; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SelectItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SetProperties; @@ -105,12 +107,14 @@ import java.util.Optional; import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.getOnlyElement; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; import static org.apache.iotdb.db.queryengine.plan.relational.sql.util.ExpressionFormatter.formatGroupBy; import static org.apache.iotdb.db.queryengine.plan.relational.sql.util.ExpressionFormatter.formatOrderBy; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.util.RowPatternFormatter.formatPattern; public final class SqlFormatter { @@ -175,6 +179,13 @@ protected Void visitExpression(Expression node, Integer indent) { return null; } + @Override + protected Void visitRowPattern(RowPattern node, Integer indent) { + checkArgument(indent == 0, "visitRowPattern should only be called at root"); + builder.append(formatPattern(node)); + return null; + } + @Override protected Void visitQuery(Query node, Integer indent) { @@ -404,9 +415,7 @@ protected Void visitJoin(Join node, Integer indent) { @Override protected Void visitAliasedRelation(AliasedRelation node, Integer indent) { - builder.append("( "); - process(node, indent + 1); - append(indent, ")"); + processRelationSuffix(node.getRelation(), indent); builder.append(' ').append(formatName(node.getAlias())); appendAliasColumns(builder, node.getColumnNames()); @@ -414,6 +423,130 @@ protected Void visitAliasedRelation(AliasedRelation node, Integer indent) { return null; } + @Override + protected Void visitPatternRecognitionRelation( + PatternRecognitionRelation node, Integer indent) { + processRelationSuffix(node.getInput(), indent); + + builder.append(" MATCH_RECOGNIZE (\n"); + if (!node.getPartitionBy().isEmpty()) { + append(indent + 1, "PARTITION BY ") + .append( + node.getPartitionBy().stream() + .map(ExpressionFormatter::formatExpression) + .collect(joining(", "))) + .append("\n"); + } + if (node.getOrderBy().isPresent()) { + process(node.getOrderBy().get(), indent + 1); + } + if (!node.getMeasures().isEmpty()) { + append(indent + 1, "MEASURES"); + formatDefinitionList( + node.getMeasures().stream() + .map( + measure -> + formatExpression(measure.getExpression()) + + " AS " + + formatExpression(measure.getName())) + .collect(toImmutableList()), + indent + 2); + } + if (node.getRowsPerMatch().isPresent()) { + String rowsPerMatch; + switch (node.getRowsPerMatch().get()) { + case ONE: + rowsPerMatch = "ONE ROW PER MATCH"; + break; + case ALL_SHOW_EMPTY: + rowsPerMatch = "ALL ROWS PER MATCH SHOW EMPTY MATCHES"; + break; + case ALL_OMIT_EMPTY: + rowsPerMatch = "ALL ROWS PER MATCH OMIT EMPTY MATCHES"; + break; + case ALL_WITH_UNMATCHED: + rowsPerMatch = "ALL ROWS PER MATCH WITH UNMATCHED ROWS"; + break; + default: + // RowsPerMatch of type WINDOW cannot occur in MATCH_RECOGNIZE clause + throw new IllegalStateException( + "unexpected rowsPerMatch: " + node.getRowsPerMatch().get()); + } + append(indent + 1, rowsPerMatch).append("\n"); + } + if (node.getAfterMatchSkipTo().isPresent()) { + String skipTo; + switch (node.getAfterMatchSkipTo().get().getPosition()) { + case PAST_LAST: + skipTo = "AFTER MATCH SKIP PAST LAST ROW"; + break; + case NEXT: + skipTo = "AFTER MATCH SKIP TO NEXT ROW"; + break; + case LAST: + checkState( + node.getAfterMatchSkipTo().get().getIdentifier().isPresent(), + "missing identifier in AFTER MATCH SKIP TO LAST"); + skipTo = + "AFTER MATCH SKIP TO LAST " + + formatExpression(node.getAfterMatchSkipTo().get().getIdentifier().get()); + break; + case FIRST: + checkState( + node.getAfterMatchSkipTo().get().getIdentifier().isPresent(), + "missing identifier in AFTER MATCH SKIP TO FIRST"); + skipTo = + "AFTER MATCH SKIP TO FIRST " + + formatExpression(node.getAfterMatchSkipTo().get().getIdentifier().get()); + break; + default: + throw new IllegalStateException( + "unexpected skipTo: " + node.getAfterMatchSkipTo().get()); + } + append(indent + 1, skipTo).append("\n"); + } + append(indent + 1, "PATTERN (").append(formatPattern(node.getPattern())).append(")\n"); + if (!node.getSubsets().isEmpty()) { + append(indent + 1, "SUBSET"); + formatDefinitionList( + node.getSubsets().stream() + .map( + subset -> + formatExpression(subset.getName()) + + " = " + + subset.getIdentifiers().stream() + .map(ExpressionFormatter::formatExpression) + .collect(joining(", ", "(", ")"))) + .collect(toImmutableList()), + indent + 2); + } + append(indent + 1, "DEFINE"); + formatDefinitionList( + node.getVariableDefinitions().stream() + .map( + variable -> + formatExpression(variable.getName()) + + " AS " + + formatExpression(variable.getExpression())) + .collect(toImmutableList()), + indent + 2); + + builder.append(")"); + + return null; + } + + private void processRelationSuffix(Relation relation, Integer indent) { + if ((relation instanceof AliasedRelation) + || (relation instanceof PatternRecognitionRelation)) { + builder.append("( "); + process(relation, indent + 1); + append(indent, ")"); + } else { + process(relation, indent); + } + } + @Override protected Void visitValues(Values node, Integer indent) { builder.append(" VALUES "); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/RowPatternRecognitionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/RowPatternRecognitionTest.java new file mode 100644 index 000000000000..25933443a5e4 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/RowPatternRecognitionTest.java @@ -0,0 +1,458 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.analyzer; + +import org.apache.iotdb.db.protocol.session.IClientSession; +import org.apache.iotdb.db.queryengine.common.MPPQueryContext; +import org.apache.iotdb.db.queryengine.common.SessionInfo; +import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; +import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; + +import org.junit.Assert; +import org.junit.Test; + +import java.time.ZoneId; +import java.util.Collections; + +import static org.apache.iotdb.db.queryengine.execution.warnings.WarningCollector.NOOP; +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.TestUtils.QUERY_CONTEXT; +import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.TestUtils.TEST_MATADATA; +import static org.junit.Assert.fail; + +public class RowPatternRecognitionTest { + private static final NopAccessControl nopAccessControl = new NopAccessControl(); + + // table1's columns: time, tag1, tag2, tag3, attr1, attr2, s1, s2, s3 + + @Test + public void testInputColumns() { + assertTestSuccess( + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " PATTERN (A B+) " + + " DEFINE " + + " B AS B.s2 > 5 " + + ") AS m"); + + assertTestFail( + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " PATTERN (A B+) " + + " DEFINE " + + " B AS B.s4 > 5 " + + ") AS m", + "Column s4 prefixed with label B cannot be resolved"); + + assertTestFail( + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " PARTITION BY s4 " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " PATTERN (A B+) " + + " DEFINE " + + " B AS B.s2 > 5 " + + ") AS m", + "Column s4 is not present in the input relation"); + } + + @Test + public void testSubsetClause() { + assertTestFail( + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " PATTERN (A B+) " + + " SUBSET A = (B) " + + " DEFINE " + + " B AS B.s2 > 5 " + + ") AS m", + "union pattern variable name: A is a duplicate of primary pattern variable name"); + + assertTestFail( + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " PATTERN (A B+) " + + " SUBSET U = (A), " + + " U = (B) " + + " DEFINE " + + " B AS B.s2 > 5 " + + ") AS m", + "union pattern variable name: U is declared twice"); + + assertTestFail( + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " PATTERN (A B+) " + + " SUBSET U = (A, C) " + + " DEFINE " + + " B AS B.s2 > 5 " + + ") AS m", + "subset element: C is not a primary pattern variable"); + } + + @Test + public void testDefineClause() { + assertTestFail( + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " PATTERN (A B+) " + + " DEFINE " + + " B AS B.s2 > 5, " + + " C AS true " + + ") AS m", + "defined variable: C is not a primary pattern variable"); + + assertTestFail( + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " PATTERN (A B+) " + + " DEFINE " + + " B AS B.s2 > 5, " + + " B AS true " + + ") AS m", + "pattern variable with name: B is defined twice"); + + assertTestFail( + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " PATTERN (A B+) " + + " DEFINE " + + " B AS B.s2 " + + ") AS m", + "Expression defining a label must be boolean (actual type: INT64)"); + + // FINAL semantics is not supported in DEFINE clause. RUNNING semantics is supported + assertTestFail( + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " PATTERN (A B+) " + + " DEFINE " + + " B AS FINAL RPR_LAST(B.s2) > 5" + + ") AS m", + "FINAL semantics is not supported in DEFINE clause"); + + assertTestSuccess( + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " PATTERN (A B+) " + + " DEFINE " + + " B AS RUNNING RPR_LAST(B.s2) > 5" + + ") AS m"); + } + + @Test + public void testPatternExclusions() { + String sql = + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " %s " + + " PATTERN ({- A -} B+) " + + " DEFINE " + + " B AS B.s2 > 5" + + ") AS m"; + + assertTestSuccess(String.format(sql, "")); + assertTestSuccess(String.format(sql, "ONE ROW PER MATCH")); + assertTestSuccess(String.format(sql, "ALL ROWS PER MATCH")); + assertTestSuccess(String.format(sql, "ALL ROWS PER MATCH SHOW EMPTY MATCHES")); + assertTestSuccess(String.format(sql, "ALL ROWS PER MATCH OMIT EMPTY MATCHES")); + + assertTestFail( + String.format(sql, "ALL ROWS PER MATCH WITH UNMATCHED ROWS"), + "Pattern exclusion syntax is not allowed when ALL ROWS PER MATCH WITH UNMATCHED ROWS is specified"); + } + + @Test + public void testPatternQuantifiers() { + String sql = + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " PATTERN (A B%s) " + + " DEFINE " + + " B AS B.s2 > 5" + + ") AS m"; + + assertTestSuccess(String.format(sql, "*")); + assertTestSuccess(String.format(sql, "*?")); + assertTestSuccess(String.format(sql, "+")); + assertTestSuccess(String.format(sql, "+?")); + assertTestSuccess(String.format(sql, "?")); + assertTestSuccess(String.format(sql, "??")); + assertTestSuccess(String.format(sql, "{,}")); + assertTestSuccess(String.format(sql, "{5}")); + assertTestSuccess(String.format(sql, "{5,}")); + assertTestSuccess(String.format(sql, "{0,}")); + + assertTestFail( + String.format(sql, "{0}"), + "Pattern quantifier upper bound must be greater than or equal to 1"); + assertTestFail( + String.format(sql, "{,0}"), + "Pattern quantifier upper bound must be greater than or equal to 1"); + assertTestFail( + String.format(sql, "{0,0}"), + "Pattern quantifier upper bound must be greater than or equal to 1"); + assertTestFail( + String.format(sql, "{3000000000}"), + "Pattern quantifier lower bound must not exceed 2147483647"); + assertTestFail( + String.format(sql, "{5,1}"), "Pattern quantifier lower bound must not exceed upper bound"); + } + + @Test + public void testAfterMatchSkipClause() { + String sql = + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " %s " + + " PATTERN (A B+) " + + " DEFINE " + + " B AS B.s2 > 5" + + ") AS m"; + + assertTestSuccess(String.format(sql, "")); + assertTestSuccess(String.format(sql, "AFTER MATCH SKIP PAST LAST ROW")); + assertTestSuccess(String.format(sql, "AFTER MATCH SKIP TO NEXT ROW")); + assertTestSuccess(String.format(sql, "AFTER MATCH SKIP TO FIRST B")); + assertTestSuccess(String.format(sql, "AFTER MATCH SKIP TO LAST B")); + assertTestSuccess(String.format(sql, "AFTER MATCH SKIP TO B")); + + assertTestFail( + String.format(sql, "AFTER MATCH SKIP TO C"), + "C is not a primary or union pattern variable"); + } + + @Test + public void testRunningAndFinalSemantics() { + String sql = + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " %s AS col1 " + + " PATTERN (A B+) " + + " DEFINE " + + " B AS B.s2 > 5 " + + ") AS m"; + + assertTestSuccess(String.format(sql, "FINAL RPR_LAST(A.s1)")); + assertTestSuccess(String.format(sql, "FINAL RPR_FIRST(A.s1)")); + + assertTestFail( + String.format(sql, "FINAL PREV(A.s1)"), + "FINAL semantics is not supported with prev pattern recognition function"); + assertTestFail( + String.format(sql, "FINAL NEXT(A.s1)"), + "FINAL semantics is not supported with next pattern recognition function"); + assertTestFail( + String.format(sql, "FINAL CLASSIFIER(A.s1)"), + "FINAL semantics is not supported with classifier pattern recognition function"); + assertTestFail( + String.format(sql, "FINAL MATCH_NUMBER(A.s1)"), + "FINAL semantics is not supported with match_number pattern recognition function"); + } + + @Test + public void testPatternNavigationFunctions() { + String sql = + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " %s AS col1 " + + " PATTERN (A B+) " + + " DEFINE " + + " B AS B.s2 > 5 " + + ") AS m"; + + assertTestSuccess(String.format(sql, "PREV(RPR_LAST(A.s1, 2), 3)")); + + assertTestFail( + String.format(sql, "PREV()"), + "prev pattern recognition function requires 1 or 2 arguments"); + assertTestFail( + String.format(sql, "PREV(A.s1, 'str')"), + "prev pattern recognition navigation function requires a number as the second argument"); + assertTestFail( + String.format(sql, "PREV(A.s1, -5)"), + "prev pattern recognition navigation function requires a non-negative number as the second argument (actual: -5)"); + assertTestFail( + String.format(sql, "PREV(A.s1, 3000000000)"), + "The second argument of prev pattern recognition navigation function must not exceed 2147483647 (actual: 3000000000)"); + assertTestFail( + String.format(sql, "RPR_LAST(NEXT(A.s1, 2))"), + "Cannot nest next pattern navigation function inside rpr_last pattern navigation function"); + assertTestFail( + String.format(sql, "PREV(NEXT(A.s1, 2))"), + "Cannot nest next pattern navigation function inside prev pattern navigation function"); + } + + @Test + public void testClassifierFunction() { + String sql = + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " %s AS col1 " + + " PATTERN (A B+) " + + " SUBSET U = (A, B) " + + " DEFINE " + + " B AS B.s2 > 5 " + + ") AS m"; + + assertTestSuccess(String.format(sql, "CLASSIFIER()")); + assertTestSuccess(String.format(sql, "CLASSIFIER(A)")); + assertTestSuccess(String.format(sql, "CLASSIFIER(U)")); + + assertTestFail( + String.format(sql, "CLASSIFIER(A, B)"), + "CLASSIFIER pattern recognition function takes no arguments or 1 argument"); + assertTestFail( + String.format(sql, "CLASSIFIER(A.s1)"), + "CLASSIFIER function argument should be primary pattern variable or subset name. Actual: DereferenceExpression"); + assertTestFail( + String.format(sql, "CLASSIFIER(C)"), "C is not a primary pattern variable or subset name"); + } + + @Test + public void testMatchNumberFunction() { + String sql = + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " %s AS col1 " + + " PATTERN (A B+) " + + " SUBSET U = (A, B) " + + " DEFINE " + + " B AS B.s2 > 5 " + + ") AS m"; + + assertTestSuccess(String.format(sql, "MATCH_NUMBER()")); + + assertTestFail( + String.format(sql, "MATCH_NUMBER(A)"), + "MATCH_NUMBER pattern recognition function takes no arguments"); + } + + private void assertTestFail(String sql, String errMsg) { + try { + analyzeSQL(sql, TEST_MATADATA, QUERY_CONTEXT); + fail("No exception!"); + } catch (Exception e) { + Assert.assertTrue(e.getMessage(), e.getMessage().contains(errMsg)); + } + } + + private void assertTestSuccess(String sql) { + try { + analyzeSQL(sql, TEST_MATADATA, QUERY_CONTEXT); + } catch (Exception e) { + fail("Unexpected exception: " + e.getMessage()); + } + } + + public static void analyzeSQL(String sql, Metadata metadata, final MPPQueryContext context) { + SqlParser sqlParser = new SqlParser(); + Statement statement = sqlParser.createStatement(sql, ZoneId.systemDefault()); + SessionInfo session = + new SessionInfo( + 0, "test", ZoneId.systemDefault(), "testdb", IClientSession.SqlDialect.TABLE); + analyzeStatement(statement, metadata, context, sqlParser, session); + } + + public static void analyzeStatement( + final Statement statement, + final Metadata metadata, + final MPPQueryContext context, + final SqlParser sqlParser, + final SessionInfo session) { + final StatementAnalyzerFactory statementAnalyzerFactory = + new StatementAnalyzerFactory(metadata, sqlParser, nopAccessControl); + + Analyzer analyzer = + new Analyzer( + context, + session, + statementAnalyzerFactory, + Collections.emptyList(), + Collections.emptyMap(), + NOOP); + analyzer.analyze(statement); + } + + private static class NopAccessControl implements AccessControl {} +} diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index 06a21e9cd4d6..bb09d5129dbf 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -38,6 +38,10 @@ standaloneType : type EOF ; +standaloneRowPattern + : rowPattern EOF + ; + statement // Query Statement : queryStatement @@ -700,6 +704,7 @@ relation | NATURAL joinType JOIN right=aliasedRelation ) #joinRelation | aliasedRelation #relationDefault + | patternRecognition #patternRecognitionRelation ; joinType @@ -714,6 +719,54 @@ joinCriteria | USING '(' identifier (',' identifier)* ')' ; +patternRecognition + : aliasedRelation ( + MATCH_RECOGNIZE '(' + (PARTITION BY partition+=expression (',' partition+=expression)*)? + (ORDER BY sortItem (',' sortItem)*)? + (MEASURES measureDefinition (',' measureDefinition)*)? + rowsPerMatch? + (AFTER MATCH skipTo)? + (INITIAL | SEEK)? + PATTERN '(' rowPattern ')' + (SUBSET subsetDefinition (',' subsetDefinition)*)? + DEFINE variableDefinition (',' variableDefinition)* + ')' + (AS? identifier columnAliases?)? + )? + ; + +measureDefinition + : expression AS identifier + ; + +rowsPerMatch + : ONE ROW PER MATCH + | ALL ROWS PER MATCH emptyMatchHandling? + ; + +emptyMatchHandling + : SHOW EMPTY MATCHES + | OMIT EMPTY MATCHES + | WITH UNMATCHED ROWS + ; + +skipTo + : 'SKIP' TO NEXT ROW + | 'SKIP' PAST LAST ROW + | 'SKIP' TO FIRST identifier + | 'SKIP' TO LAST identifier + | 'SKIP' TO identifier + ; + +subsetDefinition + : name=identifier EQ '(' union+=identifier (',' union+=identifier)* ')' + ; + +variableDefinition + : identifier AS expression + ; + aliasedRelation : relationPrimary (AS? identifier columnAliases?)? ; @@ -765,7 +818,7 @@ primaryExpression | '(' expression (',' expression)+ ')' #rowConstructor | ROW '(' expression (',' expression)* ')' #rowConstructor | qualifiedName '(' (label=identifier '.')? ASTERISK ')' #functionCall - | qualifiedName '(' (setQuantifier? expression (',' expression)*)?')' #functionCall + | processingMode? qualifiedName '(' (setQuantifier? expression (',' expression)*)?')' #functionCall | '(' query ')' #subqueryExpression // This is an extension to ANSI SQL, which considers EXISTS to be a | EXISTS '(' query ')' #exists @@ -797,6 +850,11 @@ literalExpression | QUESTION_MARK #parameter ; +processingMode + : RUNNING + | FINAL + ; + trimsSpecification : LEADING | TRAILING @@ -849,6 +907,31 @@ whenClause : WHEN condition=expression THEN result=expression ; +rowPattern + : patternPrimary patternQuantifier? #quantifiedPrimary + | rowPattern rowPattern #patternConcatenation + | rowPattern '|' rowPattern #patternAlternation + ; + +patternPrimary + : identifier #patternVariable + | '(' ')' #emptyPattern + | PERMUTE '(' rowPattern (',' rowPattern)* ')' #patternPermutation + | '(' rowPattern ')' #groupedPattern + | '^' #partitionStartAnchor + | '$' #partitionEndAnchor + | '{-' rowPattern '-}' #excludedPattern + ; + +patternQuantifier + : ASTERISK (reluctant=QUESTION_MARK)? #zeroOrMoreQuantifier + | PLUS (reluctant=QUESTION_MARK)? #oneOrMoreQuantifier + | QUESTION_MARK (reluctant=QUESTION_MARK)? #zeroOrOneQuantifier + | '{' exactly=INTEGER_VALUE '}' (reluctant=QUESTION_MARK)? #rangeQuantifier + | '{' (atLeast=INTEGER_VALUE)? ',' (atMost=INTEGER_VALUE)? '}' (reluctant=QUESTION_MARK)? #rangeQuantifier + ; + + updateAssignment : identifier EQ expression ; @@ -948,7 +1031,7 @@ nonReserved | OBJECT | OF | OFFSET | OMIT | ONE | ONLY | OPTION | ORDINALITY | OUTPUT | OVER | OVERFLOW | PARTITION | PARTITIONS | PASSING | PAST | PATH | PATTERN | PER | PERIOD | PERMUTE | PIPE | PIPEPLUGIN | PIPEPLUGINS | PIPES | PLAN | POSITION | PRECEDING | PRECISION | PRIVILEGES | PREVIOUS | PROCESSLIST | PROCESSOR | PROPERTIES | PRUNE | QUERIES | QUERY | QUOTES - | RANGE | READ | READONLY | REFRESH | REGION | REGIONID | REGIONS | RENAME | REPAIR | REPEAT | REPEATABLE | REPLACE | RESET | RESPECT | RESTRICT | RETURN | RETURNING | RETURNS | REVOKE | ROLE | ROLES | ROLLBACK | ROW | ROWS | RUNNING + | RANGE | READ | READONLY | REFRESH | REGION | REGIONID | REGIONS | RENAME | REPAIR | REPEAT | REPEATABLE | REPLACE | RESET | RESPECT | RESTRICT | RETURN | RETURNING | RETURNS | REVOKE | ROLE | ROLES | ROLLBACK | ROW | ROWS | RPR_FIRST | RPR_LAST | RUNNING | SERIESSLOTID | SCALAR | SCHEMA | SCHEMAS | SECOND | SECURITY | SEEK | SERIALIZABLE | SESSION | SET | SETS | SHOW | SINK | SOME | SOURCE | START | STATS | STOP | SUBSCRIPTIONS | SUBSET | SUBSTRING | SYSTEM | TABLES | TABLESAMPLE | TEXT | TEXT_STRING | TIES | TIME | TIMEPARTITION | TIMESERIES | TIMESLOTID | TIMESTAMP | TO | TOPIC | TOPICS | TRAILING | TRANSACTION | TRUNCATE | TRY_CAST | TYPE @@ -1239,6 +1322,8 @@ ROLLBACK: 'ROLLBACK'; ROLLUP: 'ROLLUP'; ROW: 'ROW'; ROWS: 'ROWS'; +RPR_FIRST: 'RPR_FIRST'; +RPR_LAST: 'RPR_LAST'; RUNNING: 'RUNNING'; SERIESSLOTID: 'SERIESSLOTID'; SCALAR: 'SCALAR'; diff --git a/mvn b/mvn new file mode 100644 index 000000000000..e69de29bb2d1 From 55499b996ee1872c2c6c10593cb40d66402741e0 Mon Sep 17 00:00:00 2001 From: Young-Leo <122667095+Young-Leo@users.noreply.github.com> Date: Tue, 3 Dec 2024 22:59:56 +0800 Subject: [PATCH 2/4] resolve conflicts with upstream --- .../plan/relational/analyzer/RowPatternRecognitionTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/RowPatternRecognitionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/RowPatternRecognitionTest.java index 25933443a5e4..b5539b443854 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/RowPatternRecognitionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/RowPatternRecognitionTest.java @@ -19,6 +19,7 @@ import org.apache.iotdb.db.queryengine.common.SessionInfo; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl; +import org.apache.iotdb.db.queryengine.plan.relational.security.AllowAllAccessControl; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; @@ -34,7 +35,7 @@ import static org.junit.Assert.fail; public class RowPatternRecognitionTest { - private static final NopAccessControl nopAccessControl = new NopAccessControl(); + private static final AccessControl nopAccessControl = new AllowAllAccessControl(); // table1's columns: time, tag1, tag2, tag3, attr1, attr2, s1, s2, s3 @@ -453,6 +454,4 @@ public static void analyzeStatement( NOOP); analyzer.analyze(statement); } - - private static class NopAccessControl implements AccessControl {} } From 425eba409065f34f26a862cd8df03232a081339f Mon Sep 17 00:00:00 2001 From: Young-Leo <122667095+Young-Leo@users.noreply.github.com> Date: Wed, 4 Dec 2024 14:05:48 +0800 Subject: [PATCH 3/4] update --- .../sql/util/ExpressionFormatter.java | 21 ------------------- .../relational/grammar/sql/RelationalSql.g4 | 2 +- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java index 0f25f993267e..c6c007a98818 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/util/ExpressionFormatter.java @@ -67,7 +67,6 @@ import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SearchedCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleCaseExpression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SimpleGroupBy; -import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SortItem; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.StringLiteral; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SubqueryExpression; @@ -86,7 +85,6 @@ import java.util.Optional; import java.util.function.Function; -import static com.google.common.base.Preconditions.checkState; import static com.google.common.collect.Iterables.getOnlyElement; import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.joining; @@ -612,25 +610,6 @@ public static String formatSortItems(List sortItems) { return sortItems.stream().map(sortItemFormatterFunction()).collect(joining(", ")); } - public static String formatSkipTo(SkipTo skipTo) { - switch (skipTo.getPosition()) { - case PAST_LAST: - return "AFTER MATCH SKIP PAST LAST ROW"; - case NEXT: - return "AFTER MATCH SKIP TO NEXT ROW"; - case LAST: - checkState( - skipTo.getIdentifier().isPresent(), "missing identifier in AFTER MATCH SKIP TO LAST"); - return "AFTER MATCH SKIP TO LAST " + formatExpression(skipTo.getIdentifier().get()); - case FIRST: - checkState( - skipTo.getIdentifier().isPresent(), "missing identifier in AFTER MATCH SKIP TO FIRST"); - return "AFTER MATCH SKIP TO FIRST " + formatExpression(skipTo.getIdentifier().get()); - default: - throw new IllegalArgumentException("Invalid input: " + skipTo.getPosition()); - } - } - static String formatGroupBy(List groupingElements) { return groupingElements.stream() .map( diff --git a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 index bb09d5129dbf..1ff889128cf6 100644 --- a/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 +++ b/iotdb-core/relational-grammar/src/main/antlr4/org/apache/iotdb/db/relational/grammar/sql/RelationalSql.g4 @@ -818,7 +818,7 @@ primaryExpression | '(' expression (',' expression)+ ')' #rowConstructor | ROW '(' expression (',' expression)* ')' #rowConstructor | qualifiedName '(' (label=identifier '.')? ASTERISK ')' #functionCall - | processingMode? qualifiedName '(' (setQuantifier? expression (',' expression)*)?')' #functionCall + | processingMode? qualifiedName '(' (setQuantifier? expression (',' expression)*)?')' #functionCall | '(' query ')' #subqueryExpression // This is an extension to ANSI SQL, which considers EXISTS to be a | EXISTS '(' query ')' #exists From 9906fa102a9eab170f552bbb42a990983ac221de Mon Sep 17 00:00:00 2001 From: Young-Leo <122667095+Young-Leo@users.noreply.github.com> Date: Fri, 13 Dec 2024 16:52:41 +0800 Subject: [PATCH 4/4] introduce plan and matcher of row pattern recognition --- .../process/rowpattern/LabelEvaluator.java | 196 ++++++++++ .../rowpattern/LogicalIndexNavigation.java | 187 +++++++++ .../rowpattern/PhysicalValueAccessor.java | 22 ++ .../rowpattern/PhysicalValuePointer.java | 53 +++ .../process/rowpattern/matcher/ArrayView.java | 55 +++ .../process/rowpattern/matcher/Done.java | 45 +++ .../rowpattern/matcher/Instruction.java | 62 +++ .../process/rowpattern/matcher/IntList.java | 82 ++++ .../rowpattern/matcher/IntMultimap.java | 106 ++++++ .../process/rowpattern/matcher/IntStack.java | 60 +++ .../IrRowPatternToProgramRewriter.java | 264 +++++++++++++ .../process/rowpattern/matcher/Jump.java | 60 +++ .../process/rowpattern/matcher/MatchEnd.java | 45 +++ .../rowpattern/matcher/MatchLabel.java | 60 +++ .../rowpattern/matcher/MatchResult.java | 49 +++ .../rowpattern/matcher/MatchStart.java | 45 +++ .../process/rowpattern/matcher/Matcher.java | 277 ++++++++++++++ .../rowpattern/matcher/PatternCaptures.java | 66 ++++ .../process/rowpattern/matcher/Program.java | 83 ++++ .../process/rowpattern/matcher/Save.java | 45 +++ .../process/rowpattern/matcher/Split.java | 69 ++++ .../plan/planner/plan/node/PlanNodeType.java | 1 + .../plan/planner/plan/node/PlanVisitor.java | 5 + .../plan/relational/analyzer/Analysis.java | 14 +- .../analyzer/ExpressionAnalyzer.java | 22 +- .../analyzer/PatternRecognitionAnalysis.java | 8 +- .../plan/relational/planner/QueryPlanner.java | 13 + .../relational/planner/RelationPlanner.java | 359 ++++++++++++++++++ .../plan/relational/planner/node/Measure.java | 95 +++++ .../planner/node/PatternRecognitionNode.java | 312 +++++++++++++++ .../relational/planner/node/Patterns.java | 9 +- .../relational/planner/node/RowsPerMatch.java | 127 +++++++ .../planner/node/SkipToPosition.java | 47 +++ .../planner/optimizations/SymbolMapper.java | 69 ++++ .../UnaliasSymbolReferences.java | 14 + .../rowpattern/ClassifierValuePointer.java | 70 ++++ .../ExpressionAndValuePointers.java | 240 ++++++++++++ .../planner/rowpattern/IrAlternation.java | 99 +++++ .../planner/rowpattern/IrAnchor.java | 87 +++++ .../planner/rowpattern/IrConcatenation.java | 100 +++++ .../planner/rowpattern/IrEmpty.java | 62 +++ .../planner/rowpattern/IrExclusion.java | 79 ++++ .../planner/rowpattern/IrLabel.java | 85 +++++ .../IrPatternAlternationOptimizer.java | 141 +++++++ .../planner/rowpattern/IrPermutation.java | 96 +++++ .../planner/rowpattern/IrQuantified.java | 88 +++++ .../planner/rowpattern/IrQuantifier.java | 126 ++++++ .../planner/rowpattern/IrRowPattern.java | 117 ++++++ .../rowpattern/IrRowPatternFlattener.java | 206 ++++++++++ .../rowpattern/IrRowPatternVisitor.java | 68 ++++ .../rowpattern/LogicalIndexPointer.java | 167 ++++++++ .../rowpattern/MatchNumberValuePointer.java | 45 +++ .../planner/rowpattern/Patterns.java | 83 ++++ .../rowpattern/RowPatternToIrRewriter.java | 148 ++++++++ .../rowpattern/ScalarValuePointer.java | 82 ++++ .../planner/rowpattern/ValuePointer.java | 17 + .../plan/relational/sql/ast/FunctionCall.java | 24 +- .../relational/sql/parser/AstBuilder.java | 1 - .../process/rowpattern/MatcherTest.java | 198 ++++++++++ .../analyzer/RowPatternRecognitionTest.java | 4 +- .../planner/RowPatternRecognitionTest.java | 62 +++ .../planner/assertions/PlanMatchPattern.java | 14 +- 62 files changed, 5499 insertions(+), 36 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/LabelEvaluator.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/LogicalIndexNavigation.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/PhysicalValueAccessor.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/PhysicalValuePointer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/ArrayView.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Done.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Instruction.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IntList.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IntMultimap.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IntStack.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IrRowPatternToProgramRewriter.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Jump.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchEnd.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchLabel.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchResult.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchStart.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Matcher.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/PatternCaptures.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Program.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Save.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Split.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Measure.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/PatternRecognitionNode.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/RowsPerMatch.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SkipToPosition.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ClassifierValuePointer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ExpressionAndValuePointers.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrAlternation.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrAnchor.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrConcatenation.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrEmpty.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrExclusion.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrLabel.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrPatternAlternationOptimizer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrPermutation.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrQuantified.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrQuantifier.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrRowPattern.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrRowPatternFlattener.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrRowPatternVisitor.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/LogicalIndexPointer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/MatchNumberValuePointer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/Patterns.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/RowPatternToIrRewriter.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ScalarValuePointer.java create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ValuePointer.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/MatcherTest.java create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RowPatternRecognitionTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/LabelEvaluator.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/LabelEvaluator.java new file mode 100644 index 000000000000..2601944ce2a5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/LabelEvaluator.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern; + +import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher.ArrayView; + +import org.apache.tsfile.read.common.block.TsBlock; + +import java.util.List; + +import static java.util.Objects.requireNonNull; + +public class LabelEvaluator { + private final long matchNumber; + + private final int patternStart; + + // inclusive - the first row of the search partition + private final int partitionStart; + + // inclusive - the first row of the partition area available for pattern search. + // this area is the whole partition in case of MATCH_RECOGNIZE, and the area enclosed + // by the common base frame in case of pattern recognition in WINDOW clause. + private final int searchStart; + + // exclusive - the first row after the the partition area available for pattern search. + // this area is the whole partition in case of MATCH_RECOGNIZE, and the area enclosed + // by the common base frame in case of pattern recognition in WINDOW clause. + private final int searchEnd; + + private final List evaluations; + + // private final ProjectingPagesWindowIndex windowIndex; + + public LabelEvaluator( + long matchNumber, + int patternStart, + int partitionStart, + int searchStart, + int searchEnd, + List evaluations + // ProjectingPagesWindowIndex windowIndex + ) { + this.matchNumber = matchNumber; + this.patternStart = patternStart; + this.partitionStart = partitionStart; + this.searchStart = searchStart; + this.searchEnd = searchEnd; + this.evaluations = requireNonNull(evaluations, "evaluations is null"); + // this.windowIndex = requireNonNull(windowIndex, "windowIndex is null"); + } + + public int getInputLength() { + return searchEnd - patternStart; + } + + public boolean isMatchingAtPartitionStart() { + return patternStart == partitionStart; + } + + // evaluate the last label in matchedLabels. It has been tentatively appended to the match + public boolean evaluateLabel(ArrayView matchedLabels) { + int label = matchedLabels.get(matchedLabels.length() - 1); + Evaluation evaluation = evaluations.get(label); + return evaluation.test( + matchedLabels, + // aggregations, + partitionStart, + searchStart, + searchEnd, + patternStart, + matchNumber + // windowIndex + ); + } + + public static class Evaluation { + // compiled computation of label-defining boolean expression + // private final PageProjection projection; + + // value accessors ordered as expected by the compiled projection + private final List expectedLayout; + + // precomputed `Block`s with null values for every `PhysicalValuePointer` (see + // MeasureComputation) + private final TsBlock[] nulls; + + // mapping from int representation to label name + private final List labelNames; + + // private final ConnectorSession session; + + public Evaluation( + // PageProjection projection, + List expectedLayout, List labelNames + // ConnectorSession session + ) { + // this.projection = requireNonNull(projection, "projection is null"); + this.expectedLayout = requireNonNull(expectedLayout, "expectedLayout is null"); + this.nulls = precomputeNulls(expectedLayout); + this.labelNames = requireNonNull(labelNames, "labelNames is null"); + // this.session = requireNonNull(session, "session is null"); + } + + public static TsBlock[] precomputeNulls(List expectedLayout) { + TsBlock[] nulls = new TsBlock[expectedLayout.size()]; + for (int i = 0; i < expectedLayout.size(); i++) { + PhysicalValueAccessor accessor = expectedLayout.get(i); + if (accessor instanceof PhysicalValuePointer) { + // nulls[i] = nativeValueToBlock(((PhysicalValuePointer) accessor).getType(), + // null); + // TODO: + nulls[i] = new TsBlock(1); + } + } + return nulls; + } + + public List getExpectedLayout() { + return expectedLayout; + } + + public boolean test( + ArrayView matchedLabels, + // MatchAggregation[] aggregations, + int partitionStart, + int searchStart, + int searchEnd, + int patternStart, + long matchNumber + // ProjectingPagesWindowIndex windowIndex + ) { + int currentRow = patternStart + matchedLabels.length() - 1; + + // TODO: + // Block result = + // compute( + // currentRow, + // matchedLabels, + // aggregations, + // partitionStart, + // searchStart, + // searchEnd, + // patternStart, + // matchNumber, + // windowIndex, + // projection, + // expectedLayout, + // nulls, + // labelNames, + // session); + + // return BOOLEAN.getBoolean(result, 0); + return false; + } + } + + // public static class EvaluationSupplier { + // private final Supplier projection; + // private final List expectedLayout; + // private final List labelNames; + // private final ConnectorSession session; + // + // public EvaluationSupplier( + // Supplier projection, + // List expectedLayout, + // List labelNames, + // ConnectorSession session) { + // this.projection = requireNonNull(projection, "projection is null"); + // this.expectedLayout = requireNonNull(expectedLayout, "expectedLayout is null"); + // this.labelNames = requireNonNull(labelNames, "labelNames is null"); + // this.session = requireNonNull(session, "session is null"); + // } + // + // public Evaluation get() { + // return new Evaluation(projection.get(), expectedLayout, labelNames, session); + // } + // } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/LogicalIndexNavigation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/LogicalIndexNavigation.java new file mode 100644 index 000000000000..88d93dfec916 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/LogicalIndexNavigation.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern; + +import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher.ArrayView; + +import java.util.Collections; +import java.util.Objects; +import java.util.Set; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class LogicalIndexNavigation { + public static final LogicalIndexNavigation NO_OP = + new LogicalIndexNavigation(Collections.emptySet(), true, true, 0, 0); + + // a set of labels to navigate over: + // LAST(A.price, 3) => this is a navigation over rows with label A, so labels = {A} + // LAST(Union.price, 3) => this is a navigation over rows matching a union variable Union, so for + // SUBSET Union = (A, B, C), we have labels = {A, B, C} + // LAST(price, 3) => this is a navigation over "universal pattern variable", which is effectively + // over all rows, no matter the assigned labels. In such case labels = {} + private final Set labels; + private final boolean last; + private final boolean running; + private final int logicalOffset; + private final int physicalOffset; + + public LogicalIndexNavigation( + Set labels, boolean last, boolean running, int logicalOffset, int physicalOffset) { + this.labels = requireNonNull(labels, "labels is null"); + this.last = last; + this.running = running; + checkArgument(logicalOffset >= 0, "logical offset must be >= 0, actual: %s", logicalOffset); + this.logicalOffset = logicalOffset; + this.physicalOffset = physicalOffset; + } + + public Set getLabels() { + return labels; + } + + public boolean isLast() { + return last; + } + + public int getLogicalOffset() { + return logicalOffset; + } + + public int getPhysicalOffset() { + return physicalOffset; + } + + /** + * This method is used when evaluating labels during pattern matching, computing row pattern + * measures, and computing SKIP TO position after finding a match. Search is limited up to the + * current row in case of running semantics and to the entire match in case of final semantics. + * + * @return position within partition, or -1 if matching position was not found + */ + public int resolvePosition( + int currentRow, ArrayView matchedLabels, int searchStart, int searchEnd, int patternStart) { + checkArgument( + currentRow >= patternStart && currentRow < patternStart + matchedLabels.length(), + "current row is out of bounds of the match"); + + int relativePosition; + if (last) { + int start; + if (running) { + start = currentRow - patternStart; + } else { + start = matchedLabels.length() - 1; + } + relativePosition = findLastAndBackwards(start, matchedLabels); + } else { + relativePosition = findFirstAndForward(matchedLabels); + } + return adjustPosition(relativePosition, patternStart, searchStart, searchEnd); + } + + // LAST(A.price, 3): find the last occurrence of label "A" and go 3 occurrences backwards + private int findLastAndBackwards(int searchStart, ArrayView matchedLabels) { + int position = searchStart + 1; + int found = 0; + while (found <= logicalOffset && position > 0) { + position--; + if (labels.isEmpty() || labels.contains(matchedLabels.get(position))) { + found++; + } + } + if (found == logicalOffset + 1) { + return position; + } + return -1; + } + + // FIRST(A.price, 3): find the first occurrence of label "A" and go 3 occurrences forward + private int findFirstAndForward(ArrayView matchedLabels) { + int position = -1; + int found = 0; + while (found <= logicalOffset && position < matchedLabels.length() - 1) { + position++; + if (labels.isEmpty() || labels.contains(matchedLabels.get(position))) { + found++; + } + } + if (found == logicalOffset + 1) { + return position; + } + return -1; + } + + // adjust position by patternStart to reflect position within partition + // adjust position by physical offset: skip a certain number of rows, regardless of labels + // check if the new position is within partition bound by: partitionStart - inclusive, + // partitionEnd - exclusive + private int adjustPosition( + int relativePosition, int patternStart, int searchStart, int searchEnd) { + if (relativePosition == -1) { + return -1; + } + int start = relativePosition + patternStart; + int target = start + physicalOffset; + if (target < searchStart || target >= searchEnd) { + return -1; + } + return target; + } + + // for thread equivalence + public LogicalIndexNavigation withoutLogicalOffset() { + return withLogicalOffset(0); + } + + public LogicalIndexNavigation withLogicalOffset(int logicalOffset) { + return new LogicalIndexNavigation(labels, last, running, logicalOffset, physicalOffset); + } + + public LogicalIndexNavigation withoutPhysicalOffset() { + return withPhysicalOffset(0); + } + + public LogicalIndexNavigation withPhysicalOffset(int physicalOffset) { + return new LogicalIndexNavigation(labels, last, running, logicalOffset, physicalOffset); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + LogicalIndexNavigation that = (LogicalIndexNavigation) o; + return last == that.last + && running == that.running + && logicalOffset == that.logicalOffset + && physicalOffset == that.physicalOffset + && Objects.equals(labels, that.labels); + } + + @Override + public int hashCode() { + return Objects.hash(labels, last, running, logicalOffset, physicalOffset); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/PhysicalValueAccessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/PhysicalValueAccessor.java new file mode 100644 index 000000000000..81dd0ad095f6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/PhysicalValueAccessor.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern; + +public interface PhysicalValueAccessor {} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/PhysicalValuePointer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/PhysicalValuePointer.java new file mode 100644 index 000000000000..414705dbbf59 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/PhysicalValuePointer.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern; + +import org.apache.tsfile.read.common.type.Type; + +import static java.util.Objects.requireNonNull; + +public class PhysicalValuePointer implements PhysicalValueAccessor { + public static final int CLASSIFIER = -1; + public static final int MATCH_NUMBER = -2; + + private final int sourceChannel; + private final Type type; + private final LogicalIndexNavigation logicalIndexNavigation; + + public PhysicalValuePointer( + int sourceChannel, Type type, LogicalIndexNavigation logicalIndexNavigation) { + this.sourceChannel = sourceChannel; + this.type = requireNonNull(type, "type is null"); + this.logicalIndexNavigation = + requireNonNull(logicalIndexNavigation, "logicalIndexNavigation is null"); + } + + public int getSourceChannel() { + return sourceChannel; + } + + public Type getType() { + return type; + } + + public LogicalIndexNavigation getLogicalIndexNavigation() { + return logicalIndexNavigation; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/ArrayView.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/ArrayView.java new file mode 100644 index 000000000000..63197a26ccf6 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/ArrayView.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +import com.google.common.annotations.VisibleForTesting; + +import java.util.Arrays; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; + +public class ArrayView { + public static final ArrayView EMPTY = new ArrayView(new int[] {}, 0); + + private final int[] array; + private final int length; + + public ArrayView(int[] array, int length) { + this.array = requireNonNull(array, "array is null"); + checkArgument(length >= 0, "used slots count is negative"); + checkArgument(length <= array.length, "used slots count exceeds array size"); + this.length = length; + } + + public int get(int index) { + checkArgument(index >= 0 && index < length, "array index out of bounds"); + return array[index]; + } + + public int length() { + return length; + } + + @VisibleForTesting + public int[] toArray() { + return Arrays.copyOf(array, length); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Done.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Done.java new file mode 100644 index 000000000000..08e26baca8a8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Done.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +class Done implements Instruction { + @Override + public Type type() { + return Type.DONE; + } + + @Override + public String toString() { + return "done"; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + return (obj != null) && (getClass() == obj.getClass()); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Instruction.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Instruction.java new file mode 100644 index 000000000000..68400950271c --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Instruction.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +interface Instruction { + enum Type { + JUMP, + MATCH_LABEL, + MATCH_START, + MATCH_END, + SAVE, + SPLIT, + DONE + } + + Type type(); + + static Jump jump(int target) { + return new Jump(target); + } + + static Save save() { + return new Save(); + } + + static Split split(int first, int second) { + return new Split(first, second); + } + + static Done done() { + return new Done(); + } + + static MatchLabel match(int label) { + return new MatchLabel(label); + } + + static MatchStart matchStart() { + return new MatchStart(); + } + + static MatchEnd matchEnd() { + return new MatchEnd(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IntList.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IntList.java new file mode 100644 index 000000000000..c5fd5dd2620d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IntList.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.Arrays; + +public class IntList { + private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(IntList.class); + + private int[] values; + private int next; + + public IntList(int capacity) { + this.values = new int[capacity]; + } + + private IntList(int[] values, int next) { + this.values = values; + this.next = next; + } + + public void add(int value) { + ensureCapacity(next); + values[next] = value; + next++; + } + + public int get(int index) { + return values[index]; + } + + public void set(int index, int value) { + ensureCapacity(index); + values[index] = value; + next = Math.max(next, index + 1); + } + + public int size() { + return next; + } + + public void clear() { + next = 0; + } + + public IntList copy() { + return new IntList(values.clone(), next); + } + + public ArrayView toArrayView() { + return new ArrayView(values, next); + } + + private void ensureCapacity(int index) { + if (index >= values.length) { + values = Arrays.copyOf(values, Math.max(values.length * 2, index + 1)); + } + } + + public long getSizeInBytes() { + return INSTANCE_SIZE + RamUsageEstimator.sizeOf(values); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IntMultimap.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IntMultimap.java new file mode 100644 index 000000000000..220103642206 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IntMultimap.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.Arrays; + +class IntMultimap { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(IntMultimap.class); + + private IntList[] values; + private final int capacity; + private final int listCapacity; + private long valuesSize; + + public IntMultimap(int capacity, int listCapacity) { + this.values = new IntList[capacity]; + this.capacity = capacity; + this.listCapacity = listCapacity; + this.valuesSize = 0L; + } + + public void add(int key, int value) { + boolean expanded = ensureCapacity(key); + long listSizeBefore; + if (expanded || values[key] == null) { + listSizeBefore = 0L; + values[key] = new IntList(listCapacity); + } else { + listSizeBefore = values[key].getSizeInBytes(); + } + values[key].add(value); + valuesSize += values[key].getSizeInBytes() - listSizeBefore; + } + + public void release(int key) { + if (values[key] != null) { + valuesSize -= values[key].getSizeInBytes(); + values[key] = null; + } + } + + public void copy(int parent, int child) { + boolean expanded = ensureCapacity(child); + if (expanded || values[child] == null) { + if (values[parent] != null) { + values[child] = values[parent].copy(); + valuesSize += values[child].getSizeInBytes(); + } + } else if (values[parent] != null) { + long listSizeBefore = values[child].getSizeInBytes(); + values[child] = values[parent].copy(); + valuesSize += values[child].getSizeInBytes() - listSizeBefore; + } else { + valuesSize -= values[child].getSizeInBytes(); + values[child] = null; + } + } + + public ArrayView getArrayView(int key) { + if (values[key] == null) { + return ArrayView.EMPTY; + } + return values[key].toArrayView(); + } + + public void clear() { + values = new IntList[capacity]; + valuesSize = 0L; + } + + // returns true if the array was expanded; otherwise returns false + private boolean ensureCapacity(int key) { + if (key >= values.length) { + values = Arrays.copyOf(values, Math.max(values.length * 2, key + 1)); + return true; + } + + return false; + } + + public long getSizeInBytes() { + // TODO: + // return INSTANCE_SIZE + RamUsageEstimator.sizeOf(values) + valuesSize; + return 0; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IntStack.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IntStack.java new file mode 100644 index 000000000000..47fbc1ca5d4a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IntStack.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +import org.apache.tsfile.utils.RamUsageEstimator; + +import java.util.Arrays; + +class IntStack { + private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(IntStack.class); + + private int[] values; + private int next; + + public IntStack(int capacity) { + values = new int[capacity]; + } + + public void push(int value) { + ensureCapacity(); + values[next] = value; + next++; + } + + public int pop() { + next--; + return values[next]; + } + + public int size() { + return next; + } + + private void ensureCapacity() { + if (next == values.length) { + values = Arrays.copyOf(values, next * 2 + 1); + } + } + + public long getSizeInBytes() { + return INSTANCE_SIZE + RamUsageEstimator.sizeOf(values); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IrRowPatternToProgramRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IrRowPatternToProgramRewriter.java new file mode 100644 index 000000000000..37de296d54c5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/IrRowPatternToProgramRewriter.java @@ -0,0 +1,264 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrAlternation; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrAnchor; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrConcatenation; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrEmpty; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrExclusion; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrLabel; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrPermutation; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrQuantified; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrQuantifier; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrRowPattern; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrRowPatternVisitor; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.IntStream; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.Collections2.orderedPermutations; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; + +public class IrRowPatternToProgramRewriter { + private IrRowPatternToProgramRewriter() {} + + public static Program rewrite(IrRowPattern node, Map labelMapping) { + List instructions = new ArrayList<>(); + new Rewriter(instructions, labelMapping).process(node); + instructions.add(new Done()); + return new Program(instructions); + } + + private static class Rewriter extends IrRowPatternVisitor { + private final List instructions; + private final Map labelMapping; + + public Rewriter(List instructions, Map labelMapping) { + this.instructions = requireNonNull(instructions, "instructions is null"); + this.labelMapping = requireNonNull(labelMapping, "labelMapping is null"); + } + + @Override + protected Void visitIrRowPattern(IrRowPattern node, Void context) { + throw new UnsupportedOperationException( + "unsupported node type: " + node.getClass().getName()); + } + + @Override + protected Void visitIrLabel(IrLabel node, Void context) { + instructions.add(new MatchLabel(labelMapping.get(node))); + return null; + } + + @Override + protected Void visitIrEmpty(IrEmpty node, Void context) { + return null; + } + + @Override + protected Void visitIrAnchor(IrAnchor node, Void context) { + switch (node.getType()) { + case PARTITION_START: + instructions.add(new MatchStart()); + return null; + case PARTITION_END: + instructions.add(new MatchEnd()); + return null; + default: + throw new IllegalStateException("unexpected anchor type: " + node.getType()); + } + } + + @Override + protected Void visitIrExclusion(IrExclusion node, Void context) { + instructions.add(new Save()); + process(node.getPattern()); + instructions.add(new Save()); + return null; + } + + @Override + protected Void visitIrAlternation(IrAlternation node, Void context) { + List parts = node.getPatterns(); + List jumpPositions = new ArrayList<>(); + + for (int i = 0; i < parts.size() - 1; i++) { + int splitPosition = instructions.size(); + instructions.add(null); // placeholder for the Split instruction + int splitTarget = instructions.size(); + process(parts.get(i)); + jumpPositions.add(instructions.size()); + instructions.add(null); // placeholder for the Jump instruction + instructions.set(splitPosition, new Split(splitTarget, instructions.size())); + } + + process(parts.get(parts.size() - 1)); + + for (int position : jumpPositions) { + instructions.set(position, new Jump(instructions.size())); + } + + return null; + } + + @Override + protected Void visitIrConcatenation(IrConcatenation node, Void context) { + concatenation(node.getPatterns()); + return null; + } + + @Override + protected Void visitIrPermutation(IrPermutation node, Void context) { + checkArgument( + node.getPatterns().size() > 1, + "invalid pattern: permutation with single element. run IrRowPatternFlattener first"); + + List indexes = + IntStream.range(0, node.getPatterns().size()).boxed().collect(toImmutableList()); + + List> permutations = + orderedPermutations(indexes).stream() + .map( + permutation -> + permutation.stream().map(node.getPatterns()::get).collect(toImmutableList())) + .collect(toImmutableList()); + + alternation(permutations); + + return null; + } + + private void concatenation(List patterns) { + patterns.forEach(this::process); + } + + private void alternation(List> parts) { + List jumpPositions = new ArrayList<>(); + + for (int i = 0; i < parts.size() - 1; i++) { + int splitPosition = instructions.size(); + instructions.add(null); // placeholder for the Split instruction + int splitTarget = instructions.size(); + concatenation(parts.get(i)); + jumpPositions.add(instructions.size()); + instructions.add(null); // placeholder for the Jump instruction + instructions.set(splitPosition, new Split(splitTarget, instructions.size())); + } + + concatenation(parts.get(parts.size() - 1)); + + for (int position : jumpPositions) { + instructions.set(position, new Jump(instructions.size())); + } + } + + @Override + protected Void visitIrQuantified(IrQuantified node, Void context) { + IrRowPattern pattern = node.getPattern(); + IrQuantifier quantifier = node.getQuantifier(); + boolean greedy = quantifier.isGreedy(); + + if (quantifier.getAtMost().isPresent()) { + rangeQuantified(pattern, greedy, quantifier.getAtLeast(), quantifier.getAtMost().get()); + } else { + loopingQuantified(pattern, greedy, quantifier.getAtLeast()); + } + + return null; + } + + private void loopingQuantified(IrRowPattern pattern, boolean greedy, int min) { + checkArgument(min >= 0, "invalid min value: %s", min); + + if (min == 0) { + int startSplitPosition = instructions.size(); + instructions.add(null); // placeholder for the Split instruction + int splitTarget = instructions.size(); + int loopingPosition = instructions.size(); + + process(pattern); + loop(loopingPosition, greedy); + + if (greedy) { + instructions.set(startSplitPosition, new Split(splitTarget, instructions.size())); + } else { + instructions.set(startSplitPosition, new Split(instructions.size(), splitTarget)); + } + return; + } + + int loopingPosition = instructions.size(); + for (int i = 0; i < min; i++) { + loopingPosition = instructions.size(); + process(pattern); + } + + loop(loopingPosition, greedy); + } + + private void loop(int loopingPosition, boolean greedy) { + Split loopingSplit; + if (greedy) { + loopingSplit = new Split(loopingPosition, instructions.size() + 1); + } else { + loopingSplit = new Split(instructions.size() + 1, loopingPosition); + } + instructions.add(loopingSplit); + } + + private void rangeQuantified(IrRowPattern pattern, boolean greedy, int min, int max) { + checkArgument(min <= max, "invalid range: (%s, %s)", min, max); + + for (int i = 0; i < min; i++) { + process(pattern); + } + + // handles 0 without adding instructions + if (min == max) { + return; + } + + List splitPositions = new ArrayList<>(); + List splitTargets = new ArrayList<>(); + + for (int i = min; i < max; i++) { + splitPositions.add(instructions.size()); + instructions.add(null); // placeholder for the Split instruction + splitTargets.add(instructions.size()); + process(pattern); + } + + for (int i = 0; i < splitPositions.size(); i++) { + if (greedy) { + instructions.set( + splitPositions.get(i), new Split(splitTargets.get(i), instructions.size())); + } else { + instructions.set( + splitPositions.get(i), new Split(instructions.size(), splitTargets.get(i))); + } + } + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Jump.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Jump.java new file mode 100644 index 000000000000..c75698c97f82 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Jump.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +import java.util.Objects; + +class Jump implements Instruction { + private final int target; + + public Jump(int target) { + this.target = target; + } + + public int getTarget() { + return target; + } + + @Override + public String toString() { + return "jump " + target; + } + + @Override + public Type type() { + return Type.JUMP; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + return target == ((Jump) obj).target; + } + + @Override + public int hashCode() { + return Objects.hash(target); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchEnd.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchEnd.java new file mode 100644 index 000000000000..7d14586186ba --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchEnd.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +class MatchEnd implements Instruction { + @Override + public Type type() { + return Type.MATCH_END; + } + + @Override + public String toString() { + return "match-end"; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + return (obj != null) && (getClass() == obj.getClass()); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchLabel.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchLabel.java new file mode 100644 index 000000000000..d839508ea30e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchLabel.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +import java.util.Objects; + +class MatchLabel implements Instruction { + private final int label; + + public MatchLabel(int label) { + this.label = label; + } + + public int getLabel() { + return label; + } + + @Override + public String toString() { + return "match " + label; + } + + @Override + public Type type() { + return Type.MATCH_LABEL; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + return label == ((MatchLabel) obj).label; + } + + @Override + public int hashCode() { + return Objects.hash(label); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchResult.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchResult.java new file mode 100644 index 000000000000..fd79e55589db --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchResult.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +import static java.util.Objects.requireNonNull; + +public class MatchResult { + public static final MatchResult NO_MATCH = + new MatchResult(false, ArrayView.EMPTY, ArrayView.EMPTY); + + private final boolean matched; + private final ArrayView labels; + private final ArrayView exclusions; + + public MatchResult(boolean matched, ArrayView labels, ArrayView exclusions) { + this.matched = matched; + this.labels = requireNonNull(labels, "labels is null"); + this.exclusions = requireNonNull(exclusions, "exclusions is null"); + } + + public boolean isMatched() { + return matched; + } + + public ArrayView getLabels() { + return labels; + } + + public ArrayView getExclusions() { + return exclusions; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchStart.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchStart.java new file mode 100644 index 000000000000..f8dd5e5597a2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/MatchStart.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +class MatchStart implements Instruction { + @Override + public Type type() { + return Type.MATCH_START; + } + + @Override + public String toString() { + return "match-start"; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + return (obj != null) && (getClass() == obj.getClass()); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Matcher.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Matcher.java new file mode 100644 index 000000000000..340171927a59 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Matcher.java @@ -0,0 +1,277 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.LabelEvaluator; + +import org.apache.tsfile.utils.RamUsageEstimator; + +import static org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher.MatchResult.NO_MATCH; + +public class Matcher { + private final Program program; + + // private final ThreadEquivalence threadEquivalence; + // private final List aggregations; + + private static class Runtime { + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(Runtime.class); + + // a helper structure for identifying equivalent threads + // program pointer (instruction) --> list of threads that have reached this instruction + private final IntMultimap threadsAtInstructions; + // threads that should be killed as determined by the current iteration of the main loop + // they are killed after the iteration so that they can be used to kill other threads while the + // iteration lasts + private final IntList threadsToKill; + + private final IntList threads; + private final IntStack freeThreadIds; + private int newThreadId; + private final int inputLength; + private final boolean matchingAtPartitionStart; + private final PatternCaptures patternCaptures; + + // for each thread, array of MatchAggregations evaluated by this thread + // private final MatchAggregations aggregations; + + public Runtime(Program program, int inputLength, boolean matchingAtPartitionStart) { + int initialCapacity = 2 * program.size(); + threads = new IntList(initialCapacity); + freeThreadIds = new IntStack(initialCapacity); + this.patternCaptures = + new PatternCaptures( + initialCapacity, program.getMinSlotCount(), program.getMinLabelCount()); + this.inputLength = inputLength; + this.matchingAtPartitionStart = matchingAtPartitionStart; + // this.aggregations = + // new MatchAggregations( + // initialCapacity, aggregationInstantiators, aggregationsMemoryContext); + + this.threadsAtInstructions = new IntMultimap(program.size(), program.size()); + this.threadsToKill = new IntList(initialCapacity); + } + + private int forkThread(int parent) { + int child = newThread(); + patternCaptures.copy(parent, child); + // aggregations.copy(parent, child); + return child; + } + + private int newThread() { + if (freeThreadIds.size() > 0) { + return freeThreadIds.pop(); + } + return newThreadId++; + } + + private void scheduleKill(int threadId) { + threadsToKill.add(threadId); + } + + private void killThreads() { + for (int i = 0; i < threadsToKill.size(); i++) { + killThread(threadsToKill.get(i)); + } + threadsToKill.clear(); + } + + private void killThread(int threadId) { + freeThreadIds.push(threadId); + patternCaptures.release(threadId); + // aggregations.release(threadId); + } + + private long getSizeInBytes() { + return INSTANCE_SIZE + + threadsAtInstructions.getSizeInBytes() + + threadsToKill.getSizeInBytes() + + threads.getSizeInBytes() + + freeThreadIds.getSizeInBytes() + + patternCaptures.getSizeInBytes(); + // + aggregations.getSizeInBytes(); + } + } + + public Matcher(Program program) { + this.program = program; + } + + public MatchResult run(LabelEvaluator labelEvaluator) { + IntList current = new IntList(program.size()); + IntList next = new IntList(program.size()); + + int inputLength = labelEvaluator.getInputLength(); + boolean matchingAtPartitionStart = labelEvaluator.isMatchingAtPartitionStart(); + + Runtime runtime = new Runtime(program, inputLength, matchingAtPartitionStart); + + advanceAndSchedule(current, runtime.newThread(), 0, 0, runtime); + + MatchResult result = NO_MATCH; + + for (int index = 0; index < inputLength; index++) { + if (current.size() == 0) { + // no match found -- all threads are dead + break; + } + boolean matched = false; + // For every existing thread, consume the label if possible. Otherwise, kill the thread. + // After consuming the label, advance to the next `MATCH_LABEL`. Collect the advanced threads + // in `next`, + // which will be the starting point for the next iteration. + + // clear the structure for new input index + runtime.threadsAtInstructions.clear(); + runtime.killThreads(); + + for (int i = 0; i < current.size(); i++) { + int threadId = current.get(i); + int pointer = runtime.threads.get(threadId); + Instruction instruction = program.at(pointer); + switch (instruction.type()) { + case MATCH_LABEL: + int label = ((MatchLabel) instruction).getLabel(); + // save the label before evaluating the defining condition, because evaluating assumes + // that the label is tentatively matched + // - if the condition is true, the label is already saved + // - if the condition is false, the thread is killed along with its patternCaptures, + // so the + // incorrectly saved label does not matter + runtime.patternCaptures.saveLabel(threadId, label); + if (labelEvaluator.evaluateLabel(runtime.patternCaptures.getLabels(threadId))) { + advanceAndSchedule(next, threadId, pointer + 1, index + 1, runtime); + } else { + runtime.scheduleKill(threadId); + } + break; + case DONE: + matched = true; + result = + new MatchResult( + true, + runtime.patternCaptures.getLabels(threadId), + runtime.patternCaptures.getCaptures(threadId)); + runtime.scheduleKill(threadId); + break; + default: + throw new UnsupportedOperationException("not yet implemented"); + } + if (matched) { + // do not process the following threads, because they are on less preferred paths than the + // match found + for (int j = i + 1; j < current.size(); j++) { + runtime.scheduleKill(current.get(j)); + } + break; + } + } + + IntList temp = current; + temp.clear(); + current = next; + next = temp; + } + + // handle the case when the program still has instructions to process after consuming the whole + // input + for (int i = 0; i < current.size(); i++) { + int threadId = current.get(i); + if (program.at(runtime.threads.get(threadId)).type() == Instruction.Type.DONE) { + result = + new MatchResult( + true, + runtime.patternCaptures.getLabels(threadId), + runtime.patternCaptures.getCaptures(threadId)); + break; + } + } + + return result; + } + + /** + * For a particular thread identified by `threadId`, process consecutive instructions of the + * program, from the instruction at `pointer` up to the next instruction which consumes a label or + * to the program end. The resulting thread state (the pointer of the first not processed + * instruction) is recorded in `next`. There might be multiple threads recorded in `next`, as a + * result of the instruction `SPLIT`. + */ + private void advanceAndSchedule( + IntList next, int threadId, int pointer, int inputIndex, Runtime runtime) { + // TODO + // avoid empty loop and try avoid exponential processing + // ArrayView threadsAtInstruction = runtime.threadsAtInstructions.getArrayView(pointer); + // for (int i = 0; i < threadsAtInstruction.length(); i++) { + // int thread = threadsAtInstruction.get(i); + // if (threadEquivalence.equivalent( + // thread, + // runtime.patternCaptures.getLabels(thread), + // runtime.aggregations.get(thread), + // threadId, + // runtime.patternCaptures.getLabels(threadId), + // runtime.aggregations.get(threadId), + // pointer)) { + // // in case of equivalent threads, kill the one that comes later, because it is on a + // less + // // preferred path + // runtime.scheduleKill(threadId); + // return; + // } + // } + // runtime.threadsAtInstructions.add(pointer, threadId); + + Instruction instruction = program.at(pointer); + switch (instruction.type()) { + case MATCH_START: + if (inputIndex == 0 && runtime.matchingAtPartitionStart) { + advanceAndSchedule(next, threadId, pointer + 1, inputIndex, runtime); + } else { + runtime.scheduleKill(threadId); + } + break; + case MATCH_END: + if (inputIndex == runtime.inputLength) { + advanceAndSchedule(next, threadId, pointer + 1, inputIndex, runtime); + } else { + runtime.scheduleKill(threadId); + } + break; + case JUMP: + advanceAndSchedule(next, threadId, ((Jump) instruction).getTarget(), inputIndex, runtime); + break; + case SPLIT: + int forked = runtime.forkThread(threadId); + advanceAndSchedule(next, threadId, ((Split) instruction).getFirst(), inputIndex, runtime); + advanceAndSchedule(next, forked, ((Split) instruction).getSecond(), inputIndex, runtime); + break; + case SAVE: + runtime.patternCaptures.save(threadId, inputIndex); + advanceAndSchedule(next, threadId, pointer + 1, inputIndex, runtime); + break; + default: // MATCH_LABEL or DONE + runtime.threads.set(threadId, pointer); + next.add(threadId); + break; + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/PatternCaptures.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/PatternCaptures.java new file mode 100644 index 000000000000..640ae54e98c2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/PatternCaptures.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +import org.apache.iotdb.db.queryengine.plan.relational.utils.matching.Captures; + +import org.apache.tsfile.utils.RamUsageEstimator; + +class PatternCaptures { + private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(Captures.class); + + private final IntMultimap captures; + private final IntMultimap labels; + + public PatternCaptures(int initialCapacity, int slotCount, int labelCount) { + this.captures = new IntMultimap(initialCapacity, slotCount); + this.labels = new IntMultimap(initialCapacity, labelCount); + } + + public void save(int threadId, int value) { + captures.add(threadId, value); + } + + public void saveLabel(int threadId, int value) { + labels.add(threadId, value); + } + + public void copy(int parent, int child) { + captures.copy(parent, child); + labels.copy(parent, child); + } + + public ArrayView getCaptures(int threadId) { + return captures.getArrayView(threadId); + } + + public ArrayView getLabels(int threadId) { + return labels.getArrayView(threadId); + } + + public void release(int threadId) { + captures.release(threadId); + labels.release(threadId); + } + + public long getSizeInBytes() { + return INSTANCE_SIZE + captures.getSizeInBytes() + labels.getSizeInBytes(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Program.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Program.java new file mode 100644 index 000000000000..d679cad42dcb --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Program.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +import static java.lang.Math.toIntExact; + +public class Program { + private final List instructions; + private final int minSlotCount; + private final int minLabelCount; + + public Program(List instructions) { + this.instructions = ImmutableList.copyOf(instructions); + this.minSlotCount = + toIntExact( + instructions.stream() + .filter(instruction -> instruction.type() == Instruction.Type.SAVE) + .count()); + this.minLabelCount = + toIntExact( + instructions.stream() + .filter(instruction -> instruction.type() == Instruction.Type.MATCH_LABEL) + .count()); + } + + public Instruction at(int pointer) { + return instructions.get(pointer); + } + + public int size() { + return instructions.size(); + } + + public List getInstructions() { + return ImmutableList.copyOf(instructions); + } + + public int getMinSlotCount() { + return minSlotCount; + } + + public int getMinLabelCount() { + return minLabelCount; + } + + public String dump() { + StringBuilder builder = new StringBuilder(); + + builder + .append("Min slots: ") + .append(minSlotCount) + .append("\n") + .append("Min labels: ") + .append(minLabelCount) + .append("\n"); + for (int i = 0; i < instructions.size(); i++) { + builder.append(String.format("%s: %s\n", i, instructions.get(i))); + } + + return builder.toString(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Save.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Save.java new file mode 100644 index 000000000000..fceac1428825 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Save.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +class Save implements Instruction { + @Override + public String toString() { + return "save"; + } + + @Override + public Type type() { + return Type.SAVE; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + return (obj != null) && (getClass() == obj.getClass()); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Split.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Split.java new file mode 100644 index 000000000000..5376ad44ba26 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/matcher/Split.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher; + +import java.util.Objects; + +import static java.lang.String.format; + +class Split implements Instruction { + private final int first; + private final int second; + + public Split(int first, int second) { + this.first = first; + this.second = second; + } + + public int getFirst() { + return first; + } + + public int getSecond() { + return second; + } + + @Override + public String toString() { + return format("split %s, %s", first, second); + } + + @Override + public Type type() { + return Type.SPLIT; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + Split o = (Split) obj; + return first == o.first && second == o.second; + } + + @Override + public int hashCode() { + return Objects.hash(first, second); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java index dd67e1293609..6f767f1019d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanNodeType.java @@ -282,6 +282,7 @@ public enum PlanNodeType { TABLE_EXPLAIN_ANALYZE_NODE((short) 1019), TABLE_ENFORCE_SINGLE_ROW_NODE((short) 1020), INFORMATION_SCHEMA_TABLE_SCAN_NODE((short) 1021), + TABLE_PATTERN_RECOGNITION_NODE((short) 1022), RELATIONAL_INSERT_TABLET((short) 2000), RELATIONAL_INSERT_ROW((short) 2001), diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java index 9c172b53b1c2..865d58f1706e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/PlanVisitor.java @@ -125,6 +125,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PatternRecognitionNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ValueFillNode; @@ -770,4 +771,8 @@ public R visitAggregationTableScan( C context) { return visitDeviceTableScan(node, context); } + + public R visitPatternRecognition(PatternRecognitionNode node, C context) { + return visitPlan(node, context); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java index 7b930febf75b..c0df52abb591 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/Analysis.java @@ -32,7 +32,7 @@ import org.apache.iotdb.db.queryengine.plan.execution.memory.TableModelStatementMemorySourceContext; import org.apache.iotdb.db.queryengine.plan.execution.memory.TableModelStatementMemorySourceVisitor; import org.apache.iotdb.db.queryengine.plan.planner.plan.TimePredicate; -import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.PatternInputAnalysis; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.PatternFunctionAnalysis; import org.apache.iotdb.db.queryengine.plan.relational.metadata.ColumnSchema; import org.apache.iotdb.db.queryengine.plan.relational.metadata.QualifiedObjectName; import org.apache.iotdb.db.queryengine.plan.relational.metadata.ResolvedFunction; @@ -148,7 +148,7 @@ public class Analysis implements IAnalysis { // Pattern function analysis (classifier, match_number, aggregations and prev/next/first/last) in // the context of the given node - private final Map, List> patternInputsAnalysis = + private final Map, List> patternFunctionAnalysis = new LinkedHashMap<>(); // FunctionCall nodes corresponding to any of the special pattern recognition functions @@ -696,20 +696,20 @@ public Set getUndefinedLabels(RowPattern pattern) { } public void addPatternRecognitionInputs( - Map, List> functions) { - patternInputsAnalysis.putAll(functions); + Map, List> functions) { + patternFunctionAnalysis.putAll(functions); functions.values().stream() .flatMap(List::stream) - .map(PatternInputAnalysis::getExpression) + .map(PatternFunctionAnalysis::getExpression) .filter(FunctionCall.class::isInstance) .map(FunctionCall.class::cast) .map(NodeRef::of) .forEach(patternRecognitionFunctionCalls::add); } - public List getPatternInputsAnalysis(Expression expression) { - return patternInputsAnalysis.get(NodeRef.of(expression)); + public List getPatternInputsAnalysis(Expression expression) { + return patternFunctionAnalysis.get(NodeRef.of(expression)); } public void addPatternNavigationFunctions(Set> functions) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java index a88735baf5cf..cd54542bf7b3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/ExpressionAnalyzer.java @@ -29,7 +29,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.MatchNumberDescriptor; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.Navigation; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.NavigationMode; -import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.PatternInputAnalysis; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.PatternFunctionAnalysis; import org.apache.iotdb.db.queryengine.plan.relational.analyzer.PatternRecognitionAnalysis.ScalarInputDescriptor; import org.apache.iotdb.db.queryengine.plan.relational.function.BoundSignature; import org.apache.iotdb.db.queryengine.plan.relational.function.FunctionId; @@ -176,7 +176,7 @@ public class ExpressionAnalyzer { // Pattern function analysis (classifier, match_number, aggregations and prev/next/first/last) in // the context of the given node - private final Map, List> patternRecognitionInputs = + private final Map, List> patternRecognitionInputs = new LinkedHashMap<>(); private final Set> patternNavigationFunctions = new LinkedHashSet<>(); @@ -295,6 +295,10 @@ public Type analyze(Expression expression, Scope scope, CorrelationSupport corre private Type analyze(Expression expression, Scope scope, Set labels) { Visitor visitor = new Visitor(scope, warningCollector); + + // TODO + patternRecognitionInputs.put(NodeRef.of(expression), visitor.getPatternRecognitionInputs()); + return visitor.process( expression, new StackableAstVisitor.StackableAstVisitorContext<>( @@ -347,7 +351,7 @@ public Map, Set> getSubsetLabels() { return subsets; } - public Map, List> getPatternRecognitionInputs() { + public Map, List> getPatternRecognitionInputs() { return patternRecognitionInputs; } @@ -360,13 +364,17 @@ private class Visitor extends StackableAstVisitor { private final Scope baseScope; private final WarningCollector warningCollector; - private final List patternRecognitionInputs = new ArrayList<>(); + private final List patternRecognitionInputs = new ArrayList<>(); public Visitor(Scope baseScope, WarningCollector warningCollector) { this.baseScope = requireNonNull(baseScope, "baseScope is null"); this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); } + public List getPatternRecognitionInputs() { + return patternRecognitionInputs; + } + @Override public Type process(Node node, @Nullable StackableAstVisitorContext context) { if (node instanceof Expression) { @@ -505,7 +513,7 @@ protected Type visitDereferenceExpression( labels.put(NodeRef.of(node), Optional.of(label)); patternRecognitionInputs.add( - new PatternInputAnalysis( + new PatternFunctionAnalysis( node, new ScalarInputDescriptor( Optional.of(label), @@ -988,7 +996,7 @@ private Type analyzeMatchNumber( throw new SemanticException("MATCH_NUMBER pattern recognition function takes no arguments"); } - patternRecognitionInputs.add(new PatternInputAnalysis(node, new MatchNumberDescriptor())); + patternRecognitionInputs.add(new PatternFunctionAnalysis(node, new MatchNumberDescriptor())); return INT64; } @@ -1023,7 +1031,7 @@ private Type analyzeClassifier(FunctionCall node, StackableAstVisitorContext orderingScheme = Optional.empty(); + if (!orderings.isEmpty()) { + orderingScheme = + Optional.of(new OrderingScheme(ImmutableList.copyOf(orderings.keySet()), orderings)); + } + + outputLayout.addAll(partitionBy); + if (!oneRowOutput) { + getSortItemsFromOrderBy(node.getOrderBy()).stream() + .map(SortItem::getSortKey) + .map(planBuilder::translate) + .forEach(outputLayout::add); + } + + planBuilder = + subqueryPlanner.handleSubqueries( + planBuilder, + extractPatternRecognitionExpressions(node.getVariableDefinitions(), node.getMeasures()), + analysis.getSubqueries(node)); + + PatternRecognitionComponents components = + planPatternRecognitionComponents( + planBuilder.getTranslations(), + node.getSubsets(), + node.getMeasures(), + node.getAfterMatchSkipTo(), + node.getPattern(), + node.getVariableDefinitions()); + + outputLayout.addAll(components.getMeasureOutputs()); + + if (!oneRowOutput) { + Set inputSymbolsOnOutput = ImmutableSet.copyOf(outputLayout.build()); + subPlan.getFieldMappings().stream() + .filter(symbol -> !inputSymbolsOnOutput.contains(symbol)) + .forEach(outputLayout::add); + } + + PatternRecognitionNode planNode = + new PatternRecognitionNode( + idAllocator.genPlanNodeId(), + planBuilder.getRoot(), + partitionBy, + orderingScheme, + Optional.empty(), + components.getMeasures(), + rowsPerMatch, + components.getSkipToLabels(), + components.getSkipToPosition(), + components.getPattern(), + components.getVariableDefinitions()); + + return new RelationPlan(planNode, analysis.getScope(node), outputLayout.build(), outerContext); + } + + private RowsPerMatch mapRowsPerMatch(PatternRecognitionRelation.RowsPerMatch rowsPerMatch) { + switch (rowsPerMatch) { + case ONE: + return RowsPerMatch.ONE; + case ALL_SHOW_EMPTY: + return RowsPerMatch.ALL_SHOW_EMPTY; + case ALL_OMIT_EMPTY: + return RowsPerMatch.ALL_OMIT_EMPTY; + case ALL_WITH_UNMATCHED: + return RowsPerMatch.ALL_WITH_UNMATCHED; + default: + throw new IllegalArgumentException("Unexpected value: " + rowsPerMatch); + } + } + + public PatternRecognitionComponents planPatternRecognitionComponents( + TranslationMap translations, + List subsets, + List measures, + Optional skipTo, + RowPattern pattern, + List variableDefinitions) { + // NOTE: There might be aggregate functions in measure definitions and variable definitions. + // They are handled different than top level aggregations in a query: + // 1. Their arguments are not pre-projected and replaced with single symbols. This is because + // the arguments might + // not be eligible for pre-projection, when they contain references to CLASSIFIER() or + // MATCH_NUMBER() functions + // which are evaluated at runtime. If some aggregation arguments can be pre-projected, it + // will be done in the + // Optimizer. + // 2. Their arguments do not need to be coerced by hand. Since the pattern aggregation arguments + // are rewritten as + // parts of enclosing expressions, and not as standalone expressions, all necessary coercions + // will be applied by the + // TranslationMap. + + // rewrite subsets + ImmutableMap.Builder> rewrittenSubsetsBuilder = ImmutableMap.builder(); + for (SubsetDefinition subsetDefinition : subsets) { + String label = analysis.getResolvedLabel(subsetDefinition.getName()); + Set elements = + analysis.getSubsetLabels(subsetDefinition).stream() + .map(IrLabel::new) + .collect(toImmutableSet()); + rewrittenSubsetsBuilder.put(new IrLabel(label), elements); + } + Map> rewrittenSubsets = rewrittenSubsetsBuilder.buildOrThrow(); + + // rewrite measures + ImmutableMap.Builder rewrittenMeasures = ImmutableMap.builder(); + ImmutableList.Builder measureOutputs = ImmutableList.builder(); + + for (MeasureDefinition definition : measures) { + Type type = analysis.getType(definition.getExpression()); + Symbol symbol = symbolAllocator.newSymbol(definition.getName().getValue(), type); + ExpressionAndValuePointers measure = + planPatternRecognitionExpression( + translations, + rewrittenSubsets, + definition.getName().getValue(), + definition.getExpression()); + rewrittenMeasures.put(symbol, new Measure(measure, type)); + measureOutputs.add(symbol); + } + + // rewrite variable definitions + ImmutableMap.Builder rewrittenVariableDefinitions = + ImmutableMap.builder(); + for (VariableDefinition definition : variableDefinitions) { + String label = analysis.getResolvedLabel(definition.getName()); + ExpressionAndValuePointers variable = + planPatternRecognitionExpression( + translations, + rewrittenSubsets, + definition.getName().getValue(), + definition.getExpression()); + rewrittenVariableDefinitions.put(new IrLabel(label), variable); + } + // add `true` definition for undefined labels + for (String label : analysis.getUndefinedLabels(pattern)) { + IrLabel irLabel = new IrLabel(label); + rewrittenVariableDefinitions.put(irLabel, ExpressionAndValuePointers.TRUE); + } + + Set skipToLabels = + skipTo + .flatMap(SkipTo::getIdentifier) + .map(Identifier::getValue) + .map( + label -> + rewrittenSubsets.getOrDefault( + new IrLabel(label), ImmutableSet.of(new IrLabel(label)))) + .orElse(ImmutableSet.of()); + + return new PatternRecognitionComponents( + rewrittenMeasures.buildOrThrow(), + measureOutputs.build(), + skipToLabels, + mapSkipToPosition(skipTo.map(SkipTo::getPosition).orElse(PAST_LAST)), + RowPatternToIrRewriter.rewrite(pattern, analysis), + rewrittenVariableDefinitions.buildOrThrow()); + } + + private ExpressionAndValuePointers planPatternRecognitionExpression( + TranslationMap translations, + Map> subsets, + String name, + Expression expression) { + Map, Symbol> patternVariableTranslations = new HashMap<>(); + + ImmutableList.Builder assignments = ImmutableList.builder(); + for (PatternRecognitionAnalysis.PatternFunctionAnalysis accessor : + analysis.getPatternInputsAnalysis(expression)) { + ValuePointer pointer; + if (accessor.getDescriptor() instanceof MatchNumberDescriptor) { + pointer = new MatchNumberValuePointer(); + } else if (accessor.getDescriptor() instanceof ClassifierDescriptor) { + ClassifierDescriptor descriptor = (ClassifierDescriptor) accessor.getDescriptor(); + pointer = + new ClassifierValuePointer( + planValuePointer(descriptor.getLabel(), descriptor.getNavigation(), subsets)); + } else if (accessor.getDescriptor() instanceof ScalarInputDescriptor) { + ScalarInputDescriptor descriptor = (ScalarInputDescriptor) accessor.getDescriptor(); + pointer = + new ScalarValuePointer( + planValuePointer(descriptor.getLabel(), descriptor.getNavigation(), subsets), + Symbol.from(translations.rewrite(accessor.getExpression()))); + } else { + throw new IllegalArgumentException( + "Unexpected descriptor type: " + accessor.getDescriptor().getClass().getName()); + } + + Symbol symbol = symbolAllocator.newSymbol(name, analysis.getType(accessor.getExpression())); + assignments.add(new Assignment(symbol, pointer)); + + patternVariableTranslations.put(NodeRef.of(accessor.getExpression()), symbol); + } + + Expression rewritten = + translations + .withAdditionalIdentityMappings(patternVariableTranslations) + .rewrite(expression); + + return new ExpressionAndValuePointers(rewritten, assignments.build()); + } + + private Set planLabels(Optional label, Map> subsets) { + return label + .map(IrLabel::new) + .map(value -> subsets.getOrDefault(value, ImmutableSet.of(value))) + .orElse(ImmutableSet.of()); + } + + private LogicalIndexPointer planValuePointer( + Optional label, Navigation navigation, Map> subsets) { + return new LogicalIndexPointer( + planLabels(label, subsets), + navigation.getAnchor() == LAST, + navigation.getMode() == RUNNING, + navigation.getLogicalOffset(), + navigation.getPhysicalOffset()); + } + + private SkipToPosition mapSkipToPosition(SkipTo.Position position) { + switch (position) { + case NEXT: + return SkipToPosition.NEXT; + case PAST_LAST: + return SkipToPosition.PAST_LAST; + case FIRST: + return SkipToPosition.FIRST; + case LAST: + return SkipToPosition.LAST; + default: + throw new IllegalArgumentException("Unexpected value: " + position); + } + } + // ================================ Implemented later ===================================== @Override @@ -753,4 +1064,52 @@ protected RelationPlan visitDelete(Delete node, Void context) { Collections.emptyList(), outerContext); } + + public static class PatternRecognitionComponents { + private final Map measures; + private final List measureOutputs; + private final Set skipToLabels; + private final SkipToPosition skipToPosition; + private final IrRowPattern pattern; + private final Map variableDefinitions; + + public PatternRecognitionComponents( + Map measures, + List measureOutputs, + Set skipToLabels, + SkipToPosition skipToPosition, + IrRowPattern pattern, + Map variableDefinitions) { + this.measures = requireNonNull(measures, "measures is null"); + this.measureOutputs = requireNonNull(measureOutputs, "measureOutputs is null"); + this.skipToLabels = ImmutableSet.copyOf(skipToLabels); + this.skipToPosition = requireNonNull(skipToPosition, "skipToPosition is null"); + this.pattern = requireNonNull(pattern, "pattern is null"); + this.variableDefinitions = requireNonNull(variableDefinitions, "variableDefinitions is null"); + } + + public Map getMeasures() { + return measures; + } + + public List getMeasureOutputs() { + return measureOutputs; + } + + public Set getSkipToLabels() { + return skipToLabels; + } + + public SkipToPosition getSkipToPosition() { + return skipToPosition; + } + + public IrRowPattern getPattern() { + return pattern; + } + + public Map getVariableDefinitions() { + return variableDefinitions; + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Measure.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Measure.java new file mode 100644 index 000000000000..273f7e829026 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Measure.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.ExpressionAndValuePointers; +import org.apache.iotdb.db.queryengine.plan.relational.utils.TypeUtil; + +import org.apache.tsfile.read.common.type.Type; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; + +public class Measure { + private final ExpressionAndValuePointers expressionAndValuePointers; + private final Type type; + + public Measure(ExpressionAndValuePointers expressionAndValuePointers, Type type) { + this.expressionAndValuePointers = + requireNonNull(expressionAndValuePointers, "expressionAndValuePointers is null"); + this.type = requireNonNull(type, "type is null"); + } + + public ExpressionAndValuePointers getExpressionAndValuePointers() { + return expressionAndValuePointers; + } + + public Type getType() { + return type; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + Measure that = (Measure) o; + return Objects.equals(expressionAndValuePointers, that.expressionAndValuePointers) + && Objects.equals(type, that.type); + } + + @Override + public int hashCode() { + return Objects.hash(expressionAndValuePointers, type); + } + + @Override + public String toString() { + return toStringHelper(this) + .add("expressionAndValuePointers", expressionAndValuePointers) + .add("type", type) + .toString(); + } + + public static void serialize(Measure measure, ByteBuffer byteBuffer) { + ExpressionAndValuePointers.serialize(measure.getExpressionAndValuePointers(), byteBuffer); + TypeUtil.serialize(measure.getType(), byteBuffer); + } + + public static void serialize(Measure measure, DataOutputStream stream) throws IOException { + ExpressionAndValuePointers.serialize(measure.expressionAndValuePointers, stream); + TypeUtil.serialize(measure.getType(), stream); + } + + public static Measure deserialize(ByteBuffer byteBuffer) { + ExpressionAndValuePointers expressionAndValuePointers = + ExpressionAndValuePointers.deserialize(byteBuffer); + Type type = TypeUtil.deserialize(byteBuffer); + return new Measure(expressionAndValuePointers, type); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/PatternRecognitionNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/PatternRecognitionNode.java new file mode 100644 index 000000000000..7eaa4c38098b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/PatternRecognitionNode.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeType; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.process.SingleChildProcessNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.OrderingScheme; +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.ExpressionAndValuePointers; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrLabel; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrRowPattern; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.node.RowsPerMatch.ONE; + +public class PatternRecognitionNode extends SingleChildProcessNode { + private final List partitionBy; + private final Optional orderingScheme; + private final Optional hashSymbol; + private final Map measures; + private final RowsPerMatch rowsPerMatch; + private final Set skipToLabels; + private final SkipToPosition skipToPosition; + private final IrRowPattern pattern; + private final Map variableDefinitions; + + public PatternRecognitionNode( + PlanNodeId id, + PlanNode child, + List partitionBy, + Optional orderingScheme, + Optional hashSymbol, + Map measures, + RowsPerMatch rowsPerMatch, + Set skipToLabels, + SkipToPosition skipToPosition, + IrRowPattern pattern, + Map variableDefinitions) { + super(id, child); + + this.partitionBy = requireNonNull(partitionBy, "partitionBy is null"); + this.orderingScheme = requireNonNull(orderingScheme, "orderingScheme is null"); + this.hashSymbol = requireNonNull(hashSymbol, "hashSymbol is null"); + this.measures = requireNonNull(measures, "measures is null"); + this.rowsPerMatch = requireNonNull(rowsPerMatch, "rowsPerMatch is null"); + this.skipToLabels = requireNonNull(skipToLabels, "skipToLabels is null"); + this.skipToPosition = requireNonNull(skipToPosition, "skipToPosition is null"); + this.pattern = requireNonNull(pattern, "pattern is null"); + this.variableDefinitions = requireNonNull(variableDefinitions, "variableDefinitions is null"); + } + + @Override + // The order of symbols in the returned list might be different than expected layout of the node + public List getOutputSymbols() { + ImmutableList.Builder outputSymbols = ImmutableList.builder(); + if (rowsPerMatch == ONE) { + outputSymbols.addAll(partitionBy); + } else { + outputSymbols.addAll(child.getOutputSymbols()); + } + outputSymbols.addAll(measures.keySet()); + + return outputSymbols.build(); + } + + public Set getCreatedSymbols() { + return ImmutableSet.copyOf(measures.keySet()); + } + + public List getPartitionBy() { + return partitionBy; + } + + public Optional getOrderingScheme() { + return orderingScheme; + } + + public Optional getHashSymbol() { + return hashSymbol; + } + + public Map getMeasures() { + return measures; + } + + public RowsPerMatch getRowsPerMatch() { + return rowsPerMatch; + } + + public Set getSkipToLabels() { + return skipToLabels; + } + + public SkipToPosition getSkipToPosition() { + return skipToPosition; + } + + public IrRowPattern getPattern() { + return pattern; + } + + public Map getVariableDefinitions() { + return variableDefinitions; + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitPatternRecognition(this, context); + } + + @Override + public PlanNode replaceChildren(List newChildren) { + return new PatternRecognitionNode( + id, + Iterables.getOnlyElement(newChildren), + partitionBy, + orderingScheme, + hashSymbol, + measures, + rowsPerMatch, + skipToLabels, + skipToPosition, + pattern, + variableDefinitions); + } + + @Override + public PlanNode clone() { + return new PatternRecognitionNode( + id, + null, + partitionBy, + orderingScheme, + hashSymbol, + measures, + rowsPerMatch, + skipToLabels, + skipToPosition, + pattern, + variableDefinitions); + } + + @Override + public List getOutputColumnNames() { + return null; + } + + @Override + protected void serializeAttributes(ByteBuffer byteBuffer) { + PlanNodeType.TABLE_PATTERN_RECOGNITION_NODE.serialize(byteBuffer); + + ReadWriteIOUtils.write(partitionBy.size(), byteBuffer); + for (Symbol symbol : partitionBy) { + Symbol.serialize(symbol, byteBuffer); + } + + if (orderingScheme.isPresent()) { + ReadWriteIOUtils.write(true, byteBuffer); + orderingScheme.get().serialize(byteBuffer); + } else { + ReadWriteIOUtils.write(false, byteBuffer); + } + + if (hashSymbol.isPresent()) { + ReadWriteIOUtils.write(true, byteBuffer); + Symbol.serialize(hashSymbol.get(), byteBuffer); + } else { + ReadWriteIOUtils.write(false, byteBuffer); + } + + ReadWriteIOUtils.write(measures.size(), byteBuffer); + for (Map.Entry entry : measures.entrySet()) { + Symbol.serialize(entry.getKey(), byteBuffer); + Measure.serialize(entry.getValue(), byteBuffer); + } + + RowsPerMatch.serialize(rowsPerMatch, byteBuffer); + + ReadWriteIOUtils.write(skipToLabels.size(), byteBuffer); + for (IrLabel label : skipToLabels) { + IrLabel.serialize(label, byteBuffer); + } + + SkipToPosition.serialize(skipToPosition, byteBuffer); + + IrRowPattern.serialize(pattern, byteBuffer); + + ReadWriteIOUtils.write(variableDefinitions.size(), byteBuffer); + for (Map.Entry entry : variableDefinitions.entrySet()) { + IrLabel.serialize(entry.getKey(), byteBuffer); + ExpressionAndValuePointers.serialize(entry.getValue(), byteBuffer); + } + } + + @Override + protected void serializeAttributes(DataOutputStream stream) throws IOException { + PlanNodeType.TABLE_PATTERN_RECOGNITION_NODE.serialize(stream); + + ReadWriteIOUtils.write(partitionBy.size(), stream); + for (Symbol symbol : partitionBy) { + Symbol.serialize(symbol, stream); + } + + if (orderingScheme.isPresent()) { + ReadWriteIOUtils.write(true, stream); + orderingScheme.get().serialize(stream); + } else { + ReadWriteIOUtils.write(false, stream); + } + + if (hashSymbol.isPresent()) { + ReadWriteIOUtils.write(true, stream); + Symbol.serialize(hashSymbol.get(), stream); + } else { + ReadWriteIOUtils.write(false, stream); + } + + ReadWriteIOUtils.write(measures.size(), stream); + for (Map.Entry entry : measures.entrySet()) { + Symbol.serialize(entry.getKey(), stream); + Measure.serialize(entry.getValue(), stream); + } + + RowsPerMatch.serialize(rowsPerMatch, stream); + + ReadWriteIOUtils.write(skipToLabels.size(), stream); + for (IrLabel label : skipToLabels) { + IrLabel.serialize(label, stream); + } + + SkipToPosition.serialize(skipToPosition, stream); + + IrRowPattern.serialize(pattern, stream); + + ReadWriteIOUtils.write(variableDefinitions.size(), stream); + for (Map.Entry entry : variableDefinitions.entrySet()) { + IrLabel.serialize(entry.getKey(), stream); + ExpressionAndValuePointers.serialize(entry.getValue(), stream); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (!super.equals(o)) return false; + PatternRecognitionNode that = (PatternRecognitionNode) o; + return Objects.equals(partitionBy, that.partitionBy) + && Objects.equals(orderingScheme, that.orderingScheme) + && Objects.equals(hashSymbol, that.hashSymbol) + && Objects.equals(measures, that.measures) + && rowsPerMatch == that.rowsPerMatch + && Objects.equals(skipToLabels, that.skipToLabels) + && skipToPosition == that.skipToPosition + && Objects.equals(pattern, that.pattern) + && Objects.equals(variableDefinitions, that.variableDefinitions); + } + + @Override + public int hashCode() { + return Objects.hash( + super.hashCode(), + partitionBy, + orderingScheme, + hashSymbol, + measures, + rowsPerMatch, + skipToLabels, + skipToPosition, + pattern, + variableDefinitions); + } + + @Override + public String toString() { + return "PatternRecognitionNode-" + this.getPlanNodeId(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java index f0056e024971..dee755f2623e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/Patterns.java @@ -179,6 +179,10 @@ public static Pattern topK() { return typeOf(TopKNode.class); } + public static Pattern patternRecognition() { + return typeOf(PatternRecognitionNode.class); + } + /*public static Pattern tableWriterNode() { return typeOf(TableWriterNode.class); @@ -209,11 +213,6 @@ public static Pattern window() return typeOf(WindowNode.class); } - public static Pattern patternRecognition() - { - return typeOf(PatternRecognitionNode.class); - } - public static Pattern tableFunction() { return typeOf(TableFunctionNode.class); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/RowsPerMatch.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/RowsPerMatch.java new file mode 100644 index 000000000000..aa92f1d665dc --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/RowsPerMatch.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public enum RowsPerMatch { + ONE { + @Override + public boolean isOneRow() { + return true; + } + + @Override + public boolean isEmptyMatches() { + return true; + } + + @Override + public boolean isUnmatchedRows() { + return false; + } + }, + + // ALL_SHOW_EMPTY option applies to the MATCH_RECOGNIZE clause. + // Output all rows of every match, including empty matches. + // In the case of an empty match, output the starting row of the match attempt. + // Do not produce output for the rows matched within exclusion `{- ... -}`. + ALL_SHOW_EMPTY { + @Override + public boolean isOneRow() { + return false; + } + + @Override + public boolean isEmptyMatches() { + return true; + } + + @Override + public boolean isUnmatchedRows() { + return false; + } + }, + + // ALL_OMIT_EMPTY option applies to the MATCH_RECOGNIZE clause. + // Output all rows of every non-empty match. + // Do not produce output for the rows matched within exclusion `{- ... -}` + ALL_OMIT_EMPTY { + @Override + public boolean isOneRow() { + return false; + } + + @Override + public boolean isEmptyMatches() { + return false; + } + + @Override + public boolean isUnmatchedRows() { + return false; + } + }, + + // ALL_WITH_UNMATCHED option applies to the MATCH_RECOGNIZE clause. + // Output all rows of every match, including empty matches. + // Produce an additional output row for every unmatched row. + // Pattern exclusions are not allowed with this option. + ALL_WITH_UNMATCHED { + @Override + public boolean isOneRow() { + return false; + } + + @Override + public boolean isEmptyMatches() { + return true; + } + + @Override + public boolean isUnmatchedRows() { + return true; + } + }; + + public abstract boolean isOneRow(); + + public abstract boolean isEmptyMatches(); + + public abstract boolean isUnmatchedRows(); + + public static void serialize(RowsPerMatch rowsPerMatch, ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(rowsPerMatch.ordinal(), byteBuffer); + } + + public static void serialize(RowsPerMatch rowsPerMatch, DataOutputStream stream) + throws IOException { + ReadWriteIOUtils.write(rowsPerMatch.ordinal(), stream); + } + + public static RowsPerMatch deserialize(ByteBuffer byteBuffer) { + int ordinal = ReadWriteIOUtils.readInt(byteBuffer); + return RowsPerMatch.values()[ordinal]; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SkipToPosition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SkipToPosition.java new file mode 100644 index 000000000000..cf610f66f344 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/node/SkipToPosition.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.node; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public enum SkipToPosition { + PAST_LAST, + NEXT, + FIRST, + LAST; + + public static void serialize(SkipToPosition position, ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(position.ordinal(), byteBuffer); + } + + public static void serialize(SkipToPosition position, DataOutputStream stream) + throws IOException { + ReadWriteIOUtils.write(position.ordinal(), stream); + } + + public static SkipToPosition deserialize(ByteBuffer byteBuffer) { + int ordinal = ReadWriteIOUtils.readInt(byteBuffer); + return SkipToPosition.values()[ordinal]; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java index 66e34f269385..08a3ca09a33b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SymbolMapper.java @@ -30,7 +30,15 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ApplyNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LimitNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.Measure; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PatternRecognitionNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TopKNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.ClassifierValuePointer; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.ExpressionAndValuePointers; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrLabel; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.MatchNumberValuePointer; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.ScalarValuePointer; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.ValuePointer; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SymbolReference; @@ -242,6 +250,67 @@ public TopKNode map(TopKNode node, List source, PlanNodeId nodeId) { node.isChildrenDataInOrder()); } + public PatternRecognitionNode map(PatternRecognitionNode node, PlanNode source) { + ImmutableMap.Builder newMeasures = ImmutableMap.builder(); + node.getMeasures() + .forEach( + (symbol, measure) -> { + ExpressionAndValuePointers newExpression = + map(measure.getExpressionAndValuePointers()); + newMeasures.put(map(symbol), new Measure(newExpression, measure.getType())); + }); + + ImmutableMap.Builder newVariableDefinitions = + ImmutableMap.builder(); + node.getVariableDefinitions() + .forEach((label, expression) -> newVariableDefinitions.put(label, map(expression))); + + return new PatternRecognitionNode( + node.getPlanNodeId(), + source, + mapAndDistinct(node.getPartitionBy()), + node.getOrderingScheme(), + node.getHashSymbol().map(this::map), + newMeasures.buildOrThrow(), + node.getRowsPerMatch(), + node.getSkipToLabels(), + node.getSkipToPosition(), + node.getPattern(), + newVariableDefinitions.buildOrThrow()); + } + + private ExpressionAndValuePointers map(ExpressionAndValuePointers expressionAndValuePointers) { + // Map only the input symbols of ValuePointers. These are the symbols produced by the source + // node. + // Other symbols present in the ExpressionAndValuePointers structure are synthetic unique + // symbols + // with no outer usage or dependencies. + ImmutableList.Builder newAssignments = + ImmutableList.builder(); + for (ExpressionAndValuePointers.Assignment assignment : + expressionAndValuePointers.getAssignments()) { + ValuePointer newPointer; + if (assignment.getValuePointer() instanceof ClassifierValuePointer) { + newPointer = (ClassifierValuePointer) assignment.getValuePointer(); + } else if (assignment.getValuePointer() instanceof MatchNumberValuePointer) { + newPointer = (MatchNumberValuePointer) assignment.getValuePointer(); + } else if (assignment.getValuePointer() instanceof ScalarValuePointer) { + ScalarValuePointer pointer = (ScalarValuePointer) assignment.getValuePointer(); + newPointer = + new ScalarValuePointer(pointer.getLogicalIndexPointer(), map(pointer.getInputSymbol())); + } else { + throw new IllegalArgumentException( + "Unsupported ValuePointer type: " + assignment.getValuePointer().getClass().getName()); + } + + newAssignments.add( + new ExpressionAndValuePointers.Assignment(assignment.getSymbol(), newPointer)); + } + + return new ExpressionAndValuePointers( + expressionAndValuePointers.getExpression(), newAssignments.build()); + } + public static Builder builder() { return new Builder(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java index 348eb7823ee0..6d5c35fefe81 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/UnaliasSymbolReferences.java @@ -43,6 +43,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.LinearFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OffsetNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.OutputNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PatternRecognitionNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.PreviousFillNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ProjectNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.SortNode; @@ -693,6 +694,19 @@ public PlanAndMappings visitJoin(JoinNode node, UnaliasContext context) { node.isSpillable()), outputMapping); } + + @Override + public PlanAndMappings visitPatternRecognition( + PatternRecognitionNode node, UnaliasContext context) { + PlanAndMappings rewrittenSource = node.getChild().accept(this, context); + Map mapping = new HashMap<>(rewrittenSource.getMappings()); + SymbolMapper mapper = symbolMapper(mapping); + + PatternRecognitionNode rewrittenPatternRecognition = + mapper.map(node, rewrittenSource.getRoot()); + + return new PlanAndMappings(rewrittenPatternRecognition, mapping); + } } private static class UnaliasContext { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ClassifierValuePointer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ClassifierValuePointer.java new file mode 100644 index 000000000000..a06d3b32f4f1 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ClassifierValuePointer.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public final class ClassifierValuePointer implements ValuePointer { + private final LogicalIndexPointer logicalIndexPointer; + + public ClassifierValuePointer(LogicalIndexPointer logicalIndexPointer) { + this.logicalIndexPointer = requireNonNull(logicalIndexPointer, "logicalIndexPointer is null"); + } + + public LogicalIndexPointer getLogicalIndexPointer() { + return logicalIndexPointer; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ClassifierValuePointer that = (ClassifierValuePointer) o; + return Objects.equals(logicalIndexPointer, that.logicalIndexPointer); + } + + @Override + public int hashCode() { + return Objects.hash(logicalIndexPointer); + } + + public static void serialize(ClassifierValuePointer pointer, ByteBuffer byteBuffer) { + LogicalIndexPointer.serialize(pointer.logicalIndexPointer, byteBuffer); + } + + public static void serialize(ClassifierValuePointer pointer, DataOutputStream stream) + throws IOException { + LogicalIndexPointer.serialize(pointer.logicalIndexPointer, stream); + } + + public static ClassifierValuePointer deserialize(ByteBuffer byteBuffer) { + LogicalIndexPointer logicalIndexPointer = LogicalIndexPointer.deserialize(byteBuffer); + return new ClassifierValuePointer(logicalIndexPointer); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ExpressionAndValuePointers.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ExpressionAndValuePointers.java new file mode 100644 index 000000000000..2b93cab83e5d --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ExpressionAndValuePointers.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression; + +import com.google.common.collect.ImmutableList; +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.BooleanLiteral.TRUE_LITERAL; + +public class ExpressionAndValuePointers { + public static final ExpressionAndValuePointers TRUE = + new ExpressionAndValuePointers(TRUE_LITERAL, ImmutableList.of()); + + private final Expression expression; + private final List assignments; + + public ExpressionAndValuePointers(Expression expression, List assignments) { + this.expression = requireNonNull(expression, "expression is null"); + this.assignments = ImmutableList.copyOf(assignments); + } + + public Expression getExpression() { + return expression; + } + + public List getAssignments() { + return assignments; + } + + public List getInputSymbols() { + Set localInputs = + assignments.stream() + .filter( + assignment -> + assignment.getValuePointer() instanceof ClassifierValuePointer + || assignment.getValuePointer() instanceof MatchNumberValuePointer) + .map(Assignment::getSymbol) + .collect(toImmutableSet()); + + ImmutableList.Builder inputSymbols = ImmutableList.builder(); + for (Assignment assignment : assignments) { + ValuePointer valuePointer = assignment.getValuePointer(); + + if (valuePointer instanceof ScalarValuePointer) { + ScalarValuePointer pointer = (ScalarValuePointer) valuePointer; + Symbol symbol = pointer.getInputSymbol(); + if (!localInputs.contains(symbol)) { + inputSymbols.add(symbol); + } + } + // TODO: need to support Aggregation later + // else if (valuePointer instanceof AggregationValuePointer) { + // AggregationValuePointer pointer = (AggregationValuePointer) valuePointer; + // inputSymbols.addAll( + // pointer.getInputSymbols().stream() + // .filter(symbol -> !localInputs.contains(symbol)) + // .collect(Collectors.toList())); + // } + } + + return inputSymbols.build(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + ExpressionAndValuePointers o = (ExpressionAndValuePointers) obj; + return Objects.equals(expression, o.expression) && Objects.equals(assignments, o.assignments); + } + + @Override + public int hashCode() { + return Objects.hash(expression, assignments); + } + + public static void serialize(ExpressionAndValuePointers valuePointers, ByteBuffer byteBuffer) { + Expression.serialize(valuePointers.expression, byteBuffer); + ReadWriteIOUtils.write(valuePointers.assignments.size(), byteBuffer); + for (Assignment assignment : valuePointers.assignments) { + Assignment.serialize(assignment, byteBuffer); + } + } + + public static void serialize(ExpressionAndValuePointers valuePointers, DataOutputStream stream) + throws IOException { + Expression.serialize(valuePointers.expression, stream); + ReadWriteIOUtils.write(valuePointers.assignments.size(), stream); + for (Assignment assignment : valuePointers.assignments) { + Assignment.serialize(assignment, stream); + } + } + + public static ExpressionAndValuePointers deserialize(ByteBuffer byteBuffer) { + Expression expression = Expression.deserialize(byteBuffer); + int assignmentsSize = ReadWriteIOUtils.readInt(byteBuffer); + List assignments = new ArrayList<>(assignmentsSize); + for (int i = 0; i < assignmentsSize; i++) { + assignments.add(Assignment.deserialize(byteBuffer)); + } + return new ExpressionAndValuePointers(expression, assignments); + } + + public static class Assignment { + private final Symbol symbol; + private final ValuePointer valuePointer; + + public Assignment(Symbol symbol, ValuePointer valuePointer) { + this.symbol = symbol; + this.valuePointer = valuePointer; + } + + public Symbol getSymbol() { + return symbol; + } + + public ValuePointer getValuePointer() { + return valuePointer; + } + + @Override + public int hashCode() { + return Objects.hash(symbol, valuePointer); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Assignment that = (Assignment) o; + return Objects.equals(symbol, that.symbol) && Objects.equals(valuePointer, that.valuePointer); + } + + @Override + public String toString() { + return "Assignment{" + "symbol=" + symbol + ", valuePointer=" + valuePointer + '}'; + } + + public static void serialize(Assignment assignment, ByteBuffer byteBuffer) { + Symbol.serialize(assignment.symbol, byteBuffer); + + if (assignment.valuePointer instanceof MatchNumberValuePointer) { + ReadWriteIOUtils.write(0, byteBuffer); + } else if (assignment.valuePointer instanceof ClassifierValuePointer) { + ReadWriteIOUtils.write(1, byteBuffer); + } else if (assignment.valuePointer instanceof ScalarValuePointer) { + ReadWriteIOUtils.write(2, byteBuffer); + } else { + throw new IllegalArgumentException("Unknown ValuePointer type"); + } + + if (assignment.valuePointer instanceof MatchNumberValuePointer) { + MatchNumberValuePointer.serialize( + (MatchNumberValuePointer) assignment.valuePointer, byteBuffer); + } else if (assignment.valuePointer instanceof ClassifierValuePointer) { + ClassifierValuePointer.serialize( + (ClassifierValuePointer) assignment.valuePointer, byteBuffer); + } else if (assignment.valuePointer instanceof ScalarValuePointer) { + ScalarValuePointer.serialize((ScalarValuePointer) assignment.valuePointer, byteBuffer); + } + } + + public static void serialize(Assignment assignment, DataOutputStream stream) + throws IOException { + Symbol.serialize(assignment.symbol, stream); + + if (assignment.valuePointer instanceof MatchNumberValuePointer) { + ReadWriteIOUtils.write(0, stream); + } else if (assignment.valuePointer instanceof ClassifierValuePointer) { + ReadWriteIOUtils.write(1, stream); + } else if (assignment.valuePointer instanceof ScalarValuePointer) { + ReadWriteIOUtils.write(2, stream); + } else { + throw new IllegalArgumentException("Unknown ValuePointer type"); + } + + if (assignment.valuePointer instanceof MatchNumberValuePointer) { + MatchNumberValuePointer.serialize( + (MatchNumberValuePointer) assignment.valuePointer, stream); + } else if (assignment.valuePointer instanceof ClassifierValuePointer) { + ClassifierValuePointer.serialize((ClassifierValuePointer) assignment.valuePointer, stream); + } else if (assignment.valuePointer instanceof ScalarValuePointer) { + ScalarValuePointer.serialize((ScalarValuePointer) assignment.valuePointer, stream); + } + } + + public static Assignment deserialize(ByteBuffer byteBuffer) { + Symbol symbol = Symbol.deserialize(byteBuffer); + + int type = ReadWriteIOUtils.readInt(byteBuffer); + ValuePointer valuePointer; + + if (type == 0) { + valuePointer = MatchNumberValuePointer.deserialize(byteBuffer); + } else if (type == 1) { + valuePointer = ClassifierValuePointer.deserialize(byteBuffer); + } else if (type == 2) { + valuePointer = ScalarValuePointer.deserialize(byteBuffer); + } else { + throw new IllegalArgumentException("Unknown ValuePointer type"); + } + + return new Assignment(symbol, valuePointer); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrAlternation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrAlternation.java new file mode 100644 index 000000000000..f698b923f2a5 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrAlternation.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.joining; + +public class IrAlternation extends IrRowPattern { + private final List patterns; + + public IrAlternation(List patterns) { + this.patterns = requireNonNull(patterns, "patterns is null"); + checkArgument( + patterns.size() >= 2, + "pattern alternation must have at least 2 elements (actual: %s)", + patterns.size()); + } + + public List getPatterns() { + return patterns; + } + + @Override + public R accept(IrRowPatternVisitor visitor, C context) { + return visitor.visitIrAlternation(this, context); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + IrAlternation o = (IrAlternation) obj; + return Objects.equals(patterns, o.patterns); + } + + @Override + public int hashCode() { + return Objects.hash(patterns); + } + + @Override + public String toString() { + return patterns.stream().map(Object::toString).collect(joining(" | ", "(", ")")); + } + + public static void serialize(IrAlternation pattern, ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(pattern.patterns.size(), byteBuffer); + for (IrRowPattern subPattern : pattern.patterns) { + IrRowPattern.serialize(subPattern, byteBuffer); + } + } + + public static void serialize(IrAlternation pattern, DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(pattern.patterns.size(), stream); + for (IrRowPattern subPattern : pattern.patterns) { + IrRowPattern.serialize(subPattern, stream); + } + } + + public static IrAlternation deserialize(ByteBuffer byteBuffer) { + int size = ReadWriteIOUtils.readInt(byteBuffer); + List patterns = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + patterns.add(IrRowPattern.deserialize(byteBuffer)); + } + return new IrAlternation(patterns); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrAnchor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrAnchor.java new file mode 100644 index 000000000000..5c6651523582 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrAnchor.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrAnchor.Type.PARTITION_START; + +public class IrAnchor extends IrRowPattern { + public enum Type { + PARTITION_START, + PARTITION_END + } + + private final Type type; + + public IrAnchor(Type type) { + this.type = requireNonNull(type, "type is null"); + } + + public Type getType() { + return type; + } + + @Override + public R accept(IrRowPatternVisitor visitor, C context) { + return visitor.visitIrAnchor(this, context); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + IrAnchor o = (IrAnchor) obj; + return Objects.equals(type, o.type); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } + + @Override + public String toString() { + return type == PARTITION_START ? "^" : "$"; + } + + public static void serialize(IrAnchor pattern, ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(pattern.type.ordinal(), byteBuffer); + } + + public static void serialize(IrAnchor pattern, DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(pattern.type.ordinal(), stream); + } + + public static IrAnchor deserialize(ByteBuffer byteBuffer) { + Type type = Type.values()[ReadWriteIOUtils.readInt(byteBuffer)]; + return new IrAnchor(type); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrConcatenation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrConcatenation.java new file mode 100644 index 000000000000..e846ce2a540f --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrConcatenation.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.joining; + +public class IrConcatenation extends IrRowPattern { + private final List patterns; + + public IrConcatenation(List patterns) { + this.patterns = requireNonNull(patterns, "patterns is null"); + checkArgument( + patterns.size() >= 2, + "pattern concatenation must have at least 2 elements (actual: %s)", + patterns.size()); + } + + public List getPatterns() { + return patterns; + } + + @Override + public R accept(IrRowPatternVisitor visitor, C context) { + return visitor.visitIrConcatenation(this, context); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + IrConcatenation o = (IrConcatenation) obj; + return Objects.equals(patterns, o.patterns); + } + + @Override + public int hashCode() { + return Objects.hash(patterns); + } + + @Override + public String toString() { + return patterns.stream().map(Object::toString).collect(joining(" ", "(", ")")); + } + + public static void serialize(IrConcatenation pattern, ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(pattern.patterns.size(), byteBuffer); + for (IrRowPattern subPattern : pattern.patterns) { + IrRowPattern.serialize(subPattern, byteBuffer); + } + } + + public static void serialize(IrConcatenation pattern, DataOutputStream stream) + throws IOException { + ReadWriteIOUtils.write(pattern.patterns.size(), stream); + for (IrRowPattern subPattern : pattern.patterns) { + IrRowPattern.serialize(subPattern, stream); + } + } + + public static IrConcatenation deserialize(ByteBuffer byteBuffer) { + int size = ReadWriteIOUtils.readInt(byteBuffer); + List patterns = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + patterns.add(IrRowPattern.deserialize(byteBuffer)); + } + return new IrConcatenation(patterns); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrEmpty.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrEmpty.java new file mode 100644 index 000000000000..ba1c4143c29e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrEmpty.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class IrEmpty extends IrRowPattern { + public IrEmpty() {} + + @Override + public R accept(IrRowPatternVisitor visitor, C context) { + return visitor.visitIrEmpty(this, context); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + return true; + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } + + @Override + public String toString() { + return "()"; + } + + public static void serialize(IrEmpty pattern, ByteBuffer byteBuffer) {} + + public static void serialize(IrEmpty pattern, DataOutputStream stream) throws IOException {} + + public static IrEmpty deserialize(ByteBuffer byteBuffer) { + return new IrEmpty(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrExclusion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrExclusion.java new file mode 100644 index 000000000000..abd784d147be --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrExclusion.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class IrExclusion extends IrRowPattern { + private final IrRowPattern pattern; + + public IrExclusion(IrRowPattern pattern) { + this.pattern = requireNonNull(pattern, "pattern is null"); + } + + public IrRowPattern getPattern() { + return pattern; + } + + @Override + public R accept(IrRowPatternVisitor visitor, C context) { + return visitor.visitIrExclusion(this, context); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + IrExclusion o = (IrExclusion) obj; + return Objects.equals(pattern, o.pattern); + } + + @Override + public int hashCode() { + return Objects.hash(pattern); + } + + @Override + public String toString() { + return "{-" + pattern + "-}"; + } + + public static void serialize(IrExclusion pattern, ByteBuffer byteBuffer) { + IrRowPattern.serialize(pattern.pattern, byteBuffer); + } + + public static void serialize(IrExclusion pattern, DataOutputStream stream) throws IOException { + IrRowPattern.serialize(pattern.pattern, stream); + } + + public static IrExclusion deserialize(ByteBuffer byteBuffer) { + IrRowPattern pattern = IrRowPattern.deserialize(byteBuffer); + return new IrExclusion(pattern); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrLabel.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrLabel.java new file mode 100644 index 000000000000..35fb1b10a454 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrLabel.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class IrLabel extends IrRowPattern { + private final String name; + + /** + * Create IrLabel with given name. The name has to be in the canonical form with respect to SQL + * identifier semantics. + */ + public IrLabel(String name) { + this.name = requireNonNull(name, "name is null"); + } + + public String getName() { + return name; + } + + @Override + public R accept(IrRowPatternVisitor visitor, C context) { + return visitor.visitIrLabel(this, context); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + IrLabel o = (IrLabel) obj; + return Objects.equals(name, o.name); + } + + @Override + public int hashCode() { + return Objects.hash(name); + } + + @Override + public String toString() { + return name; + } + + public static void serialize(IrLabel label, ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(label.name, byteBuffer); + } + + public static void serialize(IrLabel label, DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(label.name, stream); + } + + public static IrLabel deserialize(ByteBuffer byteBuffer) { + String name = ReadWriteIOUtils.readString(byteBuffer); + return new IrLabel(name); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrPatternAlternationOptimizer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrPatternAlternationOptimizer.java new file mode 100644 index 000000000000..ccb4e9a89f3e --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrPatternAlternationOptimizer.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import com.google.common.collect.ImmutableList; + +import java.util.List; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrQuantifier.zeroOrOne; + +/** + * Remove empty pattern from pattern alternation and replace it with quantification of a + * neighbouring term. + */ +public final class IrPatternAlternationOptimizer { + private IrPatternAlternationOptimizer() {} + + public static IrRowPattern optimize(IrRowPattern node) { + return new Visitor().process(node); + } + + private static class Visitor extends IrRowPatternVisitor { + @Override + protected IrRowPattern visitIrRowPattern(IrRowPattern node, Void context) { + throw new UnsupportedOperationException( + "unsupported node type: " + node.getClass().getName()); + } + + @Override + protected IrRowPattern visitIrLabel(IrLabel node, Void context) { + return node; + } + + @Override + protected IrRowPattern visitIrAnchor(IrAnchor node, Void context) { + return node; + } + + @Override + protected IrRowPattern visitIrEmpty(IrEmpty node, Void context) { + return node; + } + + @Override + protected IrRowPattern visitIrExclusion(IrExclusion node, Void context) { + IrRowPattern child = process(node.getPattern()); + + return new IrExclusion(child); + } + + @Override + protected IrRowPattern visitIrAlternation(IrAlternation node, Void context) { + List children = + node.getPatterns().stream().map(this::process).collect(toImmutableList()); + + int emptyChildIndex = -1; + for (int i = 0; i < children.size(); i++) { + if (children.get(i) instanceof IrEmpty) { + checkState( + emptyChildIndex < 0, + "run IrRowPatternFlattener first to remove redundant empty pattern"); + emptyChildIndex = i; + } + } + + if (emptyChildIndex < 0) { + return new IrAlternation(children); + } + + // remove the empty child: + // (() | A) -> A?? + // (() | A | B) -> (A?? | B) + if (emptyChildIndex == 0) { + IrRowPattern child = new IrQuantified(children.get(1), zeroOrOne(false)); + if (children.size() == 2) { + return child; + } + ImmutableList.Builder builder = + ImmutableList.builder() + .add(child) + .addAll(children.subList(2, children.size())); + return new IrAlternation(builder.build()); + } + // (A | ()) -> A? + // (A | B | () | C) -> (A | B? | C) + children = + ImmutableList.builder() + .addAll(children.subList(0, emptyChildIndex - 1)) + .add(new IrQuantified(children.get(emptyChildIndex - 1), zeroOrOne(true))) + .addAll(children.subList(emptyChildIndex + 1, children.size())) + .build(); + + if (children.size() == 1) { + return children.get(0); + } + return new IrAlternation(children); + } + + @Override + protected IrRowPattern visitIrConcatenation(IrConcatenation node, Void context) { + List children = + node.getPatterns().stream().map(this::process).collect(toImmutableList()); + + return new IrConcatenation(children); + } + + @Override + protected IrRowPattern visitIrPermutation(IrPermutation node, Void context) { + List children = + node.getPatterns().stream().map(this::process).collect(toImmutableList()); + + return new IrPermutation(children); + } + + @Override + protected IrRowPattern visitIrQuantified(IrQuantified node, Void context) { + IrRowPattern child = process(node.getPattern()); + + return new IrQuantified(child, node.getQuantifier()); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrPermutation.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrPermutation.java new file mode 100644 index 000000000000..5367f9dd026a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrPermutation.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import static com.google.common.base.Preconditions.checkArgument; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.joining; + +public class IrPermutation extends IrRowPattern { + private final List patterns; + + public IrPermutation(List patterns) { + this.patterns = requireNonNull(patterns, "patterns is null"); + checkArgument(!patterns.isEmpty(), "patterns list is empty"); + } + + public List getPatterns() { + return patterns; + } + + @Override + public R accept(IrRowPatternVisitor visitor, C context) { + return visitor.visitIrPermutation(this, context); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + IrPermutation o = (IrPermutation) obj; + return Objects.equals(patterns, o.patterns); + } + + @Override + public int hashCode() { + return Objects.hash(patterns); + } + + @Override + public String toString() { + return patterns.stream().map(Object::toString).collect(joining(", ", "PERMUTE(", ")")); + } + + public static void serialize(IrPermutation pattern, ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(pattern.patterns.size(), byteBuffer); + for (IrRowPattern subPattern : pattern.patterns) { + IrRowPattern.serialize(subPattern, byteBuffer); + } + } + + public static void serialize(IrPermutation pattern, DataOutputStream stream) throws IOException { + ReadWriteIOUtils.write(pattern.patterns.size(), stream); + for (IrRowPattern subPattern : pattern.patterns) { + IrRowPattern.serialize(subPattern, stream); + } + } + + public static IrPermutation deserialize(ByteBuffer byteBuffer) { + int size = ReadWriteIOUtils.readInt(byteBuffer); + List patterns = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + patterns.add(IrRowPattern.deserialize(byteBuffer)); + } + return new IrPermutation(patterns); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrQuantified.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrQuantified.java new file mode 100644 index 000000000000..6031bf643025 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrQuantified.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public class IrQuantified extends IrRowPattern { + private final IrRowPattern pattern; + private final IrQuantifier quantifier; + + public IrQuantified(IrRowPattern pattern, IrQuantifier quantifier) { + this.pattern = requireNonNull(pattern, "pattern is null"); + this.quantifier = requireNonNull(quantifier, "quantifier is null"); + } + + public IrRowPattern getPattern() { + return pattern; + } + + public IrQuantifier getQuantifier() { + return quantifier; + } + + @Override + public R accept(IrRowPatternVisitor visitor, C context) { + return visitor.visitIrQuantified(this, context); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + IrQuantified o = (IrQuantified) obj; + return Objects.equals(pattern, o.pattern) && Objects.equals(quantifier, o.quantifier); + } + + @Override + public int hashCode() { + return Objects.hash(pattern, quantifier); + } + + @Override + public String toString() { + return pattern.toString() + quantifier; + } + + public static void serialize(IrQuantified pattern, ByteBuffer byteBuffer) { + IrRowPattern.serialize(pattern.pattern, byteBuffer); + IrQuantifier.serialize(pattern.quantifier, byteBuffer); + } + + public static void serialize(IrQuantified pattern, DataOutputStream stream) throws IOException { + IrRowPattern.serialize(pattern.pattern, stream); + IrQuantifier.serialize(pattern.quantifier, stream); + } + + public static IrQuantified deserialize(ByteBuffer byteBuffer) { + IrRowPattern pattern = IrRowPattern.deserialize(byteBuffer); + IrQuantifier quantifier = IrQuantifier.deserialize(byteBuffer); + return new IrQuantified(pattern, quantifier); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrQuantifier.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrQuantifier.java new file mode 100644 index 000000000000..6499147971f2 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrQuantifier.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; +import java.util.Optional; + +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public class IrQuantifier { + private final int atLeast; + private final Optional atMost; + private final boolean greedy; + + public static IrQuantifier zeroOrMore(boolean greedy) { + return new IrQuantifier(0, java.util.Optional.empty(), greedy); + } + + public static IrQuantifier oneOrMore(boolean greedy) { + return new IrQuantifier(1, java.util.Optional.empty(), greedy); + } + + public static IrQuantifier zeroOrOne(boolean greedy) { + return new IrQuantifier(0, Optional.of(1), greedy); + } + + public static IrQuantifier range( + Optional atLeast, Optional atMost, boolean greedy) { + return new IrQuantifier(atLeast.orElse(0), atMost, greedy); + } + + public IrQuantifier(int atLeast, Optional atMost, boolean greedy) { + this.atLeast = atLeast; + this.atMost = requireNonNull(atMost, "atMost is null"); + this.greedy = greedy; + } + + public int getAtLeast() { + return atLeast; + } + + public Optional getAtMost() { + return atMost; + } + + public boolean isGreedy() { + return greedy; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + IrQuantifier o = (IrQuantifier) obj; + return atLeast == o.atLeast && Objects.equals(atMost, o.atMost) && greedy == o.greedy; + } + + @Override + public int hashCode() { + return Objects.hash(atLeast, atMost, greedy); + } + + @Override + public String toString() { + return format("{%s, %s}", atLeast, atMost.map(Object::toString).orElse("∞")); + } + + public static void serialize(IrQuantifier quantifier, ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(quantifier.atLeast, byteBuffer); + if (quantifier.atMost.isPresent()) { + ReadWriteIOUtils.write(true, byteBuffer); + ReadWriteIOUtils.write(quantifier.atMost.get(), byteBuffer); + } else { + ReadWriteIOUtils.write(false, byteBuffer); + } + ReadWriteIOUtils.write(quantifier.greedy, byteBuffer); + } + + public static void serialize(IrQuantifier quantifier, DataOutputStream stream) + throws IOException { + ReadWriteIOUtils.write(quantifier.atLeast, stream); + if (quantifier.atMost.isPresent()) { + ReadWriteIOUtils.write(true, stream); + ReadWriteIOUtils.write(quantifier.atMost.get(), stream); + } else { + ReadWriteIOUtils.write(false, stream); + } + ReadWriteIOUtils.write(quantifier.greedy, stream); + } + + public static IrQuantifier deserialize(ByteBuffer byteBuffer) { + int atLeast = ReadWriteIOUtils.readInt(byteBuffer); + boolean hasAtMost = ReadWriteIOUtils.readBoolean(byteBuffer); + Optional atMost = + hasAtMost ? Optional.of(ReadWriteIOUtils.readInt(byteBuffer)) : java.util.Optional.empty(); + boolean greedy = ReadWriteIOUtils.readBoolean(byteBuffer); + return new IrQuantifier(atLeast, atMost, greedy); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrRowPattern.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrRowPattern.java new file mode 100644 index 000000000000..dcf5fcdf2c85 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrRowPattern.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public abstract class IrRowPattern { + protected R accept(IrRowPatternVisitor visitor, C context) { + return visitor.visitIrRowPattern(this, context); + } + + public static void serialize(IrRowPattern pattern, ByteBuffer byteBuffer) { + if (pattern instanceof IrAlternation) { + ReadWriteIOUtils.write(0, byteBuffer); // Type marker for IrAlternation + IrAlternation.serialize((IrAlternation) pattern, byteBuffer); + } else if (pattern instanceof IrAnchor) { + ReadWriteIOUtils.write(1, byteBuffer); // Type marker for IrAnchor + IrAnchor.serialize((IrAnchor) pattern, byteBuffer); + } else if (pattern instanceof IrConcatenation) { + ReadWriteIOUtils.write(2, byteBuffer); // Type marker for IrConcatenation + IrConcatenation.serialize((IrConcatenation) pattern, byteBuffer); + } else if (pattern instanceof IrEmpty) { + ReadWriteIOUtils.write(3, byteBuffer); // Type marker for IrEmpty + IrEmpty.serialize((IrEmpty) pattern, byteBuffer); + } else if (pattern instanceof IrExclusion) { + ReadWriteIOUtils.write(4, byteBuffer); // Type marker for IrExclusion + IrExclusion.serialize((IrExclusion) pattern, byteBuffer); + } else if (pattern instanceof IrLabel) { + ReadWriteIOUtils.write(5, byteBuffer); // Type marker for IrLabel + IrLabel.serialize((IrLabel) pattern, byteBuffer); + } else if (pattern instanceof IrPermutation) { + ReadWriteIOUtils.write(6, byteBuffer); // Type marker for IrPermutation + IrPermutation.serialize((IrPermutation) pattern, byteBuffer); + } else if (pattern instanceof IrQuantified) { + ReadWriteIOUtils.write(7, byteBuffer); // Type marker for IrQuantified + IrQuantified.serialize((IrQuantified) pattern, byteBuffer); + } else { + throw new IllegalArgumentException("Unknown IrRowPattern type"); + } + } + + public static void serialize(IrRowPattern pattern, DataOutputStream stream) throws IOException { + if (pattern instanceof IrAlternation) { + ReadWriteIOUtils.write(0, stream); // Type marker for IrAlternation + IrAlternation.serialize((IrAlternation) pattern, stream); + } else if (pattern instanceof IrAnchor) { + ReadWriteIOUtils.write(1, stream); // Type marker for IrAnchor + IrAnchor.serialize((IrAnchor) pattern, stream); + } else if (pattern instanceof IrConcatenation) { + ReadWriteIOUtils.write(2, stream); // Type marker for IrConcatenation + IrConcatenation.serialize((IrConcatenation) pattern, stream); + } else if (pattern instanceof IrEmpty) { + ReadWriteIOUtils.write(3, stream); // Type marker for IrEmpty + IrEmpty.serialize((IrEmpty) pattern, stream); + } else if (pattern instanceof IrExclusion) { + ReadWriteIOUtils.write(4, stream); // Type marker for IrExclusion + IrExclusion.serialize((IrExclusion) pattern, stream); + } else if (pattern instanceof IrLabel) { + ReadWriteIOUtils.write(5, stream); // Type marker for IrLabel + IrLabel.serialize((IrLabel) pattern, stream); + } else if (pattern instanceof IrPermutation) { + ReadWriteIOUtils.write(6, stream); // Type marker for IrPermutation + IrPermutation.serialize((IrPermutation) pattern, stream); + } else if (pattern instanceof IrQuantified) { + ReadWriteIOUtils.write(7, stream); // Type marker for IrQuantified + IrQuantified.serialize((IrQuantified) pattern, stream); + } else { + throw new IllegalArgumentException("Unknown IrRowPattern type"); + } + } + + public static IrRowPattern deserialize(ByteBuffer byteBuffer) { + int type = ReadWriteIOUtils.readInt(byteBuffer); + + switch (type) { + case 0: + return IrAlternation.deserialize(byteBuffer); + case 1: + return IrAnchor.deserialize(byteBuffer); + case 2: + return IrConcatenation.deserialize(byteBuffer); + case 3: + return IrEmpty.deserialize(byteBuffer); + case 4: + return IrExclusion.deserialize(byteBuffer); + case 5: + return IrLabel.deserialize(byteBuffer); + case 6: + return IrPermutation.deserialize(byteBuffer); + case 7: + return IrQuantified.deserialize(byteBuffer); + default: + throw new IllegalArgumentException("Unknown IrRowPattern type"); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrRowPatternFlattener.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrRowPatternFlattener.java new file mode 100644 index 000000000000..98f7c1c88525 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrRowPatternFlattener.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +/** + * Optimize row pattern: - remove nested exclusions - flatten alternations and concatenations - + * remove redundant empty pattern + */ +public final class IrRowPatternFlattener { + private IrRowPatternFlattener() {} + + public static IrRowPattern optimize(IrRowPattern node) { + return new Visitor().process(node, false); + } + + private static class Visitor extends IrRowPatternVisitor { + @Override + protected IrRowPattern visitIrRowPattern(IrRowPattern node, Boolean inExclusion) { + throw new UnsupportedOperationException( + "unsupported node type: " + node.getClass().getName()); + } + + @Override + protected IrRowPattern visitIrLabel(IrLabel node, Boolean inExclusion) { + return node; + } + + @Override + protected IrRowPattern visitIrAnchor(IrAnchor node, Boolean inExclusion) { + return node; + } + + @Override + protected IrRowPattern visitIrEmpty(IrEmpty node, Boolean inExclusion) { + return node; + } + + @Override + protected IrRowPattern visitIrExclusion(IrExclusion node, Boolean inExclusion) { + IrRowPattern child = process(node.getPattern(), true); + if (inExclusion) { + // skip nested exclusion. this is necessary to resolve exclusions correctly during pattern + // matching + return child; + } + + return new IrExclusion(child); + } + + /** + * Flatten the alternation and remove redundant empty branches. This method recursively inlines + * sub-pattern lists from child alternations into the top-level alternation. The branch + * preference order is reflected in the resulting order of the elements. Examples: A | (B | C) + * -> A | B | C (A | B) | ((C | D) | E) -> A | B | C | D | E + * + *

Also, redundant empty branches are removed from the resulting sub-pattern list: all empty + * branches following the first empty branch are not achievable, as they are less preferred than + * the first empty branch, so they are considered redundant. Examples: A | (() | B) -> A | () | + * B (A | ()) | ((() | B) | ()) -> A | () | B (() | ()) | () -> () + * + *

Note: The logic of removing redundant empty branches could be extended to remove other + * duplicate sub-patterns. + * + * @return the flattened IrAlternation containing at most one empty branch, or IrEmpty in case + * when the alternation is reduced to a single empty branch + */ + @Override + protected IrRowPattern visitIrAlternation(IrAlternation node, Boolean inExclusion) { + List children = + node.getPatterns().stream() + .map(pattern -> process(pattern, inExclusion)) + .collect(toImmutableList()); + + // flatten alternation + children = + children.stream() + .flatMap( + child -> { + if (child instanceof IrAlternation) { + return ((IrAlternation) child).getPatterns().stream(); + } + return Stream.of(child); + }) + .collect(toImmutableList()); + + Optional firstEmptyChild = + children.stream().filter(IrEmpty.class::isInstance).findFirst(); + if (!firstEmptyChild.isPresent()) { + return new IrAlternation(children); + } + + // remove all empty children following the first empty child + children = + children.stream() + .filter(child -> !(child instanceof IrEmpty) || child == firstEmptyChild.get()) + .collect(toImmutableList()); + + // if there is only the empty child left, replace alternation with empty pattern + if (children.size() == 1) { + return new IrEmpty(); + } + + return new IrAlternation(children); + } + + /** + * Flatten the concatenation and remove all empty branches. This method recursively inlines + * sub-pattern lists from child concatenations into the top-level concatenation. The expected + * sub-pattern order is reflected in the resulting order of the elements. Also, all empty + * branches are removed from the resulting sub-pattern list. Examples: A (B C) -> A B C (A B) + * ((C D) E) -> A B C D E A (() B) -> A B (A ()) ((() B) ()) -> A B () (() A) -> A (() ()) () -> + * () + * + * @return the flattened IrConcatenation containing no empty branches, or a sub-pattern in case + * when the concatenation is reduced to a single branch, or IrEmpty in case when all + * sub-patterns are empty + */ + @Override + protected IrRowPattern visitIrConcatenation(IrConcatenation node, Boolean inExclusion) { + List children = + node.getPatterns().stream() + .map(pattern -> process(pattern, inExclusion)) + .collect(toImmutableList()); + + // flatten concatenation and remove all empty children + children = + children.stream() + .flatMap( + child -> { + if (child instanceof IrConcatenation) { + return ((IrConcatenation) child).getPatterns().stream(); + } + return Stream.of(child); + }) + .filter(child -> !(child instanceof IrEmpty)) + .collect(toImmutableList()); + + if (children.isEmpty()) { + return new IrEmpty(); + } + if (children.size() == 1) { + return children.get(0); + } + return new IrConcatenation(children); + } + + /** + * Remove all empty branches from the permutation. Examples: PERMUTE(A, (), B, ()) -> PERMUTE(A, + * B) PERMUTE((), A) -> A PERMUTE((), ()) -> () + * + * @return the IrPermutation containing no empty branches, or a sub-pattern in case when the + * permutation is reduced to a single branch, or IrEmpty in case when all sub-patterns are + * empty + */ + @Override + protected IrRowPattern visitIrPermutation(IrPermutation node, Boolean inExclusion) { + // process children and remove all empty children + List children = + node.getPatterns().stream() + .map(pattern -> process(pattern, inExclusion)) + .filter(child -> !(child instanceof IrEmpty)) + .collect(toImmutableList()); + + if (children.isEmpty()) { + return new IrEmpty(); + } + if (children.size() == 1) { + return children.get(0); + } + return new IrPermutation(children); + } + + @Override + protected IrRowPattern visitIrQuantified(IrQuantified node, Boolean inExclusion) { + IrRowPattern child = process(node.getPattern(), inExclusion); + + if (child instanceof IrEmpty) { + return child; + } + return new IrQuantified(child, node.getQuantifier()); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrRowPatternVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrRowPatternVisitor.java new file mode 100644 index 000000000000..4e2ec9b830c8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/IrRowPatternVisitor.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import javax.annotation.Nullable; + +public abstract class IrRowPatternVisitor { + public R process(IrRowPattern rowPattern) { + return process(rowPattern, null); + } + + public R process(IrRowPattern rowPattern, @Nullable C context) { + return rowPattern.accept(this, context); + } + + protected R visitIrRowPattern(IrRowPattern rowPattern, C context) { + return null; + } + + protected R visitIrAlternation(IrAlternation node, C context) { + return visitIrRowPattern(node, context); + } + + protected R visitIrConcatenation(IrConcatenation node, C context) { + return visitIrRowPattern(node, context); + } + + protected R visitIrQuantified(IrQuantified node, C context) { + return visitIrRowPattern(node, context); + } + + protected R visitIrAnchor(IrAnchor node, C context) { + return visitIrRowPattern(node, context); + } + + protected R visitIrEmpty(IrEmpty node, C context) { + return visitIrRowPattern(node, context); + } + + protected R visitIrExclusion(IrExclusion node, C context) { + return visitIrRowPattern(node, context); + } + + protected R visitIrPermutation(IrPermutation node, C context) { + return visitIrRowPattern(node, context); + } + + protected R visitIrLabel(IrLabel node, C context) { + return visitIrRowPattern(node, context); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/LogicalIndexPointer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/LogicalIndexPointer.java new file mode 100644 index 000000000000..7660f00492dc --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/LogicalIndexPointer.java @@ -0,0 +1,167 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.LogicalIndexNavigation; + +import org.apache.tsfile.utils.ReadWriteIOUtils; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static java.util.Objects.requireNonNull; + +public class LogicalIndexPointer { + // a set of labels to navigate over: + // LAST(A.price, 3) => this is a navigation over rows with label A, so labels = {A} + // LAST(Union.price, 3) => this is a navigation over rows matching a union variable Union, so for + // SUBSET Union = (A, B, C), we have labels = {A, B, C} + // LAST(price, 3) => this is a navigation over "universal pattern variable", which is effectively + // over all rows, no matter the assigned labels. In such case labels = {} + private final Set labels; + + // logical position is a position among rows tagged with certain label (or label from a certain + // set) + // it has the following semantics: + // start from FIRST or LAST row tagged with the label (with RUNNING or FINAL semantics), and go + // logicalOffset steps forward (for FIRST) or backward (for LAST), + // skipping to consecutive rows with matching label + // Default: RUNNING LAST offset = 0 + private final boolean last; + private final boolean running; + private final int logicalOffset; + + // physical offset is the offset in physical rows, starting from the logical position. negative + // for PREV, positive for NEXT. The default is -1 for PREV and 1 for NEXT. + // Unspecified physical offset defaults to 0. + private final int physicalOffset; + + public LogicalIndexPointer( + Set labels, boolean last, boolean running, int logicalOffset, int physicalOffset) { + this.labels = requireNonNull(labels, "labels is null"); + this.last = last; + this.running = running; + checkArgument(logicalOffset >= 0, "logical offset must be >= 0, actual: %s", logicalOffset); + this.logicalOffset = logicalOffset; + this.physicalOffset = physicalOffset; + } + + public Set getLabels() { + return labels; + } + + public boolean isLast() { + return last; + } + + public boolean isRunning() { + return running; + } + + public int getLogicalOffset() { + return logicalOffset; + } + + public int getPhysicalOffset() { + return physicalOffset; + } + + public LogicalIndexNavigation toLogicalIndexNavigation(Map mapping) { + return new LogicalIndexNavigation( + labels.stream().map(mapping::get).collect(toImmutableSet()), + last, + running, + logicalOffset, + physicalOffset); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + LogicalIndexPointer that = (LogicalIndexPointer) o; + return last == that.last + && running == that.running + && logicalOffset == that.logicalOffset + && physicalOffset == that.physicalOffset + && labels.equals(that.labels); + } + + @Override + public int hashCode() { + return Objects.hash(labels, last, running, logicalOffset, physicalOffset); + } + + public static void serialize(LogicalIndexPointer pointer, ByteBuffer byteBuffer) { + ReadWriteIOUtils.write(pointer.labels.size(), byteBuffer); + for (IrLabel label : pointer.labels) { + IrLabel.serialize(label, byteBuffer); + } + + ReadWriteIOUtils.write(pointer.last, byteBuffer); + ReadWriteIOUtils.write(pointer.running, byteBuffer); + + ReadWriteIOUtils.write(pointer.logicalOffset, byteBuffer); + ReadWriteIOUtils.write(pointer.physicalOffset, byteBuffer); + } + + public static void serialize(LogicalIndexPointer pointer, DataOutputStream stream) + throws IOException { + ReadWriteIOUtils.write(pointer.labels.size(), stream); + for (IrLabel label : pointer.labels) { + IrLabel.serialize(label, stream); + } + + ReadWriteIOUtils.write(pointer.last, stream); + ReadWriteIOUtils.write(pointer.running, stream); + + ReadWriteIOUtils.write(pointer.logicalOffset, stream); + ReadWriteIOUtils.write(pointer.physicalOffset, stream); + } + + public static LogicalIndexPointer deserialize(ByteBuffer byteBuffer) { + int labelCount = ReadWriteIOUtils.readInt(byteBuffer); + Set labels = new HashSet<>(); + for (int i = 0; i < labelCount; i++) { + labels.add(IrLabel.deserialize(byteBuffer)); + } + + boolean last = ReadWriteIOUtils.readBoolean(byteBuffer); + boolean running = ReadWriteIOUtils.readBoolean(byteBuffer); + + int logicalOffset = ReadWriteIOUtils.readInt(byteBuffer); + int physicalOffset = ReadWriteIOUtils.readInt(byteBuffer); + + return new LogicalIndexPointer(labels, last, running, logicalOffset, physicalOffset); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/MatchNumberValuePointer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/MatchNumberValuePointer.java new file mode 100644 index 000000000000..d75dc9e5133a --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/MatchNumberValuePointer.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +public final class MatchNumberValuePointer implements ValuePointer { + @Override + public int hashCode() { + return MatchNumberValuePointer.class.hashCode(); + } + + @Override + public boolean equals(Object obj) { + return obj instanceof MatchNumberValuePointer; + } + + public static void serialize(MatchNumberValuePointer pointer, ByteBuffer byteBuffer) {} + + public static void serialize(MatchNumberValuePointer pointer, DataOutputStream stream) + throws IOException {} + + public static MatchNumberValuePointer deserialize(ByteBuffer byteBuffer) { + return new MatchNumberValuePointer(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/Patterns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/Patterns.java new file mode 100644 index 000000000000..e46f918c2f8b --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/Patterns.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import com.google.common.collect.ImmutableList; + +import java.util.Optional; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrAnchor.Type.PARTITION_END; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrAnchor.Type.PARTITION_START; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrQuantifier.oneOrMore; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrQuantifier.zeroOrMore; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrQuantifier.zeroOrOne; + +public class Patterns { + private Patterns() {} + + public static IrLabel label(String name) { + return new IrLabel(name); + } + + public static IrRowPattern empty() { + return new IrEmpty(); + } + + public static IrRowPattern excluded(IrRowPattern pattern) { + return new IrExclusion(pattern); + } + + public static IrRowPattern start() { + return new IrAnchor(PARTITION_START); + } + + public static IrRowPattern end() { + return new IrAnchor(PARTITION_END); + } + + public static IrRowPattern plusQuantified(IrRowPattern pattern, boolean greedy) { + return new IrQuantified(pattern, oneOrMore(greedy)); + } + + public static IrRowPattern starQuantified(IrRowPattern pattern, boolean greedy) { + return new IrQuantified(pattern, zeroOrMore(greedy)); + } + + public static IrRowPattern questionMarkQuantified(IrRowPattern pattern, boolean greedy) { + return new IrQuantified(pattern, zeroOrOne(greedy)); + } + + public static IrRowPattern rangeQuantified( + IrRowPattern pattern, int atLeast, Optional atMost, boolean greedy) { + return new IrQuantified(pattern, new IrQuantifier(atLeast, atMost, greedy)); + } + + public static IrRowPattern alternation(IrRowPattern... parts) { + return new IrAlternation(ImmutableList.copyOf(parts)); + } + + public static IrRowPattern concatenation(IrRowPattern... parts) { + return new IrConcatenation(ImmutableList.copyOf(parts)); + } + + public static IrRowPattern permutation(IrRowPattern... parts) { + return new IrPermutation(ImmutableList.copyOf(parts)); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/RowPatternToIrRewriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/RowPatternToIrRewriter.java new file mode 100644 index 000000000000..ed93c9900393 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/RowPatternToIrRewriter.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis; +import org.apache.iotdb.db.queryengine.plan.relational.analyzer.Analysis.Range; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrAnchor.Type; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AnchorPattern; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.AstVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.EmptyPattern; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ExcludedPattern; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.OneOrMoreQuantifier; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternAlternation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternConcatenation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternPermutation; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternQuantifier; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.PatternVariable; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.QuantifiedPattern; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RangeQuantifier; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.RowPattern; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ZeroOrMoreQuantifier; +import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.ZeroOrOneQuantifier; + +import java.util.List; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Objects.requireNonNull; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrQuantifier.oneOrMore; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrQuantifier.range; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrQuantifier.zeroOrMore; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrQuantifier.zeroOrOne; + +public class RowPatternToIrRewriter extends AstVisitor { + private final Analysis analysis; + + public RowPatternToIrRewriter(Analysis analysis) { + this.analysis = requireNonNull(analysis, "analysis is null"); + } + + public static IrRowPattern rewrite(RowPattern node, Analysis analysis) { + return new RowPatternToIrRewriter(analysis).process(node); + } + + @Override + protected IrRowPattern visitPatternAlternation(PatternAlternation node, Void context) { + List patterns = + node.getPatterns().stream().map(this::process).collect(toImmutableList()); + + return new IrAlternation(patterns); + } + + @Override + protected IrRowPattern visitPatternConcatenation(PatternConcatenation node, Void context) { + List patterns = + node.getPatterns().stream().map(this::process).collect(toImmutableList()); + + return new IrConcatenation(patterns); + } + + @Override + protected IrRowPattern visitQuantifiedPattern(QuantifiedPattern node, Void context) { + IrRowPattern pattern = process(node.getPattern()); + IrQuantifier quantifier = rewritePatternQuantifier(node.getPatternQuantifier()); + + return new IrQuantified(pattern, quantifier); + } + + private IrQuantifier rewritePatternQuantifier(PatternQuantifier quantifier) { + if (quantifier instanceof ZeroOrMoreQuantifier) { + return zeroOrMore(quantifier.isGreedy()); + } + + if (quantifier instanceof OneOrMoreQuantifier) { + return oneOrMore(quantifier.isGreedy()); + } + + if (quantifier instanceof ZeroOrOneQuantifier) { + return zeroOrOne(quantifier.isGreedy()); + } + + if (quantifier instanceof RangeQuantifier) { + Range range = analysis.getRange((RangeQuantifier) quantifier); + return range(range.getAtLeast(), range.getAtMost(), quantifier.isGreedy()); + } + + throw new IllegalStateException( + "unsupported pattern quantifier type: " + quantifier.getClass().getSimpleName()); + } + + @Override + protected IrRowPattern visitAnchorPattern(AnchorPattern node, Void context) { + Type type; + switch (node.getType()) { + case PARTITION_START: + type = IrAnchor.Type.PARTITION_START; + break; + case PARTITION_END: + type = IrAnchor.Type.PARTITION_END; + break; + default: + throw new IllegalArgumentException("Unexpected value: " + node.getType()); + } + + return new IrAnchor(type); + } + + @Override + protected IrRowPattern visitEmptyPattern(EmptyPattern node, Void context) { + return new IrEmpty(); + } + + @Override + protected IrRowPattern visitExcludedPattern(ExcludedPattern node, Void context) { + IrRowPattern pattern = process(node.getPattern()); + + return new IrExclusion(pattern); + } + + @Override + protected IrRowPattern visitPatternPermutation(PatternPermutation node, Void context) { + List patterns = + node.getPatterns().stream().map(this::process).collect(toImmutableList()); + + return new IrPermutation(patterns); + } + + @Override + protected IrRowPattern visitPatternVariable(PatternVariable node, Void context) { + return new IrLabel(node.getName().getCanonicalValue()); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ScalarValuePointer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ScalarValuePointer.java new file mode 100644 index 000000000000..5957242d97d7 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ScalarValuePointer.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +import org.apache.iotdb.db.queryengine.plan.relational.planner.Symbol; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Objects; + +import static java.util.Objects.requireNonNull; + +public final class ScalarValuePointer implements ValuePointer { + private final LogicalIndexPointer logicalIndexPointer; + private final Symbol inputSymbol; + + public ScalarValuePointer(LogicalIndexPointer logicalIndexPointer, Symbol inputSymbol) { + this.logicalIndexPointer = requireNonNull(logicalIndexPointer, "logicalIndexPointer is null"); + this.inputSymbol = requireNonNull(inputSymbol, "inputSymbol is null"); + } + + public LogicalIndexPointer getLogicalIndexPointer() { + return logicalIndexPointer; + } + + public Symbol getInputSymbol() { + return inputSymbol; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if ((obj == null) || (getClass() != obj.getClass())) { + return false; + } + ScalarValuePointer o = (ScalarValuePointer) obj; + return Objects.equals(logicalIndexPointer, o.logicalIndexPointer) + && Objects.equals(inputSymbol, o.inputSymbol); + } + + @Override + public int hashCode() { + return Objects.hash(logicalIndexPointer, inputSymbol); + } + + public static void serialize(ScalarValuePointer pointer, ByteBuffer byteBuffer) { + LogicalIndexPointer.serialize(pointer.logicalIndexPointer, byteBuffer); + Symbol.serialize(pointer.inputSymbol, byteBuffer); + } + + public static void serialize(ScalarValuePointer pointer, DataOutputStream stream) + throws IOException { + LogicalIndexPointer.serialize(pointer.logicalIndexPointer, stream); + Symbol.serialize(pointer.inputSymbol, stream); + } + + public static ScalarValuePointer deserialize(ByteBuffer byteBuffer) { + LogicalIndexPointer logicalIndexPointer = LogicalIndexPointer.deserialize(byteBuffer); + Symbol inputSymbol = Symbol.deserialize(byteBuffer); + return new ScalarValuePointer(logicalIndexPointer, inputSymbol); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ValuePointer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ValuePointer.java new file mode 100644 index 000000000000..1b73e62a7246 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/rowpattern/ValuePointer.java @@ -0,0 +1,17 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern; + +public interface ValuePointer {} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java index 74a834604a15..1ef46d017df2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/FunctionCall.java @@ -39,16 +39,28 @@ public class FunctionCall extends Expression { private final List arguments; public FunctionCall(QualifiedName name, List arguments) { - this(null, name, false, Optional.empty(), arguments); + super(null); + this.name = requireNonNull(name, "name is null"); + this.distinct = false; + this.processingMode = Optional.empty(); + this.arguments = requireNonNull(arguments, "arguments is null"); } public FunctionCall(QualifiedName name, boolean distinct, List arguments) { - this(null, name, distinct, Optional.empty(), arguments); + super(null); + this.name = requireNonNull(name, "name is null"); + this.distinct = distinct; + this.processingMode = Optional.empty(); + this.arguments = requireNonNull(arguments, "arguments is null"); } public FunctionCall( QualifiedName name, Optional processingMode, List arguments) { - this(null, name, false, processingMode, arguments); + super(null); + this.name = requireNonNull(name, "name is null"); + this.distinct = false; + this.processingMode = requireNonNull(processingMode, "processingMode is null"); + this.arguments = requireNonNull(arguments, "arguments is null"); } public FunctionCall( @@ -56,7 +68,11 @@ public FunctionCall( boolean distinct, Optional processingMode, List arguments) { - this(null, name, distinct, processingMode, arguments); + super(null); + this.name = requireNonNull(name, "name is null"); + this.distinct = distinct; + this.processingMode = requireNonNull(processingMode, "processingMode is null"); + this.arguments = requireNonNull(arguments, "arguments is null"); } public FunctionCall(NodeLocation location, QualifiedName name, List arguments) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java index eb39195141d6..ff5a949d94f2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/parser/AstBuilder.java @@ -268,7 +268,6 @@ import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo.skipToFirst; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo.skipToLast; import static org.apache.iotdb.db.queryengine.plan.relational.sql.ast.SkipTo.skipToNextRow; -import static org.apache.iotdb.db.queryengine.transformation.dag.column.unary.scalar.TableBuiltinScalarFunction.DATE_BIN; import static org.apache.iotdb.db.utils.constant.SqlConstant.FIRST_AGGREGATION; import static org.apache.iotdb.db.utils.constant.SqlConstant.FIRST_BY_AGGREGATION; import static org.apache.iotdb.db.utils.constant.SqlConstant.LAST_AGGREGATION; diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/MatcherTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/MatcherTest.java new file mode 100644 index 000000000000..2ee0a143c7b1 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/process/rowpattern/MatcherTest.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern; + +import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher.ArrayView; +import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher.IrRowPatternToProgramRewriter; +import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher.MatchResult; +import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher.Matcher; +import org.apache.iotdb.db.queryengine.execution.operator.process.rowpattern.matcher.Program; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrLabel; +import org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.IrRowPattern; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.junit.Test; + +import java.util.Map; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.Patterns.alternation; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.Patterns.concatenation; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.Patterns.end; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.Patterns.excluded; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.Patterns.label; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.Patterns.permutation; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.Patterns.questionMarkQuantified; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.Patterns.starQuantified; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.rowpattern.Patterns.start; + +public class MatcherTest { + private static final Map LABEL_MAPPING = + ImmutableMap.of( + new IrLabel("A"), 0, + new IrLabel("B"), 1, + new IrLabel("C"), 2, + new IrLabel("D"), 3, + new IrLabel("E"), 4); + + @Test + public void testLabels() { + checkMatch(concatenation(label("A"), label("B")), "ABCD", new char[] {'A', 'B'}); + checkMatch( + concatenation(starQuantified(label("A"), true), label("B")), + "AABCD", + new char[] {'A', 'A', 'B'}); + checkMatch( + concatenation(starQuantified(label("A"), true), label("B")), "BCD", new char[] {'B'}); + checkMatch(concatenation(start(), label("A"), label("B")), "ABCD", new char[] {'A', 'B'}); + checkMatch(concatenation(label("A"), label("B"), end()), "AB", new char[] {'A', 'B'}); + checkMatch(concatenation(excluded(label("A")), label("B")), "ABCD", new char[] {'A', 'B'}); + checkMatch( + alternation(concatenation(label("A"), label("B")), concatenation(label("B"), label("C"))), + "ABCD", + new char[] {'A', 'B'}); + checkMatch(permutation(label("A"), label("B"), label("C")), "ABCD", new char[] {'A', 'B', 'C'}); + checkMatch( + concatenation(label("A"), questionMarkQuantified(label("B"), true)), + "ABCD", + new char[] {'A', 'B'}); + checkMatch( + concatenation(label("A"), questionMarkQuantified(label("B"), false)), + "ABCD", + new char[] {'A'}); + } + + private void checkMatch(IrRowPattern pattern, String input, char[] expectedLabels) { + MatchResult result = match(pattern, input); + if (!result.isMatched()) { + throw new AssertionError("Pattern did not match."); + } + + int[] mappedExpected = new int[expectedLabels.length]; + for (int i = 0; i < expectedLabels.length; i++) { + mappedExpected[i] = LABEL_MAPPING.get(new IrLabel(String.valueOf(expectedLabels[i]))); + } + + int[] actualLabels = result.getLabels().toArray(); + if (!java.util.Arrays.equals(actualLabels, mappedExpected)) { + throw new AssertionError( + "Expected labels: " + + java.util.Arrays.toString(mappedExpected) + + ", but got: " + + java.util.Arrays.toString(actualLabels)); + } + } + + @Test + public void testExclusionCaptures() { + checkMatch(concatenation(excluded(label("A")), label("B")), "ABCD", new int[] {0, 1}); + checkMatch(excluded(concatenation(label("A"), label("B"))), "ABCD", new int[] {0, 2}); + checkMatch(concatenation(label("A"), excluded(label("B"))), "ABCD", new int[] {1, 2}); + checkMatch( + concatenation(label("A"), starQuantified(excluded(label("B")), true)), + "ABBBCD", + new int[] {1, 2, 2, 3, 3, 4}); + checkMatch( + concatenation(label("A"), excluded(starQuantified(label("B"), true))), + "ABBBCD", + new int[] {1, 4}); + checkMatch( + concatenation( + label("A"), + starQuantified(excluded(label("B")), true), + label("C"), + excluded(starQuantified(label("D"), true))), + "ABBCDDDE", + new int[] {1, 2, 2, 3, 4, 7}); + checkMatch( + concatenation( + label("A"), + starQuantified( + concatenation( + excluded(concatenation(label("B"), label("C"))), + label("D"), + excluded(label("E"))), + true)), + "ABCDEBCDEBCDE", + new int[] {1, 3, 4, 5, 5, 7, 8, 9, 9, 11, 12, 13}); + } + + private void checkMatch(IrRowPattern pattern, String input, int[] expectedCaptures) { + MatchResult result = match(pattern, input); + if (!result.isMatched()) { + throw new AssertionError("Pattern did not match."); + } + + int[] actualCaptures = result.getExclusions().toArray(); + if (!java.util.Arrays.equals(actualCaptures, expectedCaptures)) { + throw new AssertionError( + "Expected captures: " + + java.util.Arrays.toString(expectedCaptures) + + ", but got: " + + java.util.Arrays.toString(actualCaptures)); + } + } + + private static MatchResult match(IrRowPattern pattern, String input) { + Program program = IrRowPatternToProgramRewriter.rewrite(pattern, LABEL_MAPPING); + + Matcher matcher = new Matcher(program); + + int[] mappedInput = new int[input.length()]; + for (int i = 0; i < input.length(); i++) { + mappedInput[i] = LABEL_MAPPING.get(new IrLabel(String.valueOf(input.charAt(i)))); + } + + return matcher.run(testLabelEvaluator(mappedInput)); + } + + private static LabelEvaluator testLabelEvaluator(int[] input) { + // create dummy WindowIndex for the LabelEvaluator + // PagesIndex pagesIndex = + // new PagesIndex.TestingFactory(false).newPagesIndex(ImmutableList.of(), 1); + // pagesIndex.addPage(new Page(1)); + return new testLabelEvaluator(input); + } + + private static class testLabelEvaluator extends LabelEvaluator { + private final int[] input; + + public testLabelEvaluator(int[] input) { + super(0, 0, 0, 0, 1, ImmutableList.of()); + this.input = input; + } + + @Override + public int getInputLength() { + return input.length; + } + + @Override + public boolean isMatchingAtPartitionStart() { + return true; + } + + @Override + public boolean evaluateLabel(ArrayView matchedLabels) { + int position = matchedLabels.length() - 1; + return input[position] == matchedLabels.get(position); + } + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/RowPatternRecognitionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/RowPatternRecognitionTest.java index b5539b443854..1972a9c6bcd1 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/RowPatternRecognitionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/RowPatternRecognitionTest.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.security.AllowAllAccessControl; import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Statement; import org.apache.iotdb.db.queryengine.plan.relational.sql.parser.SqlParser; +import org.apache.iotdb.db.queryengine.plan.relational.sql.rewrite.StatementRewriteFactory; import org.junit.Assert; import org.junit.Test; @@ -428,7 +429,7 @@ private void assertTestSuccess(String sql) { public static void analyzeSQL(String sql, Metadata metadata, final MPPQueryContext context) { SqlParser sqlParser = new SqlParser(); - Statement statement = sqlParser.createStatement(sql, ZoneId.systemDefault()); + Statement statement = sqlParser.createStatement(sql, ZoneId.systemDefault(), null); SessionInfo session = new SessionInfo( 0, "test", ZoneId.systemDefault(), "testdb", IClientSession.SqlDialect.TABLE); @@ -451,6 +452,7 @@ public static void analyzeStatement( statementAnalyzerFactory, Collections.emptyList(), Collections.emptyMap(), + new StatementRewriteFactory(metadata).getStatementRewrite(), NOOP); analyzer.analyze(statement); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RowPatternRecognitionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RowPatternRecognitionTest.java new file mode 100644 index 000000000000..90a607427ea2 --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RowPatternRecognitionTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.plan.relational.planner; + +import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; +import org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.Test; + +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.tableScan; + +public class RowPatternRecognitionTest { + @Test + public void Test1() { + PlanTester planTester = new PlanTester(); + + String sql = + "SELECT * " + + "FROM table1 " + + "MATCH_RECOGNIZE ( " + + " ORDER BY time " + + " MEASURES " + + " A.s1 AS col1 " + + " PATTERN (A B+) " + + " DEFINE " + + " B AS B.s2 > 5 " + + ") AS m"; + + LogicalQueryPlan logicalQueryPlan = planTester.createPlan(sql); + + PlanMatchPattern tableScan = + tableScan( + "testdb.table1", + ImmutableList.of("time", "tag1", "tag2", "tag3", "attr1", "s1", "s2", "s3"), + ImmutableSet.of("time", "tag1", "tag2", "tag3", "attr1", "s1", "s2", "s3")); + + // Verify full LogicalPlan + // Output - PatternRecognition - TableScan + // TODO: + // need to add PatternRecognitionMatcher + // assertPlan(logicalQueryPlan, output(anyTree(tableScan))); + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java index 57cfde2465ed..7bc970404aeb 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/planner/assertions/PlanMatchPattern.java @@ -377,15 +377,15 @@ public static PlanMatchPattern topNRanking(Consumer TopNRankingMatcher.Builder builder = new TopNRankingMatcher.Builder(source); handler.accept(builder); return builder.build(); - } - - public static PlanMatchPattern patternRecognition(Consumer handler, PlanMatchPattern source) - { - PatternRecognitionMatcher.Builder builder = new PatternRecognitionMatcher.Builder(source); - handler.accept(builder); - return builder.build(); }*/ + // public static PlanMatchPattern patternRecognition( + // Consumer handler, PlanMatchPattern source) { + // PatternRecognitionMatcher.Builder builder = new PatternRecognitionMatcher.Builder(source); + // handler.accept(builder); + // return builder.build(); + // } + public static PlanMatchPattern join( JoinNode.JoinType type, Consumer handler) { JoinMatcher.Builder builder = new JoinMatcher.Builder(type);