From 7c1fc08fca9494ee27c5b8b2bfc8e09ede1e86ae Mon Sep 17 00:00:00 2001 From: Steven Hawkins Date: Mon, 27 Jul 2020 20:27:04 -0400 Subject: [PATCH] TEIID-5977 adding logic to associate a virtual function with a physical (#1316) --- .../org/teiid/metadata/FunctionMethod.java | 1 + .../datamgr/LanguageBridgeFactory.java | 102 ++++++++++++++--- .../java/org/teiid/query/QueryPlugin.java | 3 +- .../java/org/teiid/query/eval/Evaluator.java | 57 ++++++++- .../query/function/FunctionDescriptor.java | 13 +-- .../query/metadata/BasicQueryMetadata.java | 6 + .../metadata/BasicQueryMetadataWrapper.java | 6 + .../query/metadata/MetadataValidator.java | 28 +++++ .../query/metadata/PushdownFunctions.java | 32 ++++++ .../metadata/QueryMetadataInterface.java | 3 + .../metadata/TransformationMetadata.java | 43 ++++++- .../relational/rules/CapabilitiesUtil.java | 8 ++ .../query/resolver/util/ResolverVisitor.java | 33 +++++- .../org/teiid/query/sql/symbol/Function.java | 22 ++++ .../resources/org/teiid/query/i18n.properties | 1 + .../query/processor/TestFunctionPushdown.java | 6 +- .../teiid/olingo/common/JTS2OlingoBridge.java | 6 +- .../runtime/HardCodedExecutionFactory.java | 4 + .../runtime/TestVirtualFunctionPushdown.java | 108 ++++++++++++++++++ 19 files changed, 442 insertions(+), 40 deletions(-) create mode 100644 engine/src/main/java/org/teiid/query/metadata/PushdownFunctions.java create mode 100644 runtime/src/test/java/org/teiid/runtime/TestVirtualFunctionPushdown.java diff --git a/api/src/main/java/org/teiid/metadata/FunctionMethod.java b/api/src/main/java/org/teiid/metadata/FunctionMethod.java index df165de866a..100d9f1fc5f 100644 --- a/api/src/main/java/org/teiid/metadata/FunctionMethod.java +++ b/api/src/main/java/org/teiid/metadata/FunctionMethod.java @@ -53,6 +53,7 @@ public class FunctionMethod extends AbstractMetadataRecord { public static final String SYSTEM_NAME = AbstractMetadataRecord.RELATIONAL_PREFIX + "system-name"; //$NON-NLS-1$ + public static final String VIRTUAL_FUNCTION = AbstractMetadataRecord.RELATIONAL_PREFIX + "virtual-function"; //$NON-NLS-1$ private static final long serialVersionUID = -8039086494296455152L; diff --git a/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java b/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java index 863b50f7d70..50d601b9ee4 100644 --- a/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java +++ b/engine/src/main/java/org/teiid/dqp/internal/datamgr/LanguageBridgeFactory.java @@ -18,7 +18,18 @@ package org.teiid.dqp.internal.datamgr; -import java.util.*; +import java.util.AbstractList; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.RandomAccess; import org.teiid.api.exception.query.FunctionExecutionException; import org.teiid.api.exception.query.QueryMetadataException; @@ -31,17 +42,49 @@ import org.teiid.core.TeiidRuntimeException; import org.teiid.core.types.ArrayImpl; import org.teiid.core.types.DataTypeManager; -import org.teiid.language.*; +import org.teiid.language.AggregateFunction; +import org.teiid.language.AndOr; +import org.teiid.language.Argument; import org.teiid.language.Argument.Direction; +import org.teiid.language.BatchedUpdates; +import org.teiid.language.BulkCommand; +import org.teiid.language.Call; +import org.teiid.language.ColumnReference; +import org.teiid.language.Comparison; import org.teiid.language.Comparison.Operator; +import org.teiid.language.Condition; import org.teiid.language.DerivedColumn; +import org.teiid.language.DerivedTable; +import org.teiid.language.Exists; +import org.teiid.language.ExpressionValueSource; +import org.teiid.language.In; +import org.teiid.language.InsertValueSource; +import org.teiid.language.IsDistinct; +import org.teiid.language.IsNull; +import org.teiid.language.Join; +import org.teiid.language.Like; +import org.teiid.language.Literal; +import org.teiid.language.NamedProcedureCall; +import org.teiid.language.NamedTable; +import org.teiid.language.Not; +import org.teiid.language.Parameter; +import org.teiid.language.QueryExpression; +import org.teiid.language.SearchedCase; +import org.teiid.language.SearchedWhenClause; import org.teiid.language.Select; +import org.teiid.language.SortSpecification; import org.teiid.language.SortSpecification.NullOrdering; import org.teiid.language.SortSpecification.Ordering; +import org.teiid.language.SubqueryComparison; import org.teiid.language.SubqueryComparison.Quantifier; +import org.teiid.language.SubqueryIn; +import org.teiid.language.TableReference; import org.teiid.language.WindowSpecification; +import org.teiid.language.With; +import org.teiid.language.WithItem; import org.teiid.metadata.BaseColumn; import org.teiid.metadata.Column; +import org.teiid.metadata.FunctionMethod; import org.teiid.metadata.Procedure; import org.teiid.metadata.ProcedureParameter; import org.teiid.metadata.Table; @@ -51,16 +94,40 @@ import org.teiid.query.metadata.QueryMetadataInterface; import org.teiid.query.metadata.TempMetadataID; import org.teiid.query.optimizer.relational.rules.RulePlaceAccess; -import org.teiid.query.sql.lang.*; +import org.teiid.query.sql.lang.BatchedUpdateCommand; import org.teiid.query.sql.lang.Command; +import org.teiid.query.sql.lang.CompareCriteria; +import org.teiid.query.sql.lang.CompoundCriteria; +import org.teiid.query.sql.lang.Criteria; import org.teiid.query.sql.lang.Delete; +import org.teiid.query.sql.lang.DependentSetCriteria; +import org.teiid.query.sql.lang.ExistsCriteria; +import org.teiid.query.sql.lang.FromClause; import org.teiid.query.sql.lang.GroupBy; import org.teiid.query.sql.lang.Insert; +import org.teiid.query.sql.lang.IsDistinctCriteria; +import org.teiid.query.sql.lang.IsNullCriteria; +import org.teiid.query.sql.lang.JoinPredicate; +import org.teiid.query.sql.lang.JoinType; import org.teiid.query.sql.lang.Limit; +import org.teiid.query.sql.lang.MatchCriteria; +import org.teiid.query.sql.lang.NotCriteria; import org.teiid.query.sql.lang.OrderBy; +import org.teiid.query.sql.lang.OrderByItem; +import org.teiid.query.sql.lang.Query; +import org.teiid.query.sql.lang.QueryCommand; +import org.teiid.query.sql.lang.SPParameter; import org.teiid.query.sql.lang.SetClause; +import org.teiid.query.sql.lang.SetClauseList; +import org.teiid.query.sql.lang.SetCriteria; import org.teiid.query.sql.lang.SetQuery; +import org.teiid.query.sql.lang.StoredProcedure; +import org.teiid.query.sql.lang.SubqueryCompareCriteria; +import org.teiid.query.sql.lang.SubqueryFromClause; +import org.teiid.query.sql.lang.SubquerySetCriteria; +import org.teiid.query.sql.lang.UnaryFromClause; import org.teiid.query.sql.lang.Update; +import org.teiid.query.sql.lang.WithQueryCommand; import org.teiid.query.sql.symbol.AggregateSymbol; import org.teiid.query.sql.symbol.AggregateSymbol.Type; import org.teiid.query.sql.symbol.AliasSymbol; @@ -814,9 +881,12 @@ org.teiid.language.Expression translate(Function function) { params.add(translate(args[i])); } String name = function.getName(); - if (function.getFunctionDescriptor() != null) { - name = function.getFunctionDescriptor().getName(); - if (!supportsConcat2 && function.getFunctionDescriptor().getMethod().getParent() == null && name.equalsIgnoreCase(SourceSystemFunctions.CONCAT2)) { + FunctionMethod method = null; + FunctionDescriptor functionDescriptor = function.getFunctionDescriptor(); + if (functionDescriptor != null) { + method = functionDescriptor.getMethod(); + name = functionDescriptor.getName(); + if (!supportsConcat2 && method.getParent() == null && name.equalsIgnoreCase(SourceSystemFunctions.CONCAT2)) { Expression[] newArgs = new Expression[args.length]; boolean useCase = true; @@ -853,7 +923,7 @@ org.teiid.language.Expression translate(Function function) { caseExpr.setType(DataTypeManager.DefaultDataClasses.STRING); return translate(caseExpr); } - if (function.getFunctionDescriptor().getMethod().getParent() == null && name.equalsIgnoreCase(SourceSystemFunctions.TIMESTAMPADD) + if (method.getParent() == null && name.equalsIgnoreCase(SourceSystemFunctions.TIMESTAMPADD) && function.getArg(1).getType() == DataTypeManager.DefaultDataClasses.LONG) { //TEIID-5406 only allow integer literal pushdown for backwards compatibility if (params.get(1) instanceof Literal) { @@ -869,11 +939,17 @@ org.teiid.language.Expression translate(Function function) { Arrays.asList(params.get(1), new Literal(DataTypeManager.DefaultDataTypes.INTEGER, DataTypeManager.DefaultDataClasses.STRING)), DataTypeManager.DefaultDataClasses.INTEGER)); } } + + if (function.getPushdownFunction() != null) { + method = function.getPushdownFunction(); + name = method.getName(); + } + //check for translator pushdown functions, and use the name in source if possible - if (function.getFunctionDescriptor().getMethod().getNameInSource() != null && - (CoreConstants.SYSTEM_MODEL.equals(function.getFunctionDescriptor().getSchema()) - || (function.getFunctionDescriptor().getMethod().getParent() != null && function.getFunctionDescriptor().getMethod().getParent().isPhysical()))) { - name = function.getFunctionDescriptor().getMethod().getNameInSource(); + if (method.getNameInSource() != null && + (CoreConstants.SYSTEM_MODEL.equals(functionDescriptor.getSchema()) + || (method.getParent() != null && method.getParent().isPhysical()))) { + name = method.getNameInSource(); } } else { name = Symbol.getShortName(name); @@ -882,9 +958,7 @@ org.teiid.language.Expression translate(Function function) { //if there is any ambiguity in the function name it will be up to the translator logic to check the //metadata org.teiid.language.Function result = new org.teiid.language.Function(name, params, function.getType()); - if (function.getFunctionDescriptor() != null) { - result.setMetadataObject(function.getFunctionDescriptor().getMethod()); - } + result.setMetadataObject(method); return result; } diff --git a/engine/src/main/java/org/teiid/query/QueryPlugin.java b/engine/src/main/java/org/teiid/query/QueryPlugin.java index dd465ef32e5..2be8f3aaea0 100644 --- a/engine/src/main/java/org/teiid/query/QueryPlugin.java +++ b/engine/src/main/java/org/teiid/query/QueryPlugin.java @@ -667,6 +667,7 @@ public static enum Event implements BundleUtil.Event{ TEIID31301, TEIID31302, TEIID31303, - TEIID31304 + TEIID31304, + TEIID31305 } } diff --git a/engine/src/main/java/org/teiid/query/eval/Evaluator.java b/engine/src/main/java/org/teiid/query/eval/Evaluator.java index 234322c5858..a61aabd9b15 100644 --- a/engine/src/main/java/org/teiid/query/eval/Evaluator.java +++ b/engine/src/main/java/org/teiid/query/eval/Evaluator.java @@ -45,7 +45,18 @@ import org.teiid.core.ComponentNotFoundException; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidProcessingException; -import org.teiid.core.types.*; +import org.teiid.core.types.ArrayImpl; +import org.teiid.core.types.BaseClobType; +import org.teiid.core.types.BaseLob; +import org.teiid.core.types.BinaryType; +import org.teiid.core.types.BlobType; +import org.teiid.core.types.InputStreamFactory; +import org.teiid.core.types.JsonType; +import org.teiid.core.types.SQLXMLImpl; +import org.teiid.core.types.Sequencable; +import org.teiid.core.types.Streamable; +import org.teiid.core.types.TransformationException; +import org.teiid.core.types.XMLType; import org.teiid.core.types.XMLType.Type; import org.teiid.core.types.basic.StringToSQLXMLTransform; import org.teiid.core.util.EquivalenceUtil; @@ -61,10 +72,48 @@ import org.teiid.query.metadata.TempMetadataID; import org.teiid.query.processor.ProcessorDataManager; import org.teiid.query.sql.LanguageObject; -import org.teiid.query.sql.lang.*; +import org.teiid.query.sql.lang.AbstractSetCriteria; +import org.teiid.query.sql.lang.CollectionValueIterator; +import org.teiid.query.sql.lang.CompareCriteria; +import org.teiid.query.sql.lang.CompoundCriteria; +import org.teiid.query.sql.lang.Criteria; +import org.teiid.query.sql.lang.DependentSetCriteria; +import org.teiid.query.sql.lang.ExistsCriteria; +import org.teiid.query.sql.lang.ExpressionCriteria; +import org.teiid.query.sql.lang.IsDistinctCriteria; +import org.teiid.query.sql.lang.IsNullCriteria; +import org.teiid.query.sql.lang.MatchCriteria; +import org.teiid.query.sql.lang.NotCriteria; +import org.teiid.query.sql.lang.SetCriteria; +import org.teiid.query.sql.lang.SubqueryCompareCriteria; +import org.teiid.query.sql.lang.SubqueryContainer; +import org.teiid.query.sql.lang.SubquerySetCriteria; import org.teiid.query.sql.proc.ExceptionExpression; -import org.teiid.query.sql.symbol.*; +import org.teiid.query.sql.symbol.Array; +import org.teiid.query.sql.symbol.CaseExpression; +import org.teiid.query.sql.symbol.Constant; +import org.teiid.query.sql.symbol.DerivedColumn; +import org.teiid.query.sql.symbol.DerivedExpression; +import org.teiid.query.sql.symbol.ElementSymbol; +import org.teiid.query.sql.symbol.Expression; +import org.teiid.query.sql.symbol.ExpressionSymbol; +import org.teiid.query.sql.symbol.Function; +import org.teiid.query.sql.symbol.GroupSymbol; +import org.teiid.query.sql.symbol.JSONObject; +import org.teiid.query.sql.symbol.QueryString; +import org.teiid.query.sql.symbol.Reference; +import org.teiid.query.sql.symbol.ScalarSubquery; +import org.teiid.query.sql.symbol.SearchedCaseExpression; +import org.teiid.query.sql.symbol.TextLine; +import org.teiid.query.sql.symbol.XMLCast; +import org.teiid.query.sql.symbol.XMLElement; +import org.teiid.query.sql.symbol.XMLExists; +import org.teiid.query.sql.symbol.XMLForest; +import org.teiid.query.sql.symbol.XMLNamespaces; import org.teiid.query.sql.symbol.XMLNamespaces.NamespaceItem; +import org.teiid.query.sql.symbol.XMLParse; +import org.teiid.query.sql.symbol.XMLQuery; +import org.teiid.query.sql.symbol.XMLSerialize; import org.teiid.query.sql.util.ValueIterator; import org.teiid.query.sql.util.ValueIteratorSource; import org.teiid.query.sql.util.VariableContext; @@ -1308,7 +1357,7 @@ private Object evaluate(Function function, List tuple) } // Execute function - return fd.invokeFunction(values, context, null); + return fd.invokeFunction(values, context, null, function.isCalledWithVarArgArrayParam()); } protected Object evaluatePushdown(Function function, List tuple, diff --git a/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java b/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java index 65c49ef0357..5b2f47db743 100644 --- a/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java +++ b/engine/src/main/java/org/teiid/query/function/FunctionDescriptor.java @@ -60,7 +60,6 @@ public class FunctionDescriptor implements Serializable, Cloneable { private FunctionMethod method; private String schema; //TODO: remove me - we need to create a proper schema for udf and system functions private boolean hasWrappedArgs; - private boolean calledWithVarArgArrayParam; //TODO: could store this on the function and pass to invoke // This is transient as it would be useless to invoke this method in // a different VM. This function descriptor can be used to look up @@ -205,6 +204,10 @@ void setReturnType(Class returnType) { * @return Result of invoking the function */ public Object invokeFunction(Object[] values, CommandContext context, Object functionTarget) throws FunctionExecutionException, BlockedException { + return invokeFunction(values, context, functionTarget, false); + } + + public Object invokeFunction(Object[] values, CommandContext context, Object functionTarget, boolean calledWithVarArgArrayParam) throws FunctionExecutionException, BlockedException { if (!isNullDependent()) { for (int i = requiresContext?1:0; i < values.length; i++) { if (values[i] == null) { @@ -337,14 +340,6 @@ public static Object importValue(Object result, Class expectedType, CommandCo return result; } - public boolean isCalledWithVarArgArrayParam() { - return calledWithVarArgArrayParam; - } - - public void setCalledWithVarArgArrayParam(boolean calledWithVarArgArrayParam) { - this.calledWithVarArgArrayParam = calledWithVarArgArrayParam; - } - public boolean isSystemFunction(String name) { return this.getName().equalsIgnoreCase(name) && CoreConstants.SYSTEM_MODEL.equals(this.getSchema()); } diff --git a/engine/src/main/java/org/teiid/query/metadata/BasicQueryMetadata.java b/engine/src/main/java/org/teiid/query/metadata/BasicQueryMetadata.java index 3f093bf4443..025ef9cb0fa 100644 --- a/engine/src/main/java/org/teiid/query/metadata/BasicQueryMetadata.java +++ b/engine/src/main/java/org/teiid/query/metadata/BasicQueryMetadata.java @@ -31,6 +31,7 @@ import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidProcessingException; import org.teiid.core.types.DataTypeManager; +import org.teiid.metadata.FunctionMethod; import org.teiid.query.eval.TeiidScriptEngine; import org.teiid.query.function.FunctionLibrary; import org.teiid.query.mapping.relational.QueryNode; @@ -541,4 +542,9 @@ public List getModelIDs() { return Collections.emptyList(); } + @Override + public FunctionMethod getPushdownFunction(Object modelID, String fullName) { + return null; + } + } diff --git a/engine/src/main/java/org/teiid/query/metadata/BasicQueryMetadataWrapper.java b/engine/src/main/java/org/teiid/query/metadata/BasicQueryMetadataWrapper.java index 7c5a63e01cb..a0911aaa555 100644 --- a/engine/src/main/java/org/teiid/query/metadata/BasicQueryMetadataWrapper.java +++ b/engine/src/main/java/org/teiid/query/metadata/BasicQueryMetadataWrapper.java @@ -29,6 +29,7 @@ import org.teiid.api.exception.query.QueryMetadataException; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidProcessingException; +import org.teiid.metadata.FunctionMethod; import org.teiid.query.function.FunctionLibrary; import org.teiid.query.mapping.relational.QueryNode; import org.teiid.query.sql.symbol.Expression; @@ -437,4 +438,9 @@ public List getModelIDs() { return actualMetadata.getModelIDs(); } + @Override + public FunctionMethod getPushdownFunction(Object modelID, String fullName) { + return actualMetadata.getPushdownFunction(modelID, fullName); + } + } diff --git a/engine/src/main/java/org/teiid/query/metadata/MetadataValidator.java b/engine/src/main/java/org/teiid/query/metadata/MetadataValidator.java index a4719f12ba7..82c8473aeb2 100644 --- a/engine/src/main/java/org/teiid/query/metadata/MetadataValidator.java +++ b/engine/src/main/java/org/teiid/query/metadata/MetadataValidator.java @@ -82,6 +82,7 @@ import org.teiid.metadata.Table.TriggerEvent; import org.teiid.metadata.Trigger; import org.teiid.query.QueryPlugin; +import org.teiid.query.function.FunctionDescriptor; import org.teiid.query.function.metadata.FunctionMetadataValidator; import org.teiid.query.mapping.relational.QueryNode; import org.teiid.query.metadata.MaterializationMetadataRepository.Scope; @@ -321,6 +322,12 @@ public void execute(VDBMetaData vdb, MetadataStore store, ValidatorReport report } + QueryMetadataInterface metadata = vdb.getAttachment(QueryMetadataInterface.class); + PushdownFunctions pushdownFunctions = null; + if (schema.isPhysical()) { + pushdownFunctions = new PushdownFunctions(); + model.addAttachment(PushdownFunctions.class, pushdownFunctions); + } for (FunctionMethod func:schema.getFunctions().values()) { for (FunctionParameter param : func.getInputParameters()) { if (param.isVarArg() && param != func.getInputParameters().get(func.getInputParameterCount() -1)) { @@ -330,6 +337,27 @@ public void execute(VDBMetaData vdb, MetadataStore store, ValidatorReport report if (func.getPushdown().equals(FunctionMethod.PushDown.MUST_PUSHDOWN) && !model.isSource()) { metadataValidator.log(report, model, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31078, func.getFullName(), model.getName())); } + + if (pushdownFunctions != null) { + String virtualName = func.getProperty(FunctionMethod.VIRTUAL_FUNCTION); + + if (virtualName != null) { + Class[] types = new Class[func.getInputParameterCount()]; + int i = 0; + for (FunctionParameter fp : func.getInputParameters()) { + types[i++] = fp.getJavaType(); + } + FunctionDescriptor virtualFunction = metadata.getFunctionLibrary().findFunction(virtualName, types); + //if it exists and is a proper match, add it to the pushdown mapping + if (virtualFunction != null + //&& !virtualFunction.getMethod().getParent().isPhysical() -- sys/sysadmin are physical, so we won't check this just yet + && !(func.isVarArgs() ^ virtualFunction.getMethod().isVarArgs())) { + pushdownFunctions.put(virtualName, func); + } else { + metadataValidator.log(report, model, Severity.WARNING, QueryPlugin.Util.gs(QueryPlugin.Event.TEIID31305, virtualName, func.getFullName()), func); + } + } + } } } } diff --git a/engine/src/main/java/org/teiid/query/metadata/PushdownFunctions.java b/engine/src/main/java/org/teiid/query/metadata/PushdownFunctions.java new file mode 100644 index 00000000000..e8bd294fd49 --- /dev/null +++ b/engine/src/main/java/org/teiid/query/metadata/PushdownFunctions.java @@ -0,0 +1,32 @@ +/* + * Copyright Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags and + * the COPYRIGHT.txt file distributed with this work. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.teiid.query.metadata; + +import java.util.TreeMap; + +import org.teiid.metadata.FunctionMethod; + +class PushdownFunctions extends TreeMap { + private static final long serialVersionUID = 1L; + + public PushdownFunctions() { + super(String.CASE_INSENSITIVE_ORDER); + } + +} \ No newline at end of file diff --git a/engine/src/main/java/org/teiid/query/metadata/QueryMetadataInterface.java b/engine/src/main/java/org/teiid/query/metadata/QueryMetadataInterface.java index 4735321c7ee..66d4bd3fef2 100644 --- a/engine/src/main/java/org/teiid/query/metadata/QueryMetadataInterface.java +++ b/engine/src/main/java/org/teiid/query/metadata/QueryMetadataInterface.java @@ -29,6 +29,7 @@ import org.teiid.api.exception.query.QueryMetadataException; import org.teiid.core.TeiidComponentException; import org.teiid.core.TeiidProcessingException; +import org.teiid.metadata.FunctionMethod; import org.teiid.query.function.FunctionLibrary; import org.teiid.query.mapping.relational.QueryNode; import org.teiid.query.sql.symbol.Expression; @@ -643,4 +644,6 @@ String getExtensionProperty(Object metadataID, String key, boolean isLongRanks(); List getModelIDs(); + + FunctionMethod getPushdownFunction(Object modelID, String fullName); } diff --git a/engine/src/main/java/org/teiid/query/metadata/TransformationMetadata.java b/engine/src/main/java/org/teiid/query/metadata/TransformationMetadata.java index e001fa4a152..08c2ea08e7e 100644 --- a/engine/src/main/java/org/teiid/query/metadata/TransformationMetadata.java +++ b/engine/src/main/java/org/teiid/query/metadata/TransformationMetadata.java @@ -22,7 +22,17 @@ import java.io.IOException; import java.io.InputStream; import java.io.Serializable; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; import javax.script.ScriptContext; @@ -31,6 +41,7 @@ import javax.script.ScriptEngineManager; import org.teiid.adminapi.impl.DataPolicyMetadata; +import org.teiid.adminapi.impl.ModelMetaData; import org.teiid.adminapi.impl.VDBMetaData; import org.teiid.api.exception.query.QueryMetadataException; import org.teiid.core.TeiidComponentException; @@ -46,10 +57,21 @@ import org.teiid.core.util.ObjectConverterUtil; import org.teiid.core.util.StringUtil; import org.teiid.dqp.internal.process.DQPWorkContext; -import org.teiid.metadata.*; +import org.teiid.metadata.AbstractMetadataRecord; import org.teiid.metadata.BaseColumn.NullType; +import org.teiid.metadata.Column; import org.teiid.metadata.Column.SearchType; +import org.teiid.metadata.ColumnSet; +import org.teiid.metadata.Datatype; +import org.teiid.metadata.ForeignKey; +import org.teiid.metadata.FunctionMethod; +import org.teiid.metadata.FunctionParameter; +import org.teiid.metadata.KeyRecord; +import org.teiid.metadata.Procedure; +import org.teiid.metadata.ProcedureParameter; import org.teiid.metadata.ProcedureParameter.Type; +import org.teiid.metadata.Schema; +import org.teiid.metadata.Table; import org.teiid.query.QueryPlugin; import org.teiid.query.function.FunctionLibrary; import org.teiid.query.function.FunctionTree; @@ -1140,4 +1162,21 @@ public void setLongRanks(boolean longRanks) { this.longRanks = longRanks; } + @Override + public FunctionMethod getPushdownFunction(Object modelID, String fullName) { + if (vdbMetaData == null || !(modelID instanceof Schema)) { + return null; + } + Schema schema = (Schema)modelID; + ModelMetaData model = vdbMetaData.getModel(schema.getName()); + if (model == null) { + return null; + } + PushdownFunctions functions = model.getAttachment(PushdownFunctions.class); + if (functions == null) { + return null; + } + return functions.get(fullName); + } + } \ No newline at end of file diff --git a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CapabilitiesUtil.java b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CapabilitiesUtil.java index e7117cd0887..fcf78a48e85 100644 --- a/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CapabilitiesUtil.java +++ b/engine/src/main/java/org/teiid/query/optimizer/relational/rules/CapabilitiesUtil.java @@ -287,6 +287,14 @@ public static boolean supportsScalarFunction(Object modelID, Function function, return (schema == null && caps.supportsFunction(SourceSystemFunctions.TIMESTAMPDIFF)); } else { + FunctionMethod functionMethod = metadata.getPushdownFunction(modelID, fullName); + if (functionMethod != null) { + //it's not great that we're setting this as a side-effect, but + //it's easier that attempting to re-associate at the connector + //or another level + function.setPushdownFunction(functionMethod); + return true; + } return false ; } } diff --git a/engine/src/main/java/org/teiid/query/resolver/util/ResolverVisitor.java b/engine/src/main/java/org/teiid/query/resolver/util/ResolverVisitor.java index 58237a57de1..86cf79f8403 100644 --- a/engine/src/main/java/org/teiid/query/resolver/util/ResolverVisitor.java +++ b/engine/src/main/java/org/teiid/query/resolver/util/ResolverVisitor.java @@ -46,11 +46,37 @@ import org.teiid.query.metadata.TempMetadataID; import org.teiid.query.sql.LanguageObject; import org.teiid.query.sql.LanguageVisitor; -import org.teiid.query.sql.lang.*; +import org.teiid.query.sql.lang.BetweenCriteria; +import org.teiid.query.sql.lang.BinaryComparison; +import org.teiid.query.sql.lang.CompareCriteria; +import org.teiid.query.sql.lang.ExpressionCriteria; +import org.teiid.query.sql.lang.GroupContext; +import org.teiid.query.sql.lang.IsDistinctCriteria; +import org.teiid.query.sql.lang.IsNullCriteria; +import org.teiid.query.sql.lang.MatchCriteria; +import org.teiid.query.sql.lang.SetClause; +import org.teiid.query.sql.lang.SetCriteria; +import org.teiid.query.sql.lang.SubqueryCompareCriteria; +import org.teiid.query.sql.lang.SubquerySetCriteria; import org.teiid.query.sql.navigator.PostOrderNavigator; import org.teiid.query.sql.proc.ExceptionExpression; -import org.teiid.query.sql.symbol.*; +import org.teiid.query.sql.symbol.AggregateSymbol; +import org.teiid.query.sql.symbol.Array; +import org.teiid.query.sql.symbol.CaseExpression; +import org.teiid.query.sql.symbol.Constant; +import org.teiid.query.sql.symbol.DerivedColumn; +import org.teiid.query.sql.symbol.ElementSymbol; import org.teiid.query.sql.symbol.ElementSymbol.DisplayMode; +import org.teiid.query.sql.symbol.Expression; +import org.teiid.query.sql.symbol.Function; +import org.teiid.query.sql.symbol.GroupSymbol; +import org.teiid.query.sql.symbol.QueryString; +import org.teiid.query.sql.symbol.Reference; +import org.teiid.query.sql.symbol.SearchedCaseExpression; +import org.teiid.query.sql.symbol.XMLCast; +import org.teiid.query.sql.symbol.XMLExists; +import org.teiid.query.sql.symbol.XMLQuery; +import org.teiid.query.sql.symbol.XMLSerialize; public class ResolverVisitor extends LanguageVisitor { @@ -796,8 +822,7 @@ void resolveFunction(Function function, FunctionLibrary library) if (fd.getMethod().isVarArgs() && fd.getTypes().length == types.length && library.isVarArgArrayParam(fd.getMethod(), types, types.length - 1, fd.getTypes()[types.length - 1])) { - fd = fd.clone(); - fd.setCalledWithVarArgArrayParam(true); + function.setCalledWithVarArgArrayParam(true); } if(FunctionLibrary.isConvert(function)) { diff --git a/engine/src/main/java/org/teiid/query/sql/symbol/Function.java b/engine/src/main/java/org/teiid/query/sql/symbol/Function.java index 9b6f208a6b8..f89525b9f80 100644 --- a/engine/src/main/java/org/teiid/query/sql/symbol/Function.java +++ b/engine/src/main/java/org/teiid/query/sql/symbol/Function.java @@ -21,6 +21,7 @@ import org.teiid.core.types.DataTypeManager; import org.teiid.core.util.EquivalenceUtil; import org.teiid.core.util.HashCodeUtil; +import org.teiid.metadata.FunctionMethod; import org.teiid.query.function.FunctionDescriptor; import org.teiid.query.sql.LanguageObject; import org.teiid.query.sql.LanguageVisitor; @@ -42,6 +43,9 @@ public class Function implements NamedExpression { private boolean implicit = false; private boolean eval = true; + private boolean calledWithVarArgArrayParam; + private FunctionMethod pushdownFunction; + /** * Construct a function with function name and array of arguments. For * functions that have no args, pass empty array, not null. @@ -231,6 +235,8 @@ public Object clone() { copy.makeImplicit(); } copy.eval = this.eval; + copy.calledWithVarArgArrayParam = this.calledWithVarArgArrayParam; + copy.pushdownFunction = this.pushdownFunction; return copy; } @@ -254,4 +260,20 @@ public void setEval(boolean eval) { this.eval = eval; } + public boolean isCalledWithVarArgArrayParam() { + return calledWithVarArgArrayParam; + } + + public void setCalledWithVarArgArrayParam(boolean calledWithVarArgArrayParam) { + this.calledWithVarArgArrayParam = calledWithVarArgArrayParam; + } + + public FunctionMethod getPushdownFunction() { + return pushdownFunction; + } + + public void setPushdownFunction(FunctionMethod pushdownFunction) { + this.pushdownFunction = pushdownFunction; + } + } diff --git a/engine/src/main/resources/org/teiid/query/i18n.properties b/engine/src/main/resources/org/teiid/query/i18n.properties index 166c942df3c..4158a353569 100644 --- a/engine/src/main/resources/org/teiid/query/i18n.properties +++ b/engine/src/main/resources/org/teiid/query/i18n.properties @@ -1664,4 +1664,5 @@ TEIID31302=Materialized View {0} does not have the partitioning column ''{1}''. TEIID31303=Materialized View {0} has a partitioning column ''{1}'' with a non-comparable type. TEIID31304=Materialized View {0} partitioning values query ''{1}'' should have a single projected column with a type that matches the partitioning column ''{2}''. +TEIID31305=Virtual function {0} does not exist or does not match the metadata for {1}. It will not be pushed down as {1}. diff --git a/engine/src/test/java/org/teiid/query/processor/TestFunctionPushdown.java b/engine/src/test/java/org/teiid/query/processor/TestFunctionPushdown.java index c4e277deca5..cdd110c3f4f 100644 --- a/engine/src/test/java/org/teiid/query/processor/TestFunctionPushdown.java +++ b/engine/src/test/java/org/teiid/query/processor/TestFunctionPushdown.java @@ -18,8 +18,10 @@ package org.teiid.query.processor; -import static org.junit.Assert.*; -import static org.teiid.query.optimizer.TestOptimizer.*; +import static org.junit.Assert.fail; +import static org.teiid.query.optimizer.TestOptimizer.checkNodeTypes; +import static org.teiid.query.optimizer.TestOptimizer.getTypicalCapabilities; +import static org.teiid.query.optimizer.TestOptimizer.helpPlan; import java.util.Arrays; import java.util.Collections; diff --git a/olingo-common/src/main/java/org/teiid/olingo/common/JTS2OlingoBridge.java b/olingo-common/src/main/java/org/teiid/olingo/common/JTS2OlingoBridge.java index 020493e402b..849bfae81e0 100644 --- a/olingo-common/src/main/java/org/teiid/olingo/common/JTS2OlingoBridge.java +++ b/olingo-common/src/main/java/org/teiid/olingo/common/JTS2OlingoBridge.java @@ -52,9 +52,7 @@ public Geospatial convert(Geometry geometry) { return result; } else if (geometry instanceof LineString) { LineString lineString = (LineString) geometry; - ArrayList points = convertLineStringToPoints(lineString.getCoordinates()); - org.apache.olingo.commons.api.edm.geo.LineString result = new org.apache.olingo.commons.api.edm.geo.LineString(dimension, srid, points); - return result; + return convertLineString(lineString.getCoordinates()); } else if (geometry instanceof Polygon) { Polygon polygon = (Polygon) geometry; return convertPolygon(polygon); @@ -68,7 +66,7 @@ public Geospatial convert(Geometry geometry) { List lineStrings = new ArrayList<>(multiLineString.getNumGeometries()); for (int i = 0; i < multiLineString.getNumGeometries(); i++) { LineString lineString = (LineString)multiLineString.getGeometryN(i); - lineStrings.add(new org.apache.olingo.commons.api.edm.geo.LineString(dimension, srid, convertLineStringToPoints(lineString.getCoordinates()))); + lineStrings.add(convertLineString(lineString.getCoordinates())); } org.apache.olingo.commons.api.edm.geo.MultiLineString result = new org.apache.olingo.commons.api.edm.geo.MultiLineString(dimension, srid, lineStrings); return result; diff --git a/runtime/src/test/java/org/teiid/runtime/HardCodedExecutionFactory.java b/runtime/src/test/java/org/teiid/runtime/HardCodedExecutionFactory.java index a224a4458ac..8e5f1b7550b 100644 --- a/runtime/src/test/java/org/teiid/runtime/HardCodedExecutionFactory.java +++ b/runtime/src/test/java/org/teiid/runtime/HardCodedExecutionFactory.java @@ -192,4 +192,8 @@ public List getCommands() { return commands; } + public void clearData() { + this.dataMap.clear(); + } + } \ No newline at end of file diff --git a/runtime/src/test/java/org/teiid/runtime/TestVirtualFunctionPushdown.java b/runtime/src/test/java/org/teiid/runtime/TestVirtualFunctionPushdown.java new file mode 100644 index 00000000000..2f78ef2db54 --- /dev/null +++ b/runtime/src/test/java/org/teiid/runtime/TestVirtualFunctionPushdown.java @@ -0,0 +1,108 @@ +/* + * Copyright Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags and + * the COPYRIGHT.txt file distributed with this work. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.teiid.runtime; + +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; + +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressWarnings("nls") +public class TestVirtualFunctionPushdown { + + private static EmbeddedServer es = new EmbeddedServer(); + private Connection conn; + private static HardCodedExecutionFactory ef; + + @BeforeClass public static void beforeClass() throws Exception { + es.start(new EmbeddedConfiguration()); + ef = new HardCodedExecutionFactory() { + @Override + public boolean supportsSelectExpression() { + return true; + } + }; + es.addTranslator("hardcoded", ef); + es.deployVDB(new ByteArrayInputStream(("CREATE DATABASE test VERSION '1';" + + "USE DATABASE test VERSION '1';" + + "CREATE SERVER s FOREIGN DATA WRAPPER hardcoded;" + + "CREATE VIRTUAL SCHEMA v; " + + "CREATE SCHEMA p SERVER s; " + + "SET SCHEMA v;" + + "CREATE VIRTUAL FUNCTION VFUNC(msg varchar) RETURNS varchar AS return msg || 'a';" + + "CREATE VIRTUAL FUNCTION VFUNC1(msg varchar) RETURNS varchar AS return msg || 'a';" + + "SET SCHEMA p;" + + "CREATE FOREIGN FUNCTION SFUNC(msg varchar) " + + " RETURNS varchar OPTIONS (\"teiid_rel:virtual-function\" 'v.vfunc');" + + "CREATE FOREIGN FUNCTION IFUNC(msg integer) RETURNS varchar OPTIONS (\"teiid_rel:virtual-function\" 'v.vfunc1');" + + "CREATE FOREIGN FUNCTION SYSFUNC(msg string) RETURNS integer OPTIONS (\"teiid_rel:virtual-function\" 'ascii');" + + "CREATE FOREIGN TABLE TBL (COL STRING);").getBytes()), true); + } + + @Before public void before() throws Exception { + conn = es.getDriver().connect("jdbc:teiid:test", null); + ef.clearData(); + } + + @AfterClass public static void afterClass() { + es.stop(); + } + + @After public void after() throws Exception { + if (conn != null) { + conn.close(); + } + } + + @Test public void testPushdown() throws SQLException { + //note the substitution of sfunc for vfunc + ef.addData("SELECT SFUNC(TBL.COL) FROM TBL", Arrays.asList(Arrays.asList("a"))); + try (Statement statement = conn.createStatement();) { + ResultSet rs = statement.executeQuery("select vfunc(col) from tbl"); + assertTrue(rs.next()); + } + } + + @Test public void testNoPushdown() throws SQLException { + ef.addData("SELECT TBL.COL FROM TBL", Arrays.asList(Arrays.asList("a"))); + try (Statement statement = conn.createStatement();) { + ResultSet rs = statement.executeQuery("select vfunc1(col) from tbl"); + assertTrue(rs.next()); + } + } + + @Test public void testSysPushdown() throws SQLException { + ef.addData("SELECT SYSFUNC(TBL.COL) FROM TBL", Arrays.asList(Arrays.asList(1))); + try (Statement statement = conn.createStatement();) { + ResultSet rs = statement.executeQuery("select ascii(col) from tbl"); + assertTrue(rs.next()); + } + } + +} \ No newline at end of file