-
Notifications
You must be signed in to change notification settings - Fork 139
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Refactor OpenSearch Relevance Functions out of Core #2019
base: main
Are you sure you want to change the base?
Changes from 5 commits
490669b
8296478
b7f3b24
e4fa5fc
0fb46fc
f008573
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,83 @@ | ||
package org.opensearch.sql.datasource.model; | ||
|
||
import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.DEFAULT_DATASOURCE_NAME; | ||
import static org.opensearch.sql.data.type.ExprCoreType.STRING; | ||
|
||
import com.google.common.collect.ImmutableMap; | ||
import java.util.Map; | ||
import java.util.Set; | ||
import org.opensearch.sql.data.type.ExprType; | ||
import org.opensearch.sql.datasource.DataSourceService; | ||
import org.opensearch.sql.planner.logical.LogicalPlan; | ||
import org.opensearch.sql.planner.physical.PhysicalPlan; | ||
import org.opensearch.sql.storage.StorageEngine; | ||
import org.opensearch.sql.storage.Table; | ||
|
||
public class EmptyDataSourceService { | ||
private static DataSourceService emptyDataSourceService = | ||
new DataSourceService() { | ||
@Override | ||
public DataSource getDataSource(String dataSourceName) { | ||
return new DataSource( | ||
DEFAULT_DATASOURCE_NAME, DataSourceType.OPENSEARCH, storageEngine()); | ||
} | ||
|
||
@Override | ||
public Set<DataSourceMetadata> getDataSourceMetadata(boolean isDefaultDataSourceRequired) { | ||
return Set.of(); | ||
} | ||
|
||
@Override | ||
public DataSourceMetadata getDataSourceMetadata(String name) { | ||
return null; | ||
} | ||
|
||
@Override | ||
public void createDataSource(DataSourceMetadata metadata) {} | ||
|
||
@Override | ||
public void updateDataSource(DataSourceMetadata dataSourceMetadata) {} | ||
|
||
@Override | ||
public void deleteDataSource(String dataSourceName) {} | ||
|
||
@Override | ||
public Boolean dataSourceExists(String dataSourceName) { | ||
return null; | ||
} | ||
}; | ||
|
||
private static StorageEngine storageEngine() { | ||
Table table = | ||
new Table() { | ||
@Override | ||
public boolean exists() { | ||
return true; | ||
} | ||
|
||
@Override | ||
public void create(Map<String, ExprType> schema) { | ||
throw new UnsupportedOperationException("Create table is not supported"); | ||
} | ||
|
||
@Override | ||
public Map<String, ExprType> getFieldTypes() { | ||
return null; | ||
} | ||
|
||
@Override | ||
public PhysicalPlan implement(LogicalPlan plan) { | ||
throw new UnsupportedOperationException(); | ||
} | ||
|
||
public Map<String, ExprType> getReservedFieldTypes() { | ||
return ImmutableMap.of("_test", STRING); | ||
} | ||
}; | ||
return (dataSourceSchemaName, tableName) -> table; | ||
} | ||
|
||
public static DataSourceService getEmptyDataSourceService() { | ||
return emptyDataSourceService; | ||
} | ||
} |
Original file line number | Diff line number | Diff line change | ||||||||
---|---|---|---|---|---|---|---|---|---|---|
|
@@ -16,12 +16,15 @@ | |||||||||
import java.util.List; | ||||||||||
import java.util.Map; | ||||||||||
import java.util.Optional; | ||||||||||
import java.util.Set; | ||||||||||
import java.util.function.Function; | ||||||||||
import java.util.stream.Collectors; | ||||||||||
import org.apache.commons.lang3.tuple.Pair; | ||||||||||
import org.opensearch.sql.common.utils.StringUtils; | ||||||||||
import org.opensearch.sql.data.type.ExprCoreType; | ||||||||||
import org.opensearch.sql.data.type.ExprType; | ||||||||||
import org.opensearch.sql.datasource.DataSourceService; | ||||||||||
import org.opensearch.sql.datasource.model.DataSourceMetadata; | ||||||||||
import org.opensearch.sql.exception.ExpressionEvaluationException; | ||||||||||
import org.opensearch.sql.expression.Expression; | ||||||||||
import org.opensearch.sql.expression.aggregation.AggregatorFunction; | ||||||||||
|
@@ -46,7 +49,7 @@ public class BuiltinFunctionRepository { | |||||||||
private final Map<FunctionName, FunctionResolver> functionResolverMap; | ||||||||||
|
||||||||||
/** The singleton instance. */ | ||||||||||
private static BuiltinFunctionRepository instance; | ||||||||||
private static final Map<Integer, BuiltinFunctionRepository> instance = new HashMap<>(); | ||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Extend the comment and probably rename to |
||||||||||
|
||||||||||
/** | ||||||||||
* Construct a function repository with the given function registered. This is only used in test. | ||||||||||
|
@@ -64,25 +67,45 @@ public class BuiltinFunctionRepository { | |||||||||
* | ||||||||||
* @return singleton instance | ||||||||||
*/ | ||||||||||
public static synchronized BuiltinFunctionRepository getInstance() { | ||||||||||
if (instance == null) { | ||||||||||
instance = new BuiltinFunctionRepository(new HashMap<>()); | ||||||||||
public static synchronized BuiltinFunctionRepository getInstance( | ||||||||||
DataSourceService dataSourceService) { | ||||||||||
Set<DataSourceMetadata> dataSourceMetadataSet = dataSourceService.getDataSourceMetadata(true); | ||||||||||
Set<Integer> dataSourceServiceHashSet = | ||||||||||
dataSourceMetadataSet.stream() | ||||||||||
.map(metadata -> metadata.hashCode()) | ||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think it could be optimized to
Suggested change
|
||||||||||
.collect(Collectors.toSet()); | ||||||||||
|
||||||||||
// Creates new Repository for every dataSourceService | ||||||||||
if (!dataSourceServiceHashSet.stream().anyMatch(hash -> instance.containsKey(hash))) { | ||||||||||
BuiltinFunctionRepository repository = new BuiltinFunctionRepository(new HashMap<>()); | ||||||||||
|
||||||||||
// Register all built-in functions | ||||||||||
ArithmeticFunction.register(instance); | ||||||||||
BinaryPredicateOperator.register(instance); | ||||||||||
MathematicalFunction.register(instance); | ||||||||||
UnaryPredicateOperator.register(instance); | ||||||||||
AggregatorFunction.register(instance); | ||||||||||
DateTimeFunction.register(instance); | ||||||||||
IntervalClause.register(instance); | ||||||||||
WindowFunctions.register(instance); | ||||||||||
TextFunction.register(instance); | ||||||||||
TypeCastOperator.register(instance); | ||||||||||
SystemFunctions.register(instance); | ||||||||||
OpenSearchFunctions.register(instance); | ||||||||||
MitchellGale marked this conversation as resolved.
Show resolved
Hide resolved
|
||||||||||
ArithmeticFunction.register(repository); | ||||||||||
BinaryPredicateOperator.register(repository); | ||||||||||
MathematicalFunction.register(repository); | ||||||||||
UnaryPredicateOperator.register(repository); | ||||||||||
AggregatorFunction.register(repository); | ||||||||||
DateTimeFunction.register(repository); | ||||||||||
IntervalClause.register(repository); | ||||||||||
WindowFunctions.register(repository); | ||||||||||
TextFunction.register(repository); | ||||||||||
TypeCastOperator.register(repository); | ||||||||||
SystemFunctions.register(repository); | ||||||||||
// Temporary as part of https://github.com/opensearch-project/sql/issues/811 | ||||||||||
// TODO: remove this resolver when Analyzers are moved to opensearch module | ||||||||||
repository.register(new NestedFunctionResolver()); | ||||||||||
|
||||||||||
for (DataSourceMetadata metadata : dataSourceMetadataSet) { | ||||||||||
dataSourceService | ||||||||||
.getDataSource(metadata.getName()) | ||||||||||
.getStorageEngine() | ||||||||||
.getFunctions() | ||||||||||
.forEach(function -> repository.register(function)); | ||||||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||||
instance.put(metadata.hashCode(), repository); | ||||||||||
} | ||||||||||
Comment on lines
+104
to
+105
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||||
return repository; | ||||||||||
} | ||||||||||
return instance; | ||||||||||
return instance.get(dataSourceServiceHashSet.iterator().next()); | ||||||||||
} | ||||||||||
|
||||||||||
/** | ||||||||||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
/* | ||
* Copyright OpenSearch Contributors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package org.opensearch.sql.expression.function; | ||
|
||
import org.apache.commons.lang3.tuple.Pair; | ||
import org.opensearch.sql.data.model.ExprValue; | ||
import org.opensearch.sql.data.type.ExprType; | ||
import org.opensearch.sql.expression.Expression; | ||
import org.opensearch.sql.expression.FunctionExpression; | ||
import org.opensearch.sql.expression.env.Environment; | ||
|
||
/** | ||
* Nested Function Resolver returns a builder to resolve nested function expressions | ||
*/ | ||
public class NestedFunctionResolver implements FunctionResolver { | ||
acarbonetto marked this conversation as resolved.
Show resolved
Hide resolved
|
||
@Override | ||
public Pair<FunctionSignature, FunctionBuilder> resolve(FunctionSignature unresolvedSignature) { | ||
return Pair.of( | ||
unresolvedSignature, | ||
(functionProperties, arguments) -> | ||
new FunctionExpression(BuiltinFunctionName.NESTED.getName(), arguments) { | ||
@Override | ||
public ExprValue valueOf(Environment<Expression, ExprValue> valueEnv) { | ||
return valueEnv.resolve(getArguments().get(0)); | ||
} | ||
|
||
@Override | ||
public ExprType type() { | ||
return getArguments().get(0).type(); | ||
} | ||
}); | ||
} | ||
|
||
@Override | ||
public FunctionName getFunctionName() { | ||
return BuiltinFunctionName.NESTED.getName(); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Need a javadoc and a good description why it is here