Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add typeof function. #867

Merged
merged 6 commits into from
Oct 20, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions core/src/main/java/org/opensearch/sql/expression/DSL.java
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,13 @@
import org.opensearch.sql.data.model.ExprShortValue;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.data.model.ExprValueUtils;
import org.opensearch.sql.data.type.ExprCoreType;
import org.opensearch.sql.data.type.ExprType;
import org.opensearch.sql.expression.aggregation.Aggregator;
import org.opensearch.sql.expression.aggregation.NamedAggregator;
import org.opensearch.sql.expression.conditional.cases.CaseClause;
import org.opensearch.sql.expression.conditional.cases.WhenClause;
import org.opensearch.sql.expression.env.Environment;
import org.opensearch.sql.expression.function.BuiltinFunctionName;
import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
import org.opensearch.sql.expression.span.SpanExpression;
Expand Down Expand Up @@ -662,6 +664,11 @@ public FunctionExpression castDatetime(Expression value) {
.compile(BuiltinFunctionName.CAST_TO_DATETIME.getName(), Arrays.asList(value));
}

public FunctionExpression typeof(Expression value) {
return (FunctionExpression) repository
.compile(BuiltinFunctionName.TYPEOF.getName(), Arrays.asList(value));
}

public FunctionExpression match(Expression... args) {
return compile(BuiltinFunctionName.MATCH, args);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.opensearch.sql.expression.operator.convert.TypeCastOperator;
import org.opensearch.sql.expression.operator.predicate.BinaryPredicateOperator;
import org.opensearch.sql.expression.operator.predicate.UnaryPredicateOperator;
import org.opensearch.sql.expression.system.SystemFunctions;
import org.opensearch.sql.expression.text.TextFunction;
import org.opensearch.sql.expression.window.WindowFunctions;
import org.springframework.context.annotation.Bean;
Expand Down Expand Up @@ -45,6 +46,7 @@ public BuiltinFunctionRepository functionRepository() {
WindowFunctions.register(builtinFunctionRepository);
TextFunction.register(builtinFunctionRepository);
TypeCastOperator.register(builtinFunctionRepository);
SystemFunctions.register(builtinFunctionRepository);
OpenSearchFunctions.register(builtinFunctionRepository);
return builtinFunctionRepository;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ public enum BuiltinFunctionName {
CAST_TO_TIME(FunctionName.of("cast_to_time")),
CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")),
CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")),
TYPEOF(FunctionName.of("typeof")),

/**
* Relevance Function.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.expression.system;

import static org.opensearch.sql.data.type.ExprCoreType.STRING;

import lombok.experimental.UtilityClass;
import org.apache.commons.lang3.tuple.Pair;
import org.opensearch.sql.data.model.ExprStringValue;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.data.type.ExprType;
import org.opensearch.sql.expression.Expression;
import org.opensearch.sql.expression.FunctionExpression;
import org.opensearch.sql.expression.env.Environment;
import org.opensearch.sql.expression.function.BuiltinFunctionName;
import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
import org.opensearch.sql.expression.function.FunctionBuilder;
import org.opensearch.sql.expression.function.FunctionName;
import org.opensearch.sql.expression.function.FunctionResolver;
import org.opensearch.sql.expression.function.FunctionSignature;

@UtilityClass
public class SystemFunctions {
/**
* Register TypeOf Operator.
*/
public static void register(BuiltinFunctionRepository repository) {
repository.register(typeof());
}

// Auxiliary function useful for debugging
private static FunctionResolver typeof() {
return new FunctionResolver() {
@Override
public Pair<FunctionSignature, FunctionBuilder> resolve(
FunctionSignature unresolvedSignature) {
return Pair.of(unresolvedSignature,
arguments -> new FunctionExpression(BuiltinFunctionName.TYPEOF.getName(), arguments) {
@Override
public ExprValue valueOf(Environment<Expression, ExprValue> valueEnv) {
return new ExprStringValue(getArguments().get(0).type().toString());
}

@Override
public ExprType type() {
return STRING;
}
});
}

@Override
public FunctionName getFunctionName() {
return BuiltinFunctionName.TYPEOF.getName();
}
};
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.expression.system;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.opensearch.sql.data.type.ExprCoreType.STRING;

import java.time.Duration;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.util.LinkedHashMap;
import java.util.List;
import org.junit.jupiter.api.Test;
import org.opensearch.sql.data.model.AbstractExprValue;
import org.opensearch.sql.data.model.ExprBooleanValue;
import org.opensearch.sql.data.model.ExprByteValue;
import org.opensearch.sql.data.model.ExprCollectionValue;
import org.opensearch.sql.data.model.ExprDateValue;
import org.opensearch.sql.data.model.ExprDatetimeValue;
import org.opensearch.sql.data.model.ExprDoubleValue;
import org.opensearch.sql.data.model.ExprFloatValue;
import org.opensearch.sql.data.model.ExprIntegerValue;
import org.opensearch.sql.data.model.ExprIntervalValue;
import org.opensearch.sql.data.model.ExprLongValue;
import org.opensearch.sql.data.model.ExprMissingValue;
import org.opensearch.sql.data.model.ExprNullValue;
import org.opensearch.sql.data.model.ExprShortValue;
import org.opensearch.sql.data.model.ExprStringValue;
import org.opensearch.sql.data.model.ExprTimeValue;
import org.opensearch.sql.data.model.ExprTimestampValue;
import org.opensearch.sql.data.model.ExprTupleValue;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.data.type.ExprCoreType;
import org.opensearch.sql.data.type.ExprType;
import org.opensearch.sql.expression.DSL;
import org.opensearch.sql.expression.config.ExpressionConfig;

public class SystemFunctionsTest {
private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository());

@Test
void typeof() {
assertEquals(STRING, dsl.typeof(DSL.literal(1)).type());

assertEquals("ARRAY", typeofGetValue(new ExprCollectionValue(List.of())));
assertEquals("BOOLEAN", typeofGetValue(ExprBooleanValue.of(false)));
assertEquals("BYTE", typeofGetValue(new ExprByteValue(0)));
assertEquals("DATE", typeofGetValue(new ExprDateValue(LocalDate.now())));
assertEquals("DATETIME", typeofGetValue(new ExprDatetimeValue(LocalDateTime.now())));
assertEquals("DOUBLE", typeofGetValue(new ExprDoubleValue(0)));
assertEquals("FLOAT", typeofGetValue(new ExprFloatValue(0)));
assertEquals("INTEGER", typeofGetValue(new ExprIntegerValue(0)));
assertEquals("INTERVAL", typeofGetValue(new ExprIntervalValue(Duration.ofDays(0))));
assertEquals("LONG", typeofGetValue(new ExprLongValue(0)));
assertEquals("SHORT", typeofGetValue(new ExprShortValue(0)));
assertEquals("STRING", typeofGetValue(new ExprStringValue("")));
assertEquals("STRUCT", typeofGetValue(new ExprTupleValue(new LinkedHashMap<>())));
assertEquals("TIME", typeofGetValue(new ExprTimeValue(LocalTime.now())));
assertEquals("TIMESTAMP", typeofGetValue(new ExprTimestampValue(Instant.now())));
assertEquals("UNDEFINED", typeofGetValue(ExprNullValue.of()));
assertEquals("UNDEFINED", typeofGetValue(ExprMissingValue.of()));
assertEquals("UNKNOWN", typeofGetValue(new AbstractExprValue() {
@Override
public int compare(ExprValue other) {
return 0;
}

@Override
public boolean equal(ExprValue other) {
return false;
}

@Override
public Object value() {
return null;
}

@Override
public ExprType type() {
return ExprCoreType.UNKNOWN;
}
}));
}

private String typeofGetValue(ExprValue input) {
return dsl.typeof(DSL.literal(input)).valueOf(null).stringValue();
}
}
24 changes: 24 additions & 0 deletions docs/user/dql/functions.rst
Original file line number Diff line number Diff line change
Expand Up @@ -3004,3 +3004,27 @@ Example searching for field Tags::
| [Winnie-the-<em>Pooh</em>] |
+----------------------------------------------+

System Functions
================

TYPEOF
------

Description
>>>>>>>>>>>

Usage: typeof(expr) function returns name of the data type of the value that is passed to it. This can be helpful for troubleshooting or dynamically constructing SQL queries.

Argument type: ANY

Return type: STRING

Example::

os> select typeof(DATE('2008-04-14')) as `typeof(date)`, typeof(1) as `typeof(int)`, typeof(now()) as `typeof(now())`, typeof(accounts) as `typeof(column)` from people
fetched rows / total rows = 1/1
+----------------+---------------+-----------------+------------------+
| typeof(date) | typeof(int) | typeof(now()) | typeof(column) |
|----------------+---------------+-----------------+------------------|
| DATE | INTEGER | DATETIME | STRUCT |
+----------------+---------------+-----------------+------------------+
31 changes: 31 additions & 0 deletions docs/user/ppl/functions/system.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
================
System Functions
================

.. rubric:: Table of contents

.. contents::
:local:
:depth: 1

TYPEOF
------

Description
>>>>>>>>>>>

Usage: typeof(expr) function returns name of the data type of the value that is passed to it. This can be helpful for troubleshooting or dynamically constructing SQL queries.

Argument type: ANY

Return type: STRING

Example::

os> source=people | eval `typeof(date)` = typeof(DATE('2008-04-14')), `typeof(int)` = typeof(1), `typeof(now())` = typeof(now()), `typeof(column)` = typeof(accounts) | fields `typeof(date)`, `typeof(int)`, `typeof(now())`, `typeof(column)`
fetched rows / total rows = 1/1
+----------------+---------------+-----------------+------------------+
| typeof(date) | typeof(int) | typeof(now()) | typeof(column) |
|----------------+---------------+-----------------+------------------|
| DATE | INTEGER | DATETIME | STRUCT |
+----------------+---------------+-----------------+------------------+
2 changes: 2 additions & 0 deletions docs/user/ppl/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ The query start with search command and then flowing a set of command delimited

- `Type Conversion Functions <functions/conversion.rst>`_

- `System Functions <functions/system.rst>`_

* **Optimization**

- `Optimization <../../user/optimization/optimization.rst>`_
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ public void test_nonnumeric_data_types() throws IOException {
schema("date_value", "timestamp"),
schema("ip_value", "ip"),
schema("object_value", "struct"),
schema("nested_value", "array"));
schema("nested_value", "array"),
schema("geo_point_value", "geo_point"));
}

@Test
Expand All @@ -71,5 +72,4 @@ public void test_long_integer_data_type() throws IOException {
schema("long1", "long"),
schema("long2", "long"));
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.ppl;

import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NONNUMERIC;
import static org.opensearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NUMERIC;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC;
import static org.opensearch.sql.util.MatcherUtils.rows;
import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;

import java.io.IOException;
import org.json.JSONObject;
import org.junit.Test;

public class SystemFunctionIT extends PPLIntegTestCase {

@Override
public void init() throws IOException {
loadIndex(DATA_TYPE_NUMERIC);
loadIndex(DATA_TYPE_NONNUMERIC);
}

@Test
public void typeof_sql_types() throws IOException {
JSONObject response = executeQuery(String.format("source=%s | eval "
+ "`str` = typeof('pewpew'), `double` = typeof(1.0),"
+ "`int` = typeof(12345), `long` = typeof(1234567891011), `interval` = typeof(INTERVAL 2 DAY)"
+ " | fields `str`, `double`, `int`, `long`, `interval`",
TEST_INDEX_DATATYPE_NUMERIC));
// TODO: test null in PPL
verifyDataRows(response,
rows("STRING", "DOUBLE", "INTEGER", "LONG", "INTERVAL"));

response = executeQuery(String.format("source=%s | eval "
+ "`timestamp` = typeof(CAST('1961-04-12 09:07:00' AS TIMESTAMP)),"
+ "`time` = typeof(CAST('09:07:00' AS TIME)),"
+ "`date` = typeof(CAST('1961-04-12' AS DATE)),"
+ "`datetime` = typeof(DATETIME('1961-04-12 09:07:00'))"
+ " | fields `timestamp`, `time`, `date`, `datetime`",
TEST_INDEX_DATATYPE_NUMERIC));
verifyDataRows(response,
rows("TIMESTAMP", "TIME", "DATE", "DATETIME"));
}

@Test
public void typeof_opensearch_types() throws IOException {
JSONObject response = executeQuery(String.format("source=%s | eval "
+ "`double` = typeof(double_number), `long` = typeof(long_number),"
+ "`integer` = typeof(integer_number), `byte` = typeof(byte_number),"
+ "`short` = typeof(short_number), `float` = typeof(float_number),"
+ "`half_float` = typeof(half_float_number), `scaled_float` = typeof(scaled_float_number)"
+ " | fields `double`, `long`, `integer`, `byte`, `short`, `float`, `half_float`, `scaled_float`",
TEST_INDEX_DATATYPE_NUMERIC));
verifyDataRows(response,
rows("DOUBLE", "LONG", "INTEGER", "BYTE", "SHORT", "FLOAT", "FLOAT", "DOUBLE"));

response = executeQuery(String.format("source=%s | eval "
+ "`text` = typeof(text_value), `date` = typeof(date_value),"
+ "`boolean` = typeof(boolean_value), `object` = typeof(object_value),"
+ "`keyword` = typeof(keyword_value), `ip` = typeof(ip_value),"
+ "`binary` = typeof(binary_value), `geo_point` = typeof(geo_point_value)"
// TODO activate this test once `ARRAY` type supported, see ExpressionAnalyzer::isTypeNotSupported
//+ ", `nested` = typeof(nested_value)"
+ " | fields `text`, `date`, `boolean`, `object`, `keyword`, `ip`, `binary`, `geo_point`",
TEST_INDEX_DATATYPE_NONNUMERIC));
verifyDataRows(response,
rows("OPENSEARCH_TEXT", "TIMESTAMP", "BOOLEAN", "STRUCT", "STRING",
"OPENSEARCH_IP", "OPENSEARCH_BINARY", "OPENSEARCH_GEO_POINT"));
}
}
Loading