diff --git a/docs/reference/sql/functions/grouping.asciidoc b/docs/reference/sql/functions/grouping.asciidoc index 0eee0426ce65a..261066799f893 100644 --- a/docs/reference/sql/functions/grouping.asciidoc +++ b/docs/reference/sql/functions/grouping.asciidoc @@ -76,3 +76,9 @@ Instead one can rewrite the query to move the expression on the histogram _insid ---- include-tagged::{sql-specs}/docs.csv-spec[histogramDateTimeExpression] ---- + +[IMPORTANT] +When the histogram in SQL is applied on **DATE** type instead of **DATETIME**, the interval specified is truncated to +the multiple of a day. E.g.: for `HISTOGRAM(CAST(birth_date AS DATE), INTERVAL '2 3:04' DAY TO MINUTE)` the interval +actually used will be `INTERVAL '2' DAY`. If the interval specified is less than 1 day, e.g.: +`HISTOGRAM(CAST(birth_date AS DATE), INTERVAL '20' HOUR)` then the interval used will be `INTERVAL '1' DAY`. diff --git a/docs/reference/sql/language/data-types.asciidoc b/docs/reference/sql/language/data-types.asciidoc index 60bdf0c5f66d3..b42620e0c5415 100644 --- a/docs/reference/sql/language/data-types.asciidoc +++ b/docs/reference/sql/language/data-types.asciidoc @@ -5,9 +5,6 @@ beta[] -Most of {es} <> are available in {es-sql}, as indicated below. -As one can see, all of {es} <> are mapped to the data type with the same -name in {es-sql}, with the exception of **date** data type which is mapped to **datetime** in {es-sql}: [cols="^,^m,^,^"] @@ -46,6 +43,13 @@ s|SQL precision |=== +[NOTE] +Most of {es} <> are available in {es-sql}, as indicated above. +As one can see, all of {es} <> are mapped to the data type with the same +name in {es-sql}, with the exception of **date** data type which is mapped to **datetime** in {es-sql}. +This is to avoid confusion with the ANSI SQL **DATE** (date only) type, which is also supported by {es-sql} +in queries (with the use of <>/<>), +but doesn't correspond to an actual mapping in {es} (see the <> below). Obviously, not all types in {es} have an equivalent in SQL and vice-versa hence why, {es-sql} uses the data type _particularities_ of the former over the latter as ultimately {es} is the backing store. @@ -53,6 +57,8 @@ uses the data type _particularities_ of the former over the latter as ultimately In addition to the types above, {es-sql} also supports at _runtime_ SQL-specific types that do not have an equivalent in {es}. Such types cannot be loaded from {es} (as it does not know about them) however can be used inside {es-sql} in queries or their results. +[[es-sql-only-types]] + The table below indicates these types: [cols="^m,^"] @@ -62,6 +68,7 @@ s|SQL type s|SQL precision +| date | 24 | interval_year | 7 | interval_month | 7 | interval_day | 23 diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java index 097bc476bcb09..6d6231bf4303f 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java @@ -28,6 +28,7 @@ public enum EsType implements SQLType { OBJECT( Types.STRUCT), NESTED( Types.STRUCT), BINARY( Types.VARBINARY), + DATE( Types.DATE), DATETIME( Types.TIMESTAMP), IP( Types.VARCHAR), INTERVAL_YEAR( ExtraTypes.INTERVAL_YEAR), diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDateUtils.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDateUtils.java index 8fbef88dca5ab..f034f67f186e5 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDateUtils.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDateUtils.java @@ -41,10 +41,9 @@ final class JdbcDateUtils { .appendFraction(MILLI_OF_SECOND, 3, 3, true) .appendOffsetId() .toFormatter(Locale.ROOT); - + static long asMillisSinceEpoch(String date) { - ZonedDateTime zdt = ISO_WITH_MILLIS.parse(date, ZonedDateTime::from); - return zdt.toInstant().toEpochMilli(); + return ISO_WITH_MILLIS.parse(date, ZonedDateTime::from).toInstant().toEpochMilli(); } static Date asDate(String date) { @@ -71,7 +70,7 @@ static R asDateTimeField(Object value, Function asDateTimeMethod, } } - private static long utcMillisRemoveTime(long l) { + static long utcMillisRemoveTime(long l) { return l - (l % DAY_IN_MILLIS); } diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java index 8c01b3112effa..f1bce51dd3464 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java @@ -33,6 +33,9 @@ import java.util.function.Function; import static java.lang.String.format; +import static org.elasticsearch.xpack.sql.jdbc.JdbcDateUtils.asDateTimeField; +import static org.elasticsearch.xpack.sql.jdbc.JdbcDateUtils.asMillisSinceEpoch; +import static org.elasticsearch.xpack.sql.jdbc.JdbcDateUtils.utcMillisRemoveTime; class JdbcResultSet implements ResultSet, JdbcWrapper { @@ -252,8 +255,11 @@ private Long dateTime(int columnIndex) throws SQLException { if (val == null) { return null; } - return JdbcDateUtils.asDateTimeField(val, JdbcDateUtils::asMillisSinceEpoch, Function.identity()); - }; + return asDateTimeField(val, JdbcDateUtils::asMillisSinceEpoch, Function.identity()); + } + if (EsType.DATE == type) { + return utcMillisRemoveTime(asMillisSinceEpoch(val.toString())); + } return val == null ? null : (Long) val; } catch (ClassCastException cce) { throw new SQLException( diff --git a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java index 9274e9061d453..469a2d37e5ef4 100644 --- a/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java +++ b/x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java @@ -213,6 +213,8 @@ static Object convert(Object v, EsType columnType, String typeString) throws SQL return doubleValue(v); // Double might be represented as string for infinity and NaN values case FLOAT: return floatValue(v); // Float might be represented as string for infinity and NaN values + case DATE: + return JdbcDateUtils.asDateTimeField(v, JdbcDateUtils::asDate, Date::new); case DATETIME: return JdbcDateUtils.asDateTimeField(v, JdbcDateUtils::asTimestamp, Timestamp::new); case INTERVAL_YEAR: diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java index d8b6375e7ca96..47e0e9c8f90df 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java @@ -36,6 +36,7 @@ public static List readScriptSpec() throws Exception { tests.addAll(readScriptSpec("/fulltext.csv-spec", parser)); tests.addAll(readScriptSpec("/agg.csv-spec", parser)); tests.addAll(readScriptSpec("/columns.csv-spec", parser)); + tests.addAll(readScriptSpec("/date.csv-spec", parser)); tests.addAll(readScriptSpec("/datetime.csv-spec", parser)); tests.addAll(readScriptSpec("/alias.csv-spec", parser)); tests.addAll(readScriptSpec("/null.csv-spec", parser)); diff --git a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java index 2817ab6df729e..bcd3d4073eabe 100644 --- a/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java +++ b/x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java @@ -139,6 +139,7 @@ public static void assertResultSetMetadata(ResultSet expected, ResultSet actual, if (expectedType == Types.TIMESTAMP_WITH_TIMEZONE) { expectedType = Types.TIMESTAMP; } + // since csv doesn't support real, we use float instead..... if (expectedType == Types.FLOAT && expected instanceof CsvResultSet) { expectedType = Types.REAL; @@ -204,6 +205,9 @@ private static void doAssertResultSetData(ResultSet expected, ResultSet actual, // fix for CSV which returns the shortName not fully-qualified name if (!columnClassName.contains(".")) { switch (columnClassName) { + case "Date": + columnClassName = "java.sql.Date"; + break; case "Timestamp": columnClassName = "java.sql.Timestamp"; break; diff --git a/x-pack/plugin/sql/qa/src/main/resources/date.csv-spec b/x-pack/plugin/sql/qa/src/main/resources/date.csv-spec new file mode 100644 index 0000000000000..f744ea9ca6c70 --- /dev/null +++ b/x-pack/plugin/sql/qa/src/main/resources/date.csv-spec @@ -0,0 +1,77 @@ +// +// Date +// + +dateExtractDateParts +SELECT +DAY(CAST(birth_date AS DATE)) d, +DAY_OF_MONTH(CAST(birth_date AS DATE)) dm, +DAY_OF_WEEK(CAST(birth_date AS DATE)) dw, +DAY_OF_YEAR(CAST(birth_date AS DATE)) dy, +ISO_DAY_OF_WEEK(CAST(birth_date AS DATE)) iso_dw, +WEEK(CAST(birth_date AS DATE)) w, +IW(CAST(birth_date AS DATE)) iso_w, +QUARTER(CAST(birth_date AS DATE)) q, +YEAR(CAST(birth_date AS DATE)) y, +birth_date, last_name l FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no; + + d:i | dm:i | dw:i | dy:i | iso_dw:i | w:i |iso_w:i | q:i | y:i | birth_date:ts | l:s +2 |2 |4 |245 |3 |36 |35 |3 |1953 |1953-09-02T00:00:00Z |Facello +2 |2 |3 |154 |2 |23 |22 |2 |1964 |1964-06-02T00:00:00Z |Simmel +3 |3 |5 |337 |4 |49 |49 |4 |1959 |1959-12-03T00:00:00Z |Bamford +1 |1 |7 |121 |6 |18 |18 |2 |1954 |1954-05-01T00:00:00Z |Koblick +21 |21 |6 |21 |5 |4 |3 |1 |1955 |1955-01-21T00:00:00Z |Maliniak +20 |20 |2 |110 |1 |17 |16 |2 |1953 |1953-04-20T00:00:00Z |Preusig +23 |23 |5 |143 |4 |21 |21 |2 |1957 |1957-05-23T00:00:00Z |Zielinski +19 |19 |4 |50 |3 |8 |8 |1 |1958 |1958-02-19T00:00:00Z |Kalloufi +19 |19 |7 |110 |6 |16 |16 |2 |1952 |1952-04-19T00:00:00Z |Peac +; + + +dateExtractTimePartsTimeSecond +SELECT +SECOND(CAST(birth_date AS DATE)) d, +MINUTE(CAST(birth_date AS DATE)) m, +HOUR(CAST(birth_date AS DATE)) h +FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no; + + d:i | m:i | h:i +0 |0 |0 +0 |0 |0 +0 |0 |0 +0 |0 |0 +0 |0 |0 +0 |0 |0 +0 |0 |0 +0 |0 |0 +0 |0 |0 +; + +dateAsFilter +SELECT birth_date, last_name FROM "test_emp" WHERE birth_date <= CAST('1955-01-21' AS DATE) ORDER BY emp_no LIMIT 5; + + birth_date:ts | last_name:s +1953-09-02T00:00:00Z |Facello +1954-05-01T00:00:00Z |Koblick +1955-01-21T00:00:00Z |Maliniak +1953-04-20T00:00:00Z |Preusig +1952-04-19T00:00:00Z |Peac +; + +dateAndFunctionAsGroupingKey +SELECT MONTH(CAST(birth_date AS DATE)) AS m, CAST(SUM(emp_no) AS INT) s FROM test_emp GROUP BY m ORDER BY m LIMIT 5; + + m:i | s:i +null |100445 +1 |60288 +2 |80388 +3 |20164 +4 |80401 +; + +dateAndInterval +SELECT YEAR(CAST('2019-01-21' AS DATE) + INTERVAL '1-2' YEAR TO MONTH) AS y, MONTH(INTERVAL '1-2' YEAR TO MONTH + CAST('2019-01-21' AS DATE)) AS m; + +y:i | m:i +2020 | 3 +; diff --git a/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec b/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec index 3748a116b7450..1bdc090ea232f 100644 --- a/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec +++ b/x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec @@ -6,8 +6,9 @@ // Time NOT IMPLEMENTED in H2 on TIMESTAMP WITH TIME ZONE - hence why these are moved to CSV // -// WEEK_OF_YEAR moved to CSV tests, because H2 builds its Calendar with the local Locale, we consider ROOT as the default Locale -// This has implications on the results, which could change given specific locales where the rules for determining the start of a year are different. +// WEEK_OF_YEAR moved to CSV tests, because H2 builds its Calendar with the local Locale, +// we consider ROOT as the default Locale. This has implications on the results, which could +// change given specific locales where the rules for determining the start of a year are different. // // DateTime @@ -31,10 +32,10 @@ SELECT MONTHNAME(CAST('2018-09-03' AS TIMESTAMP)) month FROM "test_emp" limit 1; dayNameFromStringDateTime SELECT DAYNAME(CAST('2018-09-03' AS TIMESTAMP)) day FROM "test_emp" limit 1; -quarterSelect +dateTimeQuarter SELECT QUARTER(hire_date) q, hire_date FROM test_emp ORDER BY hire_date LIMIT 15; -dayOfWeek +dateTimeDayOfWeek SELECT DAY_OF_WEEK(birth_date) day, birth_date FROM test_emp ORDER BY DAY_OF_WEEK(birth_date); // diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java index 0c374038953ab..61e1e6bc67ef0 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java @@ -95,7 +95,7 @@ public Object extract(Bucket bucket) { if (object == null) { return object; } else if (object instanceof Long) { - object = DateUtils.of(((Long) object).longValue(), zoneId); + object = DateUtils.asDateTime(((Long) object).longValue(), zoneId); } else { throw new SqlIllegalArgumentException("Invalid date key returned: {}", object); } @@ -129,4 +129,4 @@ public boolean equals(Object obj) { public String toString() { return "|" + key + "|"; } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java index ecb61e686a109..503da62dc30ee 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java @@ -130,11 +130,11 @@ private Object unwrapMultiValue(Object values) { } if (dataType == DataType.DATETIME) { if (values instanceof String) { - return DateUtils.of(Long.parseLong(values.toString())); + return DateUtils.asDateTime(Long.parseLong(values.toString())); } // returned by nested types... if (values instanceof DateTime) { - return DateUtils.of((DateTime) values); + return DateUtils.asDateTime((DateTime) values); } } if (values instanceof Long || values instanceof Double || values instanceof String || values instanceof Boolean) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java index 3198604a94c9a..ee9f98e104877 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.sql.expression; -import org.elasticsearch.common.Strings; import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; import org.elasticsearch.xpack.sql.expression.Expression.TypeResolution; import org.elasticsearch.xpack.sql.expression.gen.pipeline.Pipe; @@ -16,11 +15,13 @@ import java.util.Collection; import java.util.List; import java.util.Locale; +import java.util.StringJoiner; import java.util.function.Predicate; import static java.lang.String.format; import static java.util.Collections.emptyList; import static java.util.Collections.emptyMap; +import static org.elasticsearch.xpack.sql.type.DataType.BOOLEAN; public final class Expressions { @@ -155,7 +156,7 @@ public static List pipe(List expressions) { } public static TypeResolution typeMustBeBoolean(Expression e, String operationName, ParamOrdinal paramOrd) { - return typeMustBe(e, dt -> dt == DataType.BOOLEAN, operationName, paramOrd, "boolean"); + return typeMustBe(e, dt -> dt == BOOLEAN, operationName, paramOrd, "boolean"); } public static TypeResolution typeMustBeInteger(Expression e, String operationName, ParamOrdinal paramOrd) { @@ -171,11 +172,11 @@ public static TypeResolution typeMustBeString(Expression e, String operationName } public static TypeResolution typeMustBeDate(Expression e, String operationName, ParamOrdinal paramOrd) { - return typeMustBe(e, dt -> dt == DataType.DATETIME, operationName, paramOrd, "date"); + return typeMustBe(e, DataType::isDateBased, operationName, paramOrd, "date", "datetime"); } public static TypeResolution typeMustBeNumericOrDate(Expression e, String operationName, ParamOrdinal paramOrd) { - return typeMustBe(e, dt -> dt.isNumeric() || dt == DataType.DATETIME, operationName, paramOrd, "numeric", "date"); + return typeMustBe(e, dt -> dt.isNumeric() || dt.isDateBased(), operationName, paramOrd, "date", "datetime", "numeric"); } public static TypeResolution typeMustBe(Expression e, @@ -188,8 +189,20 @@ public static TypeResolution typeMustBe(Expression e, new TypeResolution(format(Locale.ROOT, "[%s]%s argument must be [%s], found value [%s] type [%s]", operationName, paramOrd == null || paramOrd == ParamOrdinal.DEFAULT ? "" : " " + paramOrd.name().toLowerCase(Locale.ROOT), - Strings.arrayToDelimitedString(acceptedTypes, " or "), + acceptedTypesForErrorMsg(acceptedTypes), Expressions.name(e), e.dataType().esType)); } + + private static String acceptedTypesForErrorMsg(String... acceptedTypes) { + StringJoiner sj = new StringJoiner(", "); + for (int i = 0; i < acceptedTypes.length - 1; i++) { + sj.add(acceptedTypes[i]); + } + if (acceptedTypes.length > 1) { + return sj.toString() + " or " + acceptedTypes[acceptedTypes.length - 1]; + } else { + return acceptedTypes[0]; + } + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Max.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Max.java index e66dfdebc6b36..8aa72dea7d1da 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Max.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Max.java @@ -47,4 +47,4 @@ public String innerName() { protected TypeResolution resolveType() { return Expressions.typeMustBeNumericOrDate(field(), sourceText(), ParamOrdinal.DEFAULT); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java index 46614755b7e8f..3dd4bdc992cb0 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java @@ -42,7 +42,7 @@ protected TypeResolution resolveType() { TypeResolution resolution = Expressions.typeMustBeNumericOrDate(field(), "HISTOGRAM", ParamOrdinal.FIRST); if (resolution == TypeResolution.TYPE_RESOLVED) { // interval must be Literal interval - if (field().dataType() == DataType.DATETIME) { + if (field().dataType().isDateBased()) { resolution = Expressions.typeMustBe(interval, DataTypes::isInterval, "(Date) HISTOGRAM", ParamOrdinal.SECOND, "interval"); } else { resolution = Expressions.typeMustBeNumeric(interval, "(Numeric) HISTOGRAM", ParamOrdinal.SECOND); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/BaseDateTimeFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/BaseDateTimeFunction.java index 345498afd00f1..fa949007ef58a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/BaseDateTimeFunction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/BaseDateTimeFunction.java @@ -74,4 +74,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(field(), zoneId()); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java index 01d56188ed2ed..f56181bae13de 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java @@ -357,11 +357,11 @@ private static Object asDateTime(Object dateTime, boolean lenient) { return ((JodaCompatibleZonedDateTime) dateTime).getZonedDateTime(); } if (dateTime instanceof ZonedDateTime) { - return (ZonedDateTime) dateTime; + return dateTime; } if (false == lenient) { if (dateTime instanceof Number) { - return DateUtils.of(((Number) dateTime).longValue()); + return DateUtils.asDateTime(((Number) dateTime).longValue()); } throw new SqlIllegalArgumentException("Invalid date encountered [{}]", dateTime); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/script/ScriptWeaver.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/script/ScriptWeaver.java index cd13570a1ad10..5b75878920243 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/script/ScriptWeaver.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/script/ScriptWeaver.java @@ -79,7 +79,7 @@ default ScriptTemplate scriptWithScalar(ScalarFunctionAttribute scalar) { default ScriptTemplate scriptWithAggregate(AggregateFunctionAttribute aggregate) { String template = "{}"; - if (aggregate.dataType() == DataType.DATETIME) { + if (aggregate.dataType().isDateBased()) { template = "{sql}.asDateTime({})"; } return new ScriptTemplate(processScript(template), @@ -89,7 +89,7 @@ default ScriptTemplate scriptWithAggregate(AggregateFunctionAttribute aggregate) default ScriptTemplate scriptWithGrouping(GroupingFunctionAttribute grouping) { String template = "{}"; - if (grouping.dataType() == DataType.DATETIME) { + if (grouping.dataType().isDateBased()) { template = "{sql}.asDateTime({})"; } return new ScriptTemplate(processScript(template), diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java index da42ffe523b64..5be5e28718459 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/DateTimeArithmeticOperation.java @@ -45,12 +45,15 @@ protected TypeResolution resolveType() { if (DataTypeConversion.commonType(l, r) == null) { return new TypeResolution(format("[{}] has arguments with incompatible types [{}] and [{}]", symbol(), l, r)); } else { - return TypeResolution.TYPE_RESOLVED; + return resolveWithIntervals(); } } // fall-back to default checks return super.resolveType(); } - + + protected TypeResolution resolveWithIntervals() { + return TypeResolution.TYPE_RESOLVED; + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Sub.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Sub.java index 32acfa8ed685d..e2454ffd26742 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Sub.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/predicate/operator/arithmetic/Sub.java @@ -9,6 +9,9 @@ import org.elasticsearch.xpack.sql.expression.predicate.operator.arithmetic.BinaryArithmeticProcessor.BinaryArithmeticOperation; import org.elasticsearch.xpack.sql.tree.Source; import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.type.DataTypes; + +import static org.elasticsearch.common.logging.LoggerMessageFormat.format; /** * Subtraction function ({@code a - b}). @@ -28,4 +31,13 @@ protected NodeInfo info() { protected Sub replaceChildren(Expression newLeft, Expression newRight) { return new Sub(source(), newLeft, newRight); } + + @Override + protected TypeResolution resolveWithIntervals() { + if (right().dataType().isDateBased() && DataTypes.isInterval(left().dataType())) { + return new TypeResolution(format(null, "Cannot subtract a {}[{}] from an interval[{}]; do you mean the reverse?", + right().dataType().esType, right().source().text(), left().source().text())); + } + return TypeResolution.TYPE_RESOLVED; + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java index 68baa84a802f6..432872891e5c2 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java @@ -411,6 +411,8 @@ public DataType visitPrimitiveDataType(PrimitiveDataTypeContext ctx) { case "float": case "double": return DataType.DOUBLE; + case "date": + return DataType.DATE; case "datetime": case "timestamp": return DataType.DATETIME; @@ -793,7 +795,7 @@ public Literal visitDateEscapedLiteral(DateEscapedLiteralContext ctx) { } catch(IllegalArgumentException ex) { throw new ParsingException(source, "Invalid date received; {}", ex.getMessage()); } - return new Literal(source, DateUtils.of(dt), DataType.DATETIME); + return new Literal(source, DateUtils.asDateOnly(dt), DataType.DATE); } @Override @@ -829,7 +831,7 @@ public Literal visitTimestampEscapedLiteral(TimestampEscapedLiteralContext ctx) } catch (IllegalArgumentException ex) { throw new ParsingException(source, "Invalid timestamp received; {}", ex.getMessage()); } - return new Literal(source, DateUtils.of(dt), DataType.DATETIME); + return new Literal(source, DateUtils.asDateTime(dt), DataType.DATETIME); } @Override diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java index 5189a0ca4981e..da409439558c7 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryFolder.java @@ -61,7 +61,6 @@ import org.elasticsearch.xpack.sql.rule.Rule; import org.elasticsearch.xpack.sql.rule.RuleExecutor; import org.elasticsearch.xpack.sql.session.EmptyExecutable; -import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.util.Check; import org.elasticsearch.xpack.sql.util.DateUtils; @@ -284,7 +283,7 @@ protected PhysicalPlan rule(AggregateExec a) { if (matchingGroup != null) { if (exp instanceof Attribute || exp instanceof ScalarFunction || exp instanceof GroupingFunction) { Processor action = null; - ZoneId zi = DataType.DATETIME == exp.dataType() ? DateUtils.UTC : null; + ZoneId zi = exp.dataType().isDateBased() ? DateUtils.UTC : null; /* * special handling of dates since aggs return the typed Date object which needs * extraction instead of handling this in the scroller, the folder handles this @@ -335,7 +334,7 @@ protected PhysicalPlan rule(AggregateExec a) { // check if the field is a date - if so mark it as such to interpret the long as a date // UTC is used since that's what the server uses and there's no conversion applied // (like for date histograms) - ZoneId zi = DataType.DATETIME == child.dataType() ? DateUtils.UTC : null; + ZoneId zi = child.dataType().isDateBased() ? DateUtils.UTC : null; queryC = queryC.addColumn(new GroupByRef(matchingGroup.id(), null, zi)); } // handle histogram @@ -359,7 +358,7 @@ else if (child instanceof GroupingFunction) { matchingGroup = groupingContext.groupFor(ne); Check.notNull(matchingGroup, "Cannot find group [{}]", Expressions.name(ne)); - ZoneId zi = DataType.DATETIME == ne.dataType() ? DateUtils.UTC : null; + ZoneId zi = ne.dataType().isDateBased() ? DateUtils.UTC : null; queryC = queryC.addColumn(new GroupByRef(matchingGroup.id(), null, zi)); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java index 489e1506edf1a..1a5ceb686e609 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java @@ -91,8 +91,8 @@ import org.elasticsearch.xpack.sql.querydsl.query.TermsQuery; import org.elasticsearch.xpack.sql.querydsl.query.WildcardQuery; import org.elasticsearch.xpack.sql.tree.Source; -import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.util.Check; +import org.elasticsearch.xpack.sql.util.DateUtils; import org.elasticsearch.xpack.sql.util.ReflectionUtils; import java.util.Arrays; @@ -106,6 +106,7 @@ import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.sql.expression.Foldables.doubleValuesOf; import static org.elasticsearch.xpack.sql.expression.Foldables.valueOf; +import static org.elasticsearch.xpack.sql.type.DataType.DATE; final class QueryTranslator { @@ -275,8 +276,15 @@ else if (exp instanceof GroupingFunction) { Expression field = h.field(); // date histogram - if (h.dataType() == DataType.DATETIME) { + if (h.dataType().isDateBased()) { long intervalAsMillis = Intervals.inMillis(h.interval()); + + // When the histogram in SQL is applied on DATE type instead of DATETIME, the interval + // specified is truncated to the multiple of a day. If the interval specified is less + // than 1 day, then the interval used will be `INTERVAL '1' DAY`. + if (h.dataType() == DATE) { + intervalAsMillis = DateUtils.minDayInterval(intervalAsMillis); + } // TODO: set timezone if (field instanceof FieldAttribute) { key = new GroupByDateHistogram(aggId, nameOf(field), intervalAsMillis, h.zoneId()); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByDateHistogram.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByDateHistogram.java index 936f565827980..24367fc5e1f2f 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByDateHistogram.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByDateHistogram.java @@ -37,6 +37,11 @@ private GroupByDateHistogram(String id, String fieldName, ScriptTemplate script, } + // For testing + public long interval() { + return interval; + } + @Override protected CompositeValuesSourceBuilder createSourceBuilder() { return new DateHistogramValuesSourceBuilder(id()) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByKey.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByKey.java index 8626ea18e30c5..6f26ee1dd960c 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByKey.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/querydsl/agg/GroupByKey.java @@ -39,6 +39,8 @@ public final CompositeValuesSourceBuilder asValueSource() { builder.valueType(ValueType.DOUBLE); } else if (script.outputType().isString()) { builder.valueType(ValueType.STRING); + } else if (script.outputType() == DataType.DATE) { + builder.valueType(ValueType.LONG); } else if (script.outputType() == DataType.DATETIME) { builder.valueType(ValueType.DATE); } else if (script.outputType() == DataType.BOOLEAN) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java index f233632d0f656..3210c9ceb8a8a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java @@ -41,7 +41,8 @@ public enum DataType { OBJECT( JDBCType.STRUCT, -1, 0, 0, false, false, false), NESTED( JDBCType.STRUCT, -1, 0, 0, false, false, false), BINARY( JDBCType.VARBINARY, -1, Integer.MAX_VALUE, 0, false, false, false), - // since ODBC and JDBC interpret precision for Date as display size, + DATE( JDBCType.DATE, Long.BYTES, 10, 10, false, false, true), + // since ODBC and JDBC interpret precision for Date as display size // the precision is 23 (number of chars in ISO8601 with millis) + Z (the UTC timezone) // see https://github.com/elastic/elasticsearch/issues/30386#issuecomment-386807288 DATETIME( JDBCType.TIMESTAMP, Long.BYTES, 24, 24, false, false, true), @@ -102,7 +103,7 @@ public enum DataType { odbcToEs.put("SQL_LONGVARBINARY", BINARY); // Date - odbcToEs.put("SQL_DATE", DATETIME); + odbcToEs.put("SQL_DATE", DATE); odbcToEs.put("SQL_TIME", DATETIME); odbcToEs.put("SQL_TIMESTAMP", DATETIME); @@ -214,6 +215,10 @@ public boolean isString() { public boolean isPrimitive() { return this != OBJECT && this != NESTED; } + + public boolean isDateBased() { + return this == DATE || this == DATETIME; + } public static DataType fromOdbcType(String odbcType) { return odbcToEs.get(odbcType); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java index f3cf3d2bac1ac..a578c6a7e0644 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataTypeConversion.java @@ -17,6 +17,7 @@ import java.util.function.LongFunction; import static org.elasticsearch.xpack.sql.type.DataType.BOOLEAN; +import static org.elasticsearch.xpack.sql.type.DataType.DATE; import static org.elasticsearch.xpack.sql.type.DataType.DATETIME; import static org.elasticsearch.xpack.sql.type.DataType.LONG; import static org.elasticsearch.xpack.sql.type.DataType.NULL; @@ -73,7 +74,34 @@ public static DataType commonType(DataType left, DataType right) { return left; } } + // interval and dates + if (left == DATE) { + if (DataTypes.isInterval(right)) { + return left; + } + } + if (right == DATE) { + if (DataTypes.isInterval(left)) { + return right; + } + } + if (left == DATETIME) { + if (right == DATE) { + return left; + } + if (DataTypes.isInterval(right)) { + return left; + } + } + if (right == DATETIME) { + if (left == DATE) { + return right; + } + if (DataTypes.isInterval(left)) { + return right; + } + } if (DataTypes.isInterval(left)) { // intervals widening if (DataTypes.isInterval(right)) { @@ -82,12 +110,6 @@ public static DataType commonType(DataType left, DataType right) { } } - if (DataTypes.isInterval(right)) { - if (left == DATETIME) { - return left; - } - } - // none found return null; } @@ -145,6 +167,8 @@ private static Conversion conversion(DataType from, DataType to) { return conversionToFloat(from); case DOUBLE: return conversionToDouble(from); + case DATE: + return conversionToDate(from); case DATETIME: return conversionToDateTime(from); case BOOLEAN: @@ -156,9 +180,12 @@ private static Conversion conversion(DataType from, DataType to) { } private static Conversion conversionToString(DataType from) { - if (from == DATETIME) { + if (from == DATE) { return Conversion.DATE_TO_STRING; } + if (from == DATETIME) { + return Conversion.DATETIME_TO_STRING; + } return Conversion.OTHER_TO_STRING; } @@ -182,9 +209,12 @@ private static Conversion conversionToLong(DataType from) { if (from.isString()) { return Conversion.STRING_TO_LONG; } - if (from == DATETIME) { + if (from == DATE) { return Conversion.DATE_TO_LONG; } + if (from == DATETIME) { + return Conversion.DATETIME_TO_LONG; + } return null; } @@ -201,9 +231,12 @@ private static Conversion conversionToInt(DataType from) { if (from.isString()) { return Conversion.STRING_TO_INT; } - if (from == DATETIME) { + if (from == DATE) { return Conversion.DATE_TO_INT; } + if (from == DATETIME) { + return Conversion.DATETIME_TO_INT; + } return null; } @@ -220,9 +253,12 @@ private static Conversion conversionToShort(DataType from) { if (from.isString()) { return Conversion.STRING_TO_SHORT; } - if (from == DATETIME) { + if (from == DATE) { return Conversion.DATE_TO_SHORT; } + if (from == DATETIME) { + return Conversion.DATETIME_TO_SHORT; + } return null; } @@ -239,9 +275,12 @@ private static Conversion conversionToByte(DataType from) { if (from.isString()) { return Conversion.STRING_TO_BYTE; } - if (from == DATETIME) { + if (from == DATE) { return Conversion.DATE_TO_BYTE; } + if (from == DATETIME) { + return Conversion.DATETIME_TO_BYTE; + } return null; } @@ -258,9 +297,12 @@ private static Conversion conversionToFloat(DataType from) { if (from.isString()) { return Conversion.STRING_TO_FLOAT; } - if (from == DATETIME) { + if (from == DATE) { return Conversion.DATE_TO_FLOAT; } + if (from == DATETIME) { + return Conversion.DATETIME_TO_FLOAT; + } return null; } @@ -277,13 +319,16 @@ private static Conversion conversionToDouble(DataType from) { if (from.isString()) { return Conversion.STRING_TO_DOUBLE; } - if (from == DATETIME) { + if (from == DATE) { return Conversion.DATE_TO_DOUBLE; } + if (from == DATETIME) { + return Conversion.DATETIME_TO_DOUBLE; + } return null; } - private static Conversion conversionToDateTime(DataType from) { + private static Conversion conversionToDate(DataType from) { if (from.isRational()) { return Conversion.RATIONAL_TO_DATE; } @@ -296,6 +341,28 @@ private static Conversion conversionToDateTime(DataType from) { if (from.isString()) { return Conversion.STRING_TO_DATE; } + if (from == DATETIME) { + return Conversion.DATETIME_TO_DATE; + } + return null; + } + + private static Conversion conversionToDateTime(DataType from) { + if (from.isRational()) { + return Conversion.RATIONAL_TO_DATETIME; + } + if (from.isInteger()) { + return Conversion.INTEGER_TO_DATETIME; + } + if (from == BOOLEAN) { + return Conversion.BOOL_TO_DATETIME; // We emit an int here which is ok because of Java's casting rules + } + if (from.isString()) { + return Conversion.STRING_TO_DATETIME; + } + if (from == DATE) { + return Conversion.DATE_TO_DATETIME; + } return null; } @@ -306,36 +373,39 @@ private static Conversion conversionToBoolean(DataType from) { if (from.isString()) { return Conversion.STRING_TO_BOOLEAN; } - if (from == DATETIME) { + if (from == DATE) { return Conversion.DATE_TO_BOOLEAN; } + if (from == DATETIME) { + return Conversion.DATETIME_TO_BOOLEAN; + } return null; } public static byte safeToByte(long x) { if (x > Byte.MAX_VALUE || x < Byte.MIN_VALUE) { - throw new SqlIllegalArgumentException("[" + x + "] out of [Byte] range"); + throw new SqlIllegalArgumentException("[" + x + "] out of [byte] range"); } return (byte) x; } public static short safeToShort(long x) { if (x > Short.MAX_VALUE || x < Short.MIN_VALUE) { - throw new SqlIllegalArgumentException("[" + x + "] out of [Short] range"); + throw new SqlIllegalArgumentException("[" + x + "] out of [short] range"); } return (short) x; } public static int safeToInt(long x) { if (x > Integer.MAX_VALUE || x < Integer.MIN_VALUE) { - throw new SqlIllegalArgumentException("[" + x + "] out of [Int] range"); + throw new SqlIllegalArgumentException("[" + x + "] out of [integer] range"); } return (int) x; } public static long safeToLong(double x) { if (x > Long.MAX_VALUE || x < Long.MIN_VALUE) { - throw new SqlIllegalArgumentException("[" + x + "] out of [Long] range"); + throw new SqlIllegalArgumentException("[" + x + "] out of [long] range"); } return Math.round(x); } @@ -358,7 +428,7 @@ public static Number toInteger(double x, DataType dataType) { public static boolean convertToBoolean(String val) { String lowVal = val.toLowerCase(Locale.ROOT); if (Booleans.isBoolean(lowVal) == false) { - throw new SqlIllegalArgumentException("cannot cast [" + val + "] to [Boolean]"); + throw new SqlIllegalArgumentException("cannot cast [" + val + "] to [boolean]"); } return Booleans.parseBoolean(lowVal); } @@ -384,53 +454,68 @@ public enum Conversion { IDENTITY(Function.identity()), NULL(value -> null), - DATE_TO_STRING(o -> DateUtils.toString((ZonedDateTime) o)), + DATE_TO_STRING(o -> DateUtils.toDateString((ZonedDateTime) o)), + DATETIME_TO_STRING(o -> DateUtils.toString((ZonedDateTime) o)), OTHER_TO_STRING(String::valueOf), RATIONAL_TO_LONG(fromDouble(DataTypeConversion::safeToLong)), INTEGER_TO_LONG(fromLong(value -> value)), - STRING_TO_LONG(fromString(Long::valueOf, "Long")), - DATE_TO_LONG(fromDate(value -> value)), + STRING_TO_LONG(fromString(Long::valueOf, "long")), + DATE_TO_LONG(fromDateTime(value -> value)), + DATETIME_TO_LONG(fromDateTime(value -> value)), RATIONAL_TO_INT(fromDouble(value -> safeToInt(safeToLong(value)))), INTEGER_TO_INT(fromLong(DataTypeConversion::safeToInt)), BOOL_TO_INT(fromBool(value -> value ? 1 : 0)), - STRING_TO_INT(fromString(Integer::valueOf, "Int")), - DATE_TO_INT(fromDate(DataTypeConversion::safeToInt)), + STRING_TO_INT(fromString(Integer::valueOf, "integer")), + DATE_TO_INT(fromDateTime(DataTypeConversion::safeToInt)), + DATETIME_TO_INT(fromDateTime(DataTypeConversion::safeToInt)), RATIONAL_TO_SHORT(fromDouble(value -> safeToShort(safeToLong(value)))), INTEGER_TO_SHORT(fromLong(DataTypeConversion::safeToShort)), BOOL_TO_SHORT(fromBool(value -> value ? (short) 1 : (short) 0)), - STRING_TO_SHORT(fromString(Short::valueOf, "Short")), - DATE_TO_SHORT(fromDate(DataTypeConversion::safeToShort)), + STRING_TO_SHORT(fromString(Short::valueOf, "short")), + DATE_TO_SHORT(fromDateTime(DataTypeConversion::safeToShort)), + DATETIME_TO_SHORT(fromDateTime(DataTypeConversion::safeToShort)), RATIONAL_TO_BYTE(fromDouble(value -> safeToByte(safeToLong(value)))), INTEGER_TO_BYTE(fromLong(DataTypeConversion::safeToByte)), BOOL_TO_BYTE(fromBool(value -> value ? (byte) 1 : (byte) 0)), - STRING_TO_BYTE(fromString(Byte::valueOf, "Byte")), - DATE_TO_BYTE(fromDate(DataTypeConversion::safeToByte)), + STRING_TO_BYTE(fromString(Byte::valueOf, "byte")), + DATE_TO_BYTE(fromDateTime(DataTypeConversion::safeToByte)), + DATETIME_TO_BYTE(fromDateTime(DataTypeConversion::safeToByte)), // TODO floating point conversions are lossy but conversions to integer conversions are not. Are we ok with that? RATIONAL_TO_FLOAT(fromDouble(value -> (float) value)), INTEGER_TO_FLOAT(fromLong(value -> (float) value)), BOOL_TO_FLOAT(fromBool(value -> value ? 1f : 0f)), - STRING_TO_FLOAT(fromString(Float::valueOf, "Float")), - DATE_TO_FLOAT(fromDate(value -> (float) value)), + STRING_TO_FLOAT(fromString(Float::valueOf, "float")), + DATE_TO_FLOAT(fromDateTime(value -> (float) value)), + DATETIME_TO_FLOAT(fromDateTime(value -> (float) value)), RATIONAL_TO_DOUBLE(fromDouble(Double::valueOf)), INTEGER_TO_DOUBLE(fromLong(Double::valueOf)), BOOL_TO_DOUBLE(fromBool(value -> value ? 1d : 0d)), - STRING_TO_DOUBLE(fromString(Double::valueOf, "Double")), - DATE_TO_DOUBLE(fromDate(Double::valueOf)), + STRING_TO_DOUBLE(fromString(Double::valueOf, "double")), + DATE_TO_DOUBLE(fromDateTime(Double::valueOf)), + DATETIME_TO_DOUBLE(fromDateTime(Double::valueOf)), RATIONAL_TO_DATE(toDate(RATIONAL_TO_LONG)), INTEGER_TO_DATE(toDate(INTEGER_TO_LONG)), BOOL_TO_DATE(toDate(BOOL_TO_INT)), - STRING_TO_DATE(fromString(DateUtils::of, "Date")), + STRING_TO_DATE(fromString(DateUtils::asDateOnly, "date")), + DATETIME_TO_DATE(fromDatetimeToDate()), + + RATIONAL_TO_DATETIME(toDateTime(RATIONAL_TO_LONG)), + INTEGER_TO_DATETIME(toDateTime(INTEGER_TO_LONG)), + BOOL_TO_DATETIME(toDateTime(BOOL_TO_INT)), + STRING_TO_DATETIME(fromString(DateUtils::asDateTime, "datetime")), + DATE_TO_DATETIME(value -> value), NUMERIC_TO_BOOLEAN(fromLong(value -> value != 0)), - STRING_TO_BOOLEAN(fromString(DataTypeConversion::convertToBoolean, "Boolean")), - DATE_TO_BOOLEAN(fromDate(value -> value != 0)), + STRING_TO_BOOLEAN(fromString(DataTypeConversion::convertToBoolean, "boolean")), + DATE_TO_BOOLEAN(fromDateTime(value -> value != 0)), + DATETIME_TO_BOOLEAN(fromDateTime(value -> value != 0)), BOOL_TO_LONG(fromBool(value -> value ? 1L : 0L)), @@ -470,13 +555,21 @@ private static Function fromString(Function conv private static Function fromBool(Function converter) { return (Object l) -> converter.apply(((Boolean) l)); } - - private static Function fromDate(Function converter) { - return l -> ((ZonedDateTime) l).toEpochSecond(); + + private static Function fromDateTime(Function converter) { + return l -> converter.apply(((ZonedDateTime) l).toEpochSecond()); + } + + private static Function toDateTime(Conversion conversion) { + return l -> DateUtils.asDateTime(((Number) conversion.convert(l)).longValue()); } private static Function toDate(Conversion conversion) { - return l -> DateUtils.of(((Number) conversion.convert(l)).longValue()); + return l -> DateUtils.asDateOnly(((Number) conversion.convert(l)).longValue()); + } + + private static Function fromDatetimeToDate() { + return l -> DateUtils.asDateOnly((ZonedDateTime) l); } public Object convert(Object l) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java index 6aa56914a63d7..bdd455fe10f63 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/DateUtils.java @@ -17,38 +17,74 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; -public class DateUtils { +import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE; + +public final class DateUtils { + + private static final long DAY_IN_MILLIS = 60 * 60 * 24 * 1000; // TODO: do we have a java.time based parser we can use instead? private static final DateTimeFormatter UTC_DATE_FORMATTER = ISODateTimeFormat.dateOptionalTimeParser().withZoneUTC(); - public static ZoneId UTC = ZoneId.of("Z"); + public static final ZoneId UTC = ZoneId.of("Z"); private DateUtils() {} + /** + * Creates an date for SQL DATE type from the millis since epoch. + */ + public static ZonedDateTime asDateOnly(long millis) { + return ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), UTC).toLocalDate().atStartOfDay(UTC); + } /** - * Creates a date from the millis since epoch (thus the time-zone is UTC). + * Creates a datetime from the millis since epoch (thus the time-zone is UTC). */ - public static ZonedDateTime of(long millis) { + public static ZonedDateTime asDateTime(long millis) { return ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), UTC); } /** - * Creates a date from the millis since epoch then translates the date into the given timezone. + * Creates a datetime from the millis since epoch then translates the date into the given timezone. */ - public static ZonedDateTime of(long millis, ZoneId id) { + public static ZonedDateTime asDateTime(long millis, ZoneId id) { return ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), id); } + /** + * Parses the given string into a Date (SQL DATE type) using UTC as a default timezone. + */ + public static ZonedDateTime asDateOnly(String dateFormat) { + return asDateOnly(UTC_DATE_FORMATTER.parseDateTime(dateFormat)); + } + + public static ZonedDateTime asDateOnly(DateTime dateTime) { + LocalDateTime ldt = LocalDateTime.of( + dateTime.getYear(), + dateTime.getMonthOfYear(), + dateTime.getDayOfMonth(), + 0, + 0, + 0, + 0); + + return ZonedDateTime.ofStrict(ldt, + ZoneOffset.ofTotalSeconds(dateTime.getZone().getOffset(dateTime) / 1000), + org.elasticsearch.common.time.DateUtils.dateTimeZoneToZoneId(dateTime.getZone())); + } + + public static ZonedDateTime asDateOnly(ZonedDateTime zdt) { + return zdt.toLocalDate().atStartOfDay(zdt.getZone()); + } + /** * Parses the given string into a DateTime using UTC as a default timezone. */ - public static ZonedDateTime of(String dateFormat) { - return of(UTC_DATE_FORMATTER.parseDateTime(dateFormat)); + public static ZonedDateTime asDateTime(String dateFormat) { + return asDateTime(UTC_DATE_FORMATTER.parseDateTime(dateFormat)); } - public static ZonedDateTime of(DateTime dateTime) { + public static ZonedDateTime asDateTime(DateTime dateTime) { LocalDateTime ldt = LocalDateTime.of( dateTime.getYear(), dateTime.getMonthOfYear(), @@ -62,8 +98,20 @@ public static ZonedDateTime of(DateTime dateTime) { ZoneOffset.ofTotalSeconds(dateTime.getZone().getOffset(dateTime) / 1000), org.elasticsearch.common.time.DateUtils.dateTimeZoneToZoneId(dateTime.getZone())); } + public static String toString(ZonedDateTime dateTime) { return StringUtils.toString(dateTime); } -} \ No newline at end of file + + public static String toDateString(ZonedDateTime date) { + return date.format(ISO_LOCAL_DATE); + } + + public static long minDayInterval(long l) { + if (l < DAY_IN_MILLIS ) { + return DAY_IN_MILLIS; + } + return l - (l % DAY_IN_MILLIS); + } +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index e45da9d08fee9..946e8f93a7091 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -198,6 +198,12 @@ public void testExtractNonDateTime() { assertEquals("1:8: Invalid datetime field [ABS]. Use any datetime function.", error("SELECT EXTRACT(ABS FROM date) FROM test")); } + public void testSubtractFromInterval() { + assertEquals("1:8: Cannot subtract a datetime[CAST('2000-01-01' AS DATETIME)] " + + "from an interval[INTERVAL 1 MONTH]; do you mean the reverse?", + error("SELECT INTERVAL 1 MONTH - CAST('2000-01-01' AS DATETIME)")); + } + public void testMultipleColumns() { // xxx offset is that of the order by field assertEquals("1:43: Unknown column [xxx]\nline 1:8: Unknown column [xxx]", @@ -378,7 +384,7 @@ public void testNotSupportedAggregateOnDate() { } public void testNotSupportedAggregateOnString() { - assertEquals("1:8: [MAX(keyword)] argument must be [numeric or date], found value [keyword] type [keyword]", + assertEquals("1:8: [MAX(keyword)] argument must be [date, datetime or numeric], found value [keyword] type [keyword]", error("SELECT MAX(keyword) FROM test")); } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractorTests.java index 135ae74dd2075..0561b6820641d 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractorTests.java @@ -62,7 +62,7 @@ public void testExtractDate() { long millis = System.currentTimeMillis(); Bucket bucket = new TestBucket(singletonMap(extractor.key(), millis), randomLong(), new Aggregations(emptyList())); - assertEquals(DateUtils.of(millis, extractor.zoneId()), extractor.extract(bucket)); + assertEquals(DateUtils.asDateTime(millis, extractor.zoneId()), extractor.extract(bucket)); } public void testExtractIncorrectDateKey() { @@ -82,4 +82,4 @@ public void testExtractIncorrectDateKey() { private static ZoneId randomSafeZone() { return randomValueOtherThanMany(zi -> zi.getId().startsWith("SystemV"), () -> randomZone()); } -} \ No newline at end of file +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java index 395f3bf270aa6..2e66192fbcbfc 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractorTests.java @@ -145,7 +145,7 @@ public void testGetDate() { DocumentField field = new DocumentField("my_date_field", documentFieldValues); hit.fields(singletonMap("my_date_field", field)); FieldHitExtractor extractor = new FieldHitExtractor("my_date_field", DataType.DATETIME, true); - assertEquals(DateUtils.of(millis), extractor.extract(hit)); + assertEquals(DateUtils.asDateTime(millis), extractor.extract(hit)); } public void testGetSource() throws IOException { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessorTests.java index 831978705d061..7e9cb64b01950 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/CastProcessorTests.java @@ -39,7 +39,7 @@ public void testApply() { assertEquals(null, proc.process(null)); assertEquals(1, proc.process("1")); Exception e = expectThrows(SqlIllegalArgumentException.class, () -> proc.process("1.2")); - assertEquals("cannot cast [1.2] to [Int]", e.getMessage()); + assertEquals("cannot cast [1.2] to [integer]", e.getMessage()); } { CastProcessor proc = new CastProcessor(Conversion.BOOL_TO_INT); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeTestUtils.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeTestUtils.java index 164fe1fe931a4..2ae6e571ac9d2 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeTestUtils.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeTestUtils.java @@ -26,6 +26,10 @@ public static ZonedDateTime dateTime(int year, int month, int day, int hour, int } public static ZonedDateTime dateTime(long millisSinceEpoch) { - return DateUtils.of(millisSinceEpoch); + return DateUtils.asDateTime(millisSinceEpoch); + } + + public static ZonedDateTime date(long millisSinceEpoch) { + return DateUtils.asDateOnly(millisSinceEpoch); } } diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java index f3bf9fc03e777..01b1d0d077930 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java @@ -170,7 +170,7 @@ public void testFunctionWithFunctionWithArgAndParams() { public void testDateLiteral() { Literal l = dateLiteral("2012-01-01"); - assertThat(l.dataType(), is(DataType.DATETIME)); + assertThat(l.dataType(), is(DataType.DATE)); } public void testDateLiteralValidation() { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java index 6ed46b74d4512..e737258ef1982 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysParserTests.java @@ -61,7 +61,7 @@ public void testSysTypes() throws Exception { Command cmd = sql("SYS TYPES").v1(); List names = asList("BYTE", "LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "SCALED_FLOAT", "FLOAT", "DOUBLE", - "KEYWORD", "TEXT", "IP", "BOOLEAN", "DATETIME", + "KEYWORD", "TEXT", "IP", "BOOLEAN", "DATE", "DATETIME", "INTERVAL_YEAR", "INTERVAL_MONTH", "INTERVAL_DAY", "INTERVAL_HOUR", "INTERVAL_MINUTE", "INTERVAL_SECOND", "INTERVAL_YEAR_TO_MONTH", "INTERVAL_DAY_TO_HOUR", "INTERVAL_DAY_TO_MINUTE", "INTERVAL_DAY_TO_SECOND", "INTERVAL_HOUR_TO_MINUTE", "INTERVAL_HOUR_TO_SECOND", "INTERVAL_MINUTE_TO_SECOND", diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java index 92f734e539780..41ddb518ce6d9 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/plan/logical/command/sys/SysTypesTests.java @@ -44,7 +44,7 @@ public void testSysTypes() throws Exception { Command cmd = sql("SYS TYPES").v1(); List names = asList("BYTE", "LONG", "BINARY", "NULL", "INTEGER", "SHORT", "HALF_FLOAT", "SCALED_FLOAT", "FLOAT", "DOUBLE", - "KEYWORD", "TEXT", "IP", "BOOLEAN", "DATETIME", + "KEYWORD", "TEXT", "IP", "BOOLEAN", "DATE", "DATETIME", "INTERVAL_YEAR", "INTERVAL_MONTH", "INTERVAL_DAY", "INTERVAL_HOUR", "INTERVAL_MINUTE", "INTERVAL_SECOND", "INTERVAL_YEAR_TO_MONTH", "INTERVAL_DAY_TO_HOUR", "INTERVAL_DAY_TO_MINUTE", "INTERVAL_DAY_TO_SECOND", "INTERVAL_HOUR_TO_MINUTE", "INTERVAL_HOUR_TO_SECOND", "INTERVAL_MINUTE_TO_SECOND", diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java index 8ee94194845a5..704e4d7147e78 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.sql.expression.FieldAttribute; import org.elasticsearch.xpack.sql.expression.function.FunctionRegistry; import org.elasticsearch.xpack.sql.expression.function.grouping.Histogram; +import org.elasticsearch.xpack.sql.expression.function.scalar.Cast; import org.elasticsearch.xpack.sql.expression.function.scalar.math.MathProcessor.MathOperation; import org.elasticsearch.xpack.sql.expression.gen.script.ScriptTemplate; import org.elasticsearch.xpack.sql.optimizer.Optimizer; @@ -34,6 +35,7 @@ import org.elasticsearch.xpack.sql.plan.physical.PhysicalPlan; import org.elasticsearch.xpack.sql.planner.QueryTranslator.QueryTranslation; import org.elasticsearch.xpack.sql.querydsl.agg.AggFilter; +import org.elasticsearch.xpack.sql.querydsl.agg.GroupByDateHistogram; import org.elasticsearch.xpack.sql.querydsl.query.ExistsQuery; import org.elasticsearch.xpack.sql.querydsl.query.NotQuery; import org.elasticsearch.xpack.sql.querydsl.query.Query; @@ -180,7 +182,7 @@ public void testDateRangeCast() { assertTrue(query instanceof RangeQuery); RangeQuery rq = (RangeQuery) query; assertEquals("date", rq.field()); - assertEquals(DateUtils.of("1969-05-13T12:34:56Z"), rq.lower()); + assertEquals(DateUtils.asDateTime("1969-05-13T12:34:56Z"), rq.lower()); } public void testLikeConstructsNotSupported() { @@ -482,6 +484,52 @@ public void testGroupByHistogram() { assertEquals(FieldAttribute.class, field.getClass()); assertEquals(DataType.DATETIME, field.dataType()); } + + public void testGroupByHistogramWithDate() { + LogicalPlan p = plan("SELECT MAX(int) FROM test GROUP BY HISTOGRAM(CAST(date AS DATE), INTERVAL 2 MONTHS)"); + assertTrue(p instanceof Aggregate); + Aggregate a = (Aggregate) p; + List groupings = a.groupings(); + assertEquals(1, groupings.size()); + Expression exp = groupings.get(0); + assertEquals(Histogram.class, exp.getClass()); + Histogram h = (Histogram) exp; + assertEquals("+0-2", h.interval().fold().toString()); + Expression field = h.field(); + assertEquals(Cast.class, field.getClass()); + assertEquals(DataType.DATE, field.dataType()); + } + + public void testGroupByHistogramWithDateAndSmallInterval() { + PhysicalPlan p = optimizeAndPlan("SELECT MAX(int) FROM test GROUP BY " + + "HISTOGRAM(CAST(date AS DATE), INTERVAL 5 MINUTES)"); + assertEquals(EsQueryExec.class, p.getClass()); + EsQueryExec eqe = (EsQueryExec) p; + assertEquals(1, eqe.queryContainer().aggs().groups().size()); + assertEquals(GroupByDateHistogram.class, eqe.queryContainer().aggs().groups().get(0).getClass()); + assertEquals(86400000L, ((GroupByDateHistogram) eqe.queryContainer().aggs().groups().get(0)).interval()); + } + + public void testGroupByHistogramWithDateTruncateIntervalToDayMultiples() { + { + PhysicalPlan p = optimizeAndPlan("SELECT MAX(int) FROM test GROUP BY " + + "HISTOGRAM(CAST(date AS DATE), INTERVAL '2 3:04' DAY TO MINUTE)"); + assertEquals(EsQueryExec.class, p.getClass()); + EsQueryExec eqe = (EsQueryExec) p; + assertEquals(1, eqe.queryContainer().aggs().groups().size()); + assertEquals(GroupByDateHistogram.class, eqe.queryContainer().aggs().groups().get(0).getClass()); + assertEquals(172800000L, ((GroupByDateHistogram) eqe.queryContainer().aggs().groups().get(0)).interval()); + } + { + PhysicalPlan p = optimizeAndPlan("SELECT MAX(int) FROM test GROUP BY " + + "HISTOGRAM(CAST(date AS DATE), INTERVAL 4409 MINUTES)"); + assertEquals(EsQueryExec.class, p.getClass()); + EsQueryExec eqe = (EsQueryExec) p; + assertEquals(1, eqe.queryContainer().aggs().groups().size()); + assertEquals(GroupByDateHistogram.class, eqe.queryContainer().aggs().groups().get(0).getClass()); + assertEquals(259200000L, ((GroupByDateHistogram) eqe.queryContainer().aggs().groups().get(0)).interval()); + } + } public void testCountAndCountDistinctFolding() { PhysicalPlan p = optimizeAndPlan("SELECT COUNT(DISTINCT keyword) dkey, COUNT(keyword) key FROM test"); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java index ac744c3365a54..c42159bfaa35d 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/DataTypeConversionTests.java @@ -12,16 +12,27 @@ import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.tree.Source; import org.elasticsearch.xpack.sql.type.DataTypeConversion.Conversion; +import org.elasticsearch.xpack.sql.util.DateUtils; import java.time.ZonedDateTime; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import static org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeTestUtils.date; import static org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeTestUtils.dateTime; import static org.elasticsearch.xpack.sql.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.sql.type.DataType.BYTE; +import static org.elasticsearch.xpack.sql.type.DataType.DATE; import static org.elasticsearch.xpack.sql.type.DataType.DATETIME; import static org.elasticsearch.xpack.sql.type.DataType.DOUBLE; import static org.elasticsearch.xpack.sql.type.DataType.FLOAT; import static org.elasticsearch.xpack.sql.type.DataType.INTEGER; +import static org.elasticsearch.xpack.sql.type.DataType.INTERVAL_HOUR_TO_MINUTE; +import static org.elasticsearch.xpack.sql.type.DataType.INTERVAL_HOUR_TO_SECOND; +import static org.elasticsearch.xpack.sql.type.DataType.INTERVAL_MONTH; +import static org.elasticsearch.xpack.sql.type.DataType.INTERVAL_SECOND; +import static org.elasticsearch.xpack.sql.type.DataType.INTERVAL_YEAR; +import static org.elasticsearch.xpack.sql.type.DataType.INTERVAL_YEAR_TO_MONTH; import static org.elasticsearch.xpack.sql.type.DataType.IP; import static org.elasticsearch.xpack.sql.type.DataType.KEYWORD; import static org.elasticsearch.xpack.sql.type.DataType.LONG; @@ -33,17 +44,30 @@ import static org.elasticsearch.xpack.sql.type.DataType.values; import static org.elasticsearch.xpack.sql.type.DataTypeConversion.commonType; import static org.elasticsearch.xpack.sql.type.DataTypeConversion.conversionFor; +import static org.elasticsearch.xpack.sql.util.DateUtils.asDateTime; public class DataTypeConversionTests extends ESTestCase { - public void testConversionToString() { - Conversion conversion = conversionFor(DOUBLE, KEYWORD); - assertNull(conversion.convert(null)); - assertEquals("10.0", conversion.convert(10.0)); - conversion = conversionFor(DATETIME, KEYWORD); - assertNull(conversion.convert(null)); - assertEquals("1970-01-01T00:00:00.000Z", conversion.convert(dateTime(0))); + public void testConversionToString() { + DataType to = KEYWORD; + { + Conversion conversion = conversionFor(DOUBLE, to); + assertNull(conversion.convert(null)); + assertEquals("10.0", conversion.convert(10.0)); + } + { + Conversion conversion = conversionFor(DATE, to); + assertNull(conversion.convert(null)); + assertEquals("1973-11-29", conversion.convert(DateUtils.asDateOnly(123456789101L))); + assertEquals("1966-02-02", conversion.convert(DateUtils.asDateOnly(-123456789101L))); + } + { + Conversion conversion = conversionFor(DATETIME, to); + assertNull(conversion.convert(null)); + assertEquals("1973-11-29T21:33:09.101Z", conversion.convert(asDateTime(123456789101L))); + assertEquals("1966-02-02T02:26:50.899Z", conversion.convert(asDateTime(-123456789101L))); + } } /** @@ -58,7 +82,7 @@ public void testConversionToLong() { assertEquals(10L, conversion.convert(10.1)); assertEquals(11L, conversion.convert(10.6)); Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert(Double.MAX_VALUE)); - assertEquals("[" + Double.MAX_VALUE + "] out of [Long] range", e.getMessage()); + assertEquals("[" + Double.MAX_VALUE + "] out of [long] range", e.getMessage()); } { Conversion conversion = conversionFor(INTEGER, to); @@ -72,12 +96,74 @@ public void testConversionToLong() { assertEquals(1L, conversion.convert(true)); assertEquals(0L, conversion.convert(false)); } - Conversion conversion = conversionFor(KEYWORD, to); - assertNull(conversion.convert(null)); - assertEquals(1L, conversion.convert("1")); - assertEquals(0L, conversion.convert("-0")); - Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff")); - assertEquals("cannot cast [0xff] to [Long]", e.getMessage()); + { + Conversion conversion = conversionFor(DATE, to); + assertNull(conversion.convert(null)); + assertEquals(123379200L, conversion.convert(DateUtils.asDateOnly(123456789101L))); + assertEquals(-123465600L, conversion.convert(DateUtils.asDateOnly(-123456789101L))); + } + { + Conversion conversion = conversionFor(DATETIME, to); + assertNull(conversion.convert(null)); + assertEquals(123456789L, conversion.convert(asDateTime(123456789101L))); + assertEquals(-123456790L, conversion.convert(asDateTime(-123456789101L))); + } + { + Conversion conversion = conversionFor(KEYWORD, to); + assertNull(conversion.convert(null)); + assertEquals(1L, conversion.convert("1")); + assertEquals(0L, conversion.convert("-0")); + Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff")); + assertEquals("cannot cast [0xff] to [long]", e.getMessage()); + } + } + + public void testConversionToDate() { + DataType to = DATE; + { + Conversion conversion = conversionFor(DOUBLE, to); + assertNull(conversion.convert(null)); + assertEquals(date(10L), conversion.convert(10.0)); + assertEquals(date(10L), conversion.convert(10.1)); + assertEquals(date(11L), conversion.convert(10.6)); + Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert(Double.MAX_VALUE)); + assertEquals("[" + Double.MAX_VALUE + "] out of [long] range", e.getMessage()); + } + { + Conversion conversion = conversionFor(INTEGER, to); + assertNull(conversion.convert(null)); + assertEquals(date(10L), conversion.convert(10)); + assertEquals(date(-134L), conversion.convert(-134)); + } + { + Conversion conversion = conversionFor(BOOLEAN, to); + assertNull(conversion.convert(null)); + assertEquals(date(1), conversion.convert(true)); + assertEquals(date(0), conversion.convert(false)); + } + { + Conversion conversion = conversionFor(DATETIME, to); + assertNull(conversion.convert(null)); + assertEquals(date(123456780000L), conversion.convert(asDateTime(123456789101L))); + assertEquals(date(-123456789101L), conversion.convert(asDateTime(-123456789101L))); + } + { + Conversion conversion = conversionFor(KEYWORD, to); + assertNull(conversion.convert(null)); + + assertEquals(date(0L), conversion.convert("1970-01-01T00:10:01Z")); + assertEquals(date(1483228800000L), conversion.convert("2017-01-01T00:11:00Z")); + assertEquals(date(-1672531200000L), conversion.convert("1917-01-01T00:11:00Z")); + assertEquals(date(18000000L), conversion.convert("1970-01-01T03:10:20-05:00")); + + // double check back and forth conversion + ZonedDateTime zdt = TestUtils.now(); + Conversion forward = conversionFor(DATE, KEYWORD); + Conversion back = conversionFor(KEYWORD, DATE); + assertEquals(DateUtils.asDateOnly(zdt), back.convert(forward.convert(zdt))); + Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff")); + assertEquals("cannot cast [0xff] to [date]:Invalid format: \"0xff\" is malformed at \"xff\"", e.getMessage()); + } } public void testConversionToDateTime() { @@ -89,7 +175,7 @@ public void testConversionToDateTime() { assertEquals(dateTime(10L), conversion.convert(10.1)); assertEquals(dateTime(11L), conversion.convert(10.6)); Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert(Double.MAX_VALUE)); - assertEquals("[" + Double.MAX_VALUE + "] out of [Long] range", e.getMessage()); + assertEquals("[" + Double.MAX_VALUE + "] out of [long] range", e.getMessage()); } { Conversion conversion = conversionFor(INTEGER, to); @@ -103,84 +189,121 @@ public void testConversionToDateTime() { assertEquals(dateTime(1), conversion.convert(true)); assertEquals(dateTime(0), conversion.convert(false)); } - Conversion conversion = conversionFor(KEYWORD, to); - assertNull(conversion.convert(null)); + { + Conversion conversion = conversionFor(DATE, to); + assertNull(conversion.convert(null)); + assertEquals(dateTime(123379200000L), conversion.convert(DateUtils.asDateOnly(123456789101L))); + assertEquals(dateTime(-123465600000L), conversion.convert(DateUtils.asDateOnly(-123456789101L))); + } + { + Conversion conversion = conversionFor(KEYWORD, to); + assertNull(conversion.convert(null)); + + assertEquals(dateTime(1000L), conversion.convert("1970-01-01T00:00:01Z")); + assertEquals(dateTime(1483228800000L), conversion.convert("2017-01-01T00:00:00Z")); + assertEquals(dateTime(1483228800000L), conversion.convert("2017-01-01T00:00:00Z")); + assertEquals(dateTime(18000000L), conversion.convert("1970-01-01T00:00:00-05:00")); - assertEquals(dateTime(1000L), conversion.convert("1970-01-01T00:00:01Z")); - assertEquals(dateTime(1483228800000L), conversion.convert("2017-01-01T00:00:00Z")); - assertEquals(dateTime(18000000L), conversion.convert("1970-01-01T00:00:00-05:00")); - - // double check back and forth conversion - ZonedDateTime dt = TestUtils.now(); - Conversion forward = conversionFor(DATETIME, KEYWORD); - Conversion back = conversionFor(KEYWORD, DATETIME); - assertEquals(dt, back.convert(forward.convert(dt))); - Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff")); - assertEquals("cannot cast [0xff] to [Date]:Invalid format: \"0xff\" is malformed at \"xff\"", e.getMessage()); + // double check back and forth conversion + ZonedDateTime dt = TestUtils.now(); + Conversion forward = conversionFor(DATETIME, KEYWORD); + Conversion back = conversionFor(KEYWORD, DATETIME); + assertEquals(dt, back.convert(forward.convert(dt))); + Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff")); + assertEquals("cannot cast [0xff] to [datetime]:Invalid format: \"0xff\" is malformed at \"xff\"", e.getMessage()); + } } public void testConversionToDouble() { + DataType to = DOUBLE; { - Conversion conversion = conversionFor(FLOAT, DOUBLE); + Conversion conversion = conversionFor(FLOAT, to); assertNull(conversion.convert(null)); assertEquals(10.0, (double) conversion.convert(10.0f), 0.00001); assertEquals(10.1, (double) conversion.convert(10.1f), 0.00001); assertEquals(10.6, (double) conversion.convert(10.6f), 0.00001); } { - Conversion conversion = conversionFor(INTEGER, DOUBLE); + Conversion conversion = conversionFor(INTEGER, to); assertNull(conversion.convert(null)); assertEquals(10.0, (double) conversion.convert(10), 0.00001); assertEquals(-134.0, (double) conversion.convert(-134), 0.00001); } { - Conversion conversion = conversionFor(BOOLEAN, DOUBLE); + Conversion conversion = conversionFor(BOOLEAN, to); assertNull(conversion.convert(null)); assertEquals(1.0, (double) conversion.convert(true), 0); assertEquals(0.0, (double) conversion.convert(false), 0); } { - Conversion conversion = conversionFor(KEYWORD, DOUBLE); + Conversion conversion = conversionFor(DATE, to); + assertNull(conversion.convert(null)); + assertEquals(1.233792E8, (double) conversion.convert(DateUtils.asDateOnly(123456789101L)), 0); + assertEquals(-1.234656E8, (double) conversion.convert(DateUtils.asDateOnly(-123456789101L)), 0); + } + { + Conversion conversion = conversionFor(DATETIME, to); + assertNull(conversion.convert(null)); + assertEquals(1.23456789E8, (double) conversion.convert(asDateTime(123456789101L)), 0); + assertEquals(-1.2345679E8, (double) conversion.convert(asDateTime(-123456789101L)), 0); + } + { + Conversion conversion = conversionFor(KEYWORD, to); assertNull(conversion.convert(null)); assertEquals(1.0, (double) conversion.convert("1"), 0); assertEquals(0.0, (double) conversion.convert("-0"), 0); assertEquals(12.776, (double) conversion.convert("12.776"), 0.00001); Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0xff")); - assertEquals("cannot cast [0xff] to [Double]", e.getMessage()); + assertEquals("cannot cast [0xff] to [double]", e.getMessage()); } } public void testConversionToBoolean() { + DataType to = BOOLEAN; { - Conversion conversion = conversionFor(FLOAT, BOOLEAN); + Conversion conversion = conversionFor(FLOAT, to); assertNull(conversion.convert(null)); assertEquals(true, conversion.convert(10.0f)); assertEquals(true, conversion.convert(-10.0f)); assertEquals(false, conversion.convert(0.0f)); } { - Conversion conversion = conversionFor(INTEGER, BOOLEAN); + Conversion conversion = conversionFor(INTEGER, to); assertNull(conversion.convert(null)); assertEquals(true, conversion.convert(10)); assertEquals(true, conversion.convert(-10)); assertEquals(false, conversion.convert(0)); } { - Conversion conversion = conversionFor(LONG, BOOLEAN); + Conversion conversion = conversionFor(LONG, to); assertNull(conversion.convert(null)); assertEquals(true, conversion.convert(10L)); assertEquals(true, conversion.convert(-10L)); assertEquals(false, conversion.convert(0L)); } { - Conversion conversion = conversionFor(DOUBLE, BOOLEAN); + Conversion conversion = conversionFor(DOUBLE, to); assertNull(conversion.convert(null)); assertEquals(true, conversion.convert(10.0d)); assertEquals(true, conversion.convert(-10.0d)); assertEquals(false, conversion.convert(0.0d)); } { - Conversion conversion = conversionFor(KEYWORD, BOOLEAN); + Conversion conversion = conversionFor(DATE, to); + assertNull(conversion.convert(null)); + assertEquals(true, conversion.convert(DateUtils.asDateOnly(123456789101L))); + assertEquals(true, conversion.convert(DateUtils.asDateOnly(-123456789101L))); + assertEquals(false, conversion.convert(DateUtils.asDateOnly(0L))); + } + { + Conversion conversion = conversionFor(DATETIME, to); + assertNull(conversion.convert(null)); + assertEquals(true, conversion.convert(asDateTime(123456789101L))); + assertEquals(true, conversion.convert(asDateTime(-123456789101L))); + assertEquals(false, conversion.convert(asDateTime(0L))); + } + { + Conversion conversion = conversionFor(KEYWORD, to); assertNull(conversion.convert(null)); // We only handled upper and lower case true and false assertEquals(true, conversion.convert("true")); @@ -189,29 +312,42 @@ public void testConversionToBoolean() { assertEquals(false, conversion.convert("fAlSe")); // Everything else should fail Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("10")); - assertEquals("cannot cast [10] to [Boolean]", e.getMessage()); + assertEquals("cannot cast [10] to [boolean]", e.getMessage()); e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("-1")); - assertEquals("cannot cast [-1] to [Boolean]", e.getMessage()); + assertEquals("cannot cast [-1] to [boolean]", e.getMessage()); e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("0")); - assertEquals("cannot cast [0] to [Boolean]", e.getMessage()); + assertEquals("cannot cast [0] to [boolean]", e.getMessage()); e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("blah")); - assertEquals("cannot cast [blah] to [Boolean]", e.getMessage()); + assertEquals("cannot cast [blah] to [boolean]", e.getMessage()); e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("Yes")); - assertEquals("cannot cast [Yes] to [Boolean]", e.getMessage()); + assertEquals("cannot cast [Yes] to [boolean]", e.getMessage()); e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert("nO")); - assertEquals("cannot cast [nO] to [Boolean]", e.getMessage()); + assertEquals("cannot cast [nO] to [boolean]", e.getMessage()); } } public void testConversionToInt() { + DataType to = INTEGER; { - Conversion conversion = conversionFor(DOUBLE, INTEGER); + Conversion conversion = conversionFor(DOUBLE, to); assertNull(conversion.convert(null)); assertEquals(10, conversion.convert(10.0)); assertEquals(10, conversion.convert(10.1)); assertEquals(11, conversion.convert(10.6)); Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert(Long.MAX_VALUE)); - assertEquals("[" + Long.MAX_VALUE + "] out of [Int] range", e.getMessage()); + assertEquals("[" + Long.MAX_VALUE + "] out of [integer] range", e.getMessage()); + } + { + Conversion conversion = conversionFor(DATE, to); + assertNull(conversion.convert(null)); + assertEquals(123379200, conversion.convert(DateUtils.asDateOnly(123456789101L))); + assertEquals(-123465600, conversion.convert(DateUtils.asDateOnly(-123456789101L))); + } + { + Conversion conversion = conversionFor(DATETIME, to); + assertNull(conversion.convert(null)); + assertEquals(123456789, conversion.convert(asDateTime(123456789101L))); + assertEquals(-123456790, conversion.convert(asDateTime(-123456789101L))); } } @@ -223,7 +359,7 @@ public void testConversionToShort() { assertEquals((short) 10, conversion.convert(10.1)); assertEquals((short) 11, conversion.convert(10.6)); Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert(Integer.MAX_VALUE)); - assertEquals("[" + Integer.MAX_VALUE + "] out of [Short] range", e.getMessage()); + assertEquals("[" + Integer.MAX_VALUE + "] out of [short] range", e.getMessage()); } } @@ -235,7 +371,7 @@ public void testConversionToByte() { assertEquals((byte) 10, conversion.convert(10.1)); assertEquals((byte) 11, conversion.convert(10.6)); Exception e = expectThrows(SqlIllegalArgumentException.class, () -> conversion.convert(Short.MAX_VALUE)); - assertEquals("[" + Short.MAX_VALUE + "] out of [Byte] range", e.getMessage()); + assertEquals("[" + Short.MAX_VALUE + "] out of [byte] range", e.getMessage()); } } @@ -264,16 +400,30 @@ public void testCommonType() { assertEquals(NULL, commonType(NULL, NULL)); assertEquals(INTEGER, commonType(INTEGER, KEYWORD)); assertEquals(LONG, commonType(TEXT, LONG)); - assertEquals(null, commonType(TEXT, KEYWORD)); + assertNull(commonType(TEXT, KEYWORD)); assertEquals(SHORT, commonType(SHORT, BYTE)); assertEquals(FLOAT, commonType(BYTE, FLOAT)); assertEquals(FLOAT, commonType(FLOAT, INTEGER)); assertEquals(DOUBLE, commonType(DOUBLE, FLOAT)); + + // dates/datetimes and intervals + assertEquals(DATETIME, commonType(DATE, DATETIME)); + assertEquals(DATETIME, commonType(DATETIME, DATE)); + assertEquals(DATETIME, commonType(DATETIME, randomInterval())); + assertEquals(DATETIME, commonType(randomInterval(), DATETIME)); + assertEquals(DATE, commonType(DATE, randomInterval())); + assertEquals(DATE, commonType(randomInterval(), DATE)); + + assertEquals(INTERVAL_YEAR_TO_MONTH, commonType(INTERVAL_YEAR_TO_MONTH, INTERVAL_MONTH)); + assertEquals(INTERVAL_HOUR_TO_SECOND, commonType(INTERVAL_HOUR_TO_MINUTE, INTERVAL_HOUR_TO_SECOND)); + assertNull(commonType(INTERVAL_SECOND, INTERVAL_YEAR)); } public void testEsDataTypes() { for (DataType type : values()) { - assertEquals(type, fromTypeName(type.esType)); + if (type != DATE) { // Doesn't have a corresponding type in ES + assertEquals(type, fromTypeName(type.esType)); + } } } @@ -298,4 +448,8 @@ public void testIpToString() { Conversion stringToIp = conversionFor(KEYWORD, IP); assertEquals("10.0.0.1", ipToString.convert(stringToIp.convert(Literal.of(s, "10.0.0.1")))); } + + private DataType randomInterval() { + return randomFrom(Stream.of(DataType.values()).filter(DataTypes::isInterval).collect(Collectors.toList())); + } }