SQL: Introduce SQL DATE data type (#37693)

* SQL: Introduce SQL DATE data type

Support ANSI SQL's DATE type by introducing a runtime-only
ES SQL date type.

Closes: #37340

docs/reference/sql/functions/grouping.asciidoc

@@ -76,3 +76,9 @@ Instead one can rewrite the query to move the expression on the histogram _insid
 ----
 include-tagged::{sql-specs}/docs.csv-spec[histogramDateTimeExpression]
 ----
+
+[IMPORTANT]
+When the histogram in SQL is applied on **DATE** type instead of **DATETIME**, the interval specified is truncated to
+the multiple of a day. E.g.: for `HISTOGRAM(CAST(birth_date AS DATE), INTERVAL '2 3:04' DAY TO MINUTE)` the interval
+actually used will be `INTERVAL '2' DAY`. If the interval specified is less than 1 day, e.g.:
+`HISTOGRAM(CAST(birth_date AS DATE), INTERVAL '20' HOUR)` then the interval used will be `INTERVAL '1' DAY`.

docs/reference/sql/language/data-types.asciidoc

@@ -5,9 +5,6 @@
 
 beta[]
 
-Most of {es} <<mapping-types, data types>> are available in {es-sql}, as indicated below.
-As one can see, all of {es} <<mapping-types, data types>> are mapped to the data type with the same
-name in {es-sql}, with the exception of **date** data type which is mapped to **datetime** in {es-sql}:
 
 [cols="^,^m,^,^"]
 
@@ -46,13 +43,22 @@ s|SQL precision
 
 |===
 
+[NOTE]
+Most of {es} <<mapping-types, data types>> are available in {es-sql}, as indicated above.
+As one can see, all of {es} <<mapping-types, data types>> are mapped to the data type with the same
+name in {es-sql}, with the exception of **date** data type which is mapped to **datetime** in {es-sql}.
+This is to avoid confusion with the ANSI SQL **DATE** (date only) type, which is also supported by {es-sql}
+in queries (with the use of <<sql-functions-type-conversion-cast>>/<<sql-functions-type-conversion-convert>>),
+but doesn't correspond to an actual mapping in {es} (see the <<es-sql-only-types, `table`>> below).
 
 Obviously, not all types in {es} have an equivalent in SQL and vice-versa hence why, {es-sql}
 uses the data type _particularities_ of the former over the latter as ultimately {es} is the backing store.
 
 In addition to the types above, {es-sql} also supports at _runtime_ SQL-specific types that do not have an equivalent in {es}.
 Such types cannot be loaded from {es} (as it does not know about them) however can be used inside {es-sql} in queries or their results.
 
+[[es-sql-only-types]]
+
 The table below indicates these types:
 
 [cols="^m,^"]
@@ -62,6 +68,7 @@ s|SQL type
 s|SQL precision
 
 
+| date                      | 24
 | interval_year             | 7
 | interval_month            | 7
 | interval_day              | 23

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java

@@ -28,6 +28,7 @@ public enum EsType implements SQLType {
     OBJECT(                   Types.STRUCT),
     NESTED(                   Types.STRUCT),
     BINARY(                   Types.VARBINARY),
+    DATE(                     Types.DATE),
     DATETIME(                 Types.TIMESTAMP),
     IP(                       Types.VARCHAR),
     INTERVAL_YEAR(            ExtraTypes.INTERVAL_YEAR),

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDateUtils.java

@@ -41,10 +41,9 @@ final class JdbcDateUtils {
         .appendFraction(MILLI_OF_SECOND, 3, 3, true)
         .appendOffsetId()
         .toFormatter(Locale.ROOT);
-    
+
     static long asMillisSinceEpoch(String date) {
-        ZonedDateTime zdt = ISO_WITH_MILLIS.parse(date, ZonedDateTime::from);
-        return zdt.toInstant().toEpochMilli();
+        return ISO_WITH_MILLIS.parse(date, ZonedDateTime::from).toInstant().toEpochMilli();
     }
 
     static Date asDate(String date) {
@@ -71,7 +70,7 @@ static <R> R asDateTimeField(Object value, Function<String, R> asDateTimeMethod,
         }
     }
 
-    private static long utcMillisRemoveTime(long l) {
+    static long utcMillisRemoveTime(long l) {
         return l - (l % DAY_IN_MILLIS);
     }
 

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java

@@ -33,6 +33,9 @@
 import java.util.function.Function;
 
 import static java.lang.String.format;
+import static org.elasticsearch.xpack.sql.jdbc.JdbcDateUtils.asDateTimeField;
+import static org.elasticsearch.xpack.sql.jdbc.JdbcDateUtils.asMillisSinceEpoch;
+import static org.elasticsearch.xpack.sql.jdbc.JdbcDateUtils.utcMillisRemoveTime;
 
 class JdbcResultSet implements ResultSet, JdbcWrapper {
 
@@ -252,8 +255,11 @@ private Long dateTime(int columnIndex) throws SQLException {
                 if (val == null) {
                     return null;
                 }
-                return JdbcDateUtils.asDateTimeField(val, JdbcDateUtils::asMillisSinceEpoch, Function.identity());
-            };
+                return asDateTimeField(val, JdbcDateUtils::asMillisSinceEpoch, Function.identity());
+            }
+            if (EsType.DATE == type) {
+                return utcMillisRemoveTime(asMillisSinceEpoch(val.toString()));
+            }
             return val == null ? null : (Long) val;
         } catch (ClassCastException cce) {
             throw new SQLException(

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java

@@ -213,6 +213,8 @@ static Object convert(Object v, EsType columnType, String typeString) throws SQL
                 return doubleValue(v); // Double might be represented as string for infinity and NaN values
             case FLOAT:
                 return floatValue(v); // Float might be represented as string for infinity and NaN values
+            case DATE:
+                return JdbcDateUtils.asDateTimeField(v, JdbcDateUtils::asDate, Date::new);
             case DATETIME:
                 return JdbcDateUtils.asDateTimeField(v, JdbcDateUtils::asTimestamp, Timestamp::new);
             case INTERVAL_YEAR:

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java

@@ -36,6 +36,7 @@ public static List<Object[]> readScriptSpec() throws Exception {
         tests.addAll(readScriptSpec("/fulltext.csv-spec", parser));
         tests.addAll(readScriptSpec("/agg.csv-spec", parser));
         tests.addAll(readScriptSpec("/columns.csv-spec", parser));
+        tests.addAll(readScriptSpec("/date.csv-spec", parser));
         tests.addAll(readScriptSpec("/datetime.csv-spec", parser));
         tests.addAll(readScriptSpec("/alias.csv-spec", parser));
         tests.addAll(readScriptSpec("/null.csv-spec", parser));

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java

@@ -139,6 +139,7 @@ public static void assertResultSetMetadata(ResultSet expected, ResultSet actual,
             if (expectedType == Types.TIMESTAMP_WITH_TIMEZONE) {
                 expectedType = Types.TIMESTAMP;
             }
+
             // since csv doesn't support real, we use float instead.....
             if (expectedType == Types.FLOAT && expected instanceof CsvResultSet) {
                 expectedType = Types.REAL;
@@ -204,6 +205,9 @@ private static void doAssertResultSetData(ResultSet expected, ResultSet actual,
                         // fix for CSV which returns the shortName not fully-qualified name
                         if (!columnClassName.contains(".")) {
                             switch (columnClassName) {
+                                case "Date":
+                                    columnClassName = "java.sql.Date";
+                                    break;
                                 case "Timestamp":
                                     columnClassName = "java.sql.Timestamp";
                                     break;

x-pack/plugin/sql/qa/src/main/resources/date.csv-spec

@@ -0,0 +1,77 @@
+//
+// Date
+//
+
+dateExtractDateParts
+SELECT
+DAY(CAST(birth_date AS DATE)) d,
+DAY_OF_MONTH(CAST(birth_date AS DATE)) dm,
+DAY_OF_WEEK(CAST(birth_date AS DATE)) dw,
+DAY_OF_YEAR(CAST(birth_date AS DATE)) dy,
+ISO_DAY_OF_WEEK(CAST(birth_date AS DATE)) iso_dw,
+WEEK(CAST(birth_date AS DATE)) w,
+IW(CAST(birth_date AS DATE)) iso_w,
+QUARTER(CAST(birth_date AS DATE)) q,
+YEAR(CAST(birth_date AS DATE)) y,
+birth_date, last_name l FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no;
+
+ d:i      | dm:i     | dw:i     | dy:i    | iso_dw:i  | w:i    |iso_w:i | q:i    | y:i    | birth_date:ts        | l:s
+2         |2         |4         |245      |3          |36      |35      |3       |1953    |1953-09-02T00:00:00Z  |Facello
+2         |2         |3         |154      |2          |23      |22      |2       |1964    |1964-06-02T00:00:00Z  |Simmel
+3         |3         |5         |337      |4          |49      |49      |4       |1959    |1959-12-03T00:00:00Z  |Bamford
+1         |1         |7         |121      |6          |18      |18      |2       |1954    |1954-05-01T00:00:00Z  |Koblick
+21        |21        |6         |21       |5          |4       |3       |1       |1955    |1955-01-21T00:00:00Z  |Maliniak
+20        |20        |2         |110      |1          |17      |16      |2       |1953    |1953-04-20T00:00:00Z  |Preusig
+23        |23        |5         |143      |4          |21      |21      |2       |1957    |1957-05-23T00:00:00Z  |Zielinski
+19        |19        |4         |50       |3          |8       |8       |1       |1958    |1958-02-19T00:00:00Z  |Kalloufi
+19        |19        |7         |110      |6          |16      |16      |2       |1952    |1952-04-19T00:00:00Z  |Peac
+;
+
+
+dateExtractTimePartsTimeSecond
+SELECT
+SECOND(CAST(birth_date AS DATE)) d,
+MINUTE(CAST(birth_date AS DATE)) m,
+HOUR(CAST(birth_date AS DATE)) h
+FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no;
+
+ d:i     | m:i      | h:i
+0        |0         |0
+0        |0         |0
+0        |0         |0
+0        |0         |0
+0        |0         |0
+0        |0         |0
+0        |0         |0
+0        |0         |0
+0        |0         |0
+;
+
+dateAsFilter
+SELECT birth_date, last_name FROM "test_emp" WHERE birth_date <= CAST('1955-01-21' AS DATE) ORDER BY emp_no LIMIT 5;
+
+ birth_date:ts        | last_name:s
+1953-09-02T00:00:00Z  |Facello
+1954-05-01T00:00:00Z  |Koblick
+1955-01-21T00:00:00Z  |Maliniak
+1953-04-20T00:00:00Z  |Preusig
+1952-04-19T00:00:00Z  |Peac
+;
+
+dateAndFunctionAsGroupingKey
+SELECT MONTH(CAST(birth_date AS DATE)) AS m, CAST(SUM(emp_no) AS INT) s FROM test_emp GROUP BY m ORDER BY m LIMIT 5;
+
+ m:i        | s:i
+null        |100445
+1           |60288
+2           |80388
+3           |20164
+4           |80401
+;
+
+dateAndInterval
+SELECT YEAR(CAST('2019-01-21' AS DATE) + INTERVAL '1-2' YEAR TO MONTH) AS y, MONTH(INTERVAL '1-2' YEAR TO MONTH + CAST('2019-01-21' AS DATE)) AS m;
+
+y:i       | m:i
+2020      | 3
+;

x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec

@@ -6,8 +6,9 @@
 // Time NOT IMPLEMENTED in H2 on TIMESTAMP WITH TIME ZONE - hence why these are moved to CSV
 //
 
-// WEEK_OF_YEAR moved to CSV tests, because H2 builds its Calendar with the local Locale, we consider ROOT as the default Locale
-// This has implications on the results, which could change given specific locales where the rules for determining the start of a year are different.
+// WEEK_OF_YEAR moved to CSV tests, because H2 builds its Calendar with the local Locale,
+// we consider ROOT as the default Locale. This has implications on the results, which could
+// change given specific locales where the rules for determining the start of a year are different.
 
 //
 // DateTime
@@ -31,10 +32,10 @@ SELECT MONTHNAME(CAST('2018-09-03' AS TIMESTAMP)) month FROM "test_emp" limit 1;
 dayNameFromStringDateTime
 SELECT DAYNAME(CAST('2018-09-03' AS TIMESTAMP)) day FROM "test_emp" limit 1;
 
-quarterSelect
+dateTimeQuarter
 SELECT QUARTER(hire_date) q, hire_date FROM test_emp ORDER BY hire_date LIMIT 15;
 
-dayOfWeek
+dateTimeDayOfWeek
 SELECT DAY_OF_WEEK(birth_date) day, birth_date FROM test_emp ORDER BY DAY_OF_WEEK(birth_date);
 
 //

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java

@@ -95,7 +95,7 @@ public Object extract(Bucket bucket) {
             if (object == null) {
                 return object;
             } else if (object instanceof Long) {
-                object = DateUtils.of(((Long) object).longValue(), zoneId);
+                object = DateUtils.asDateTime(((Long) object).longValue(), zoneId);
             } else {
                 throw new SqlIllegalArgumentException("Invalid date key returned: {}", object);
             }
@@ -129,4 +129,4 @@ public boolean equals(Object obj) {
     public String toString() {
         return "|" + key + "|";
     }
-}
+}

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java

@@ -130,11 +130,11 @@ private Object unwrapMultiValue(Object values) {
         }
         if (dataType == DataType.DATETIME) {
             if (values instanceof String) {
-                return DateUtils.of(Long.parseLong(values.toString()));
+                return DateUtils.asDateTime(Long.parseLong(values.toString()));
             }
             // returned by nested types...
             if (values instanceof DateTime) {
-                return DateUtils.of((DateTime) values);
+                return DateUtils.asDateTime((DateTime) values);
             }
         }
         if (values instanceof Long || values instanceof Double || values instanceof String || values instanceof Boolean) {

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java

@@ -5,7 +5,6 @@
  */
 package org.elasticsearch.xpack.sql.expression;
 
-import org.elasticsearch.common.Strings;
 import org.elasticsearch.xpack.sql.SqlIllegalArgumentException;
 import org.elasticsearch.xpack.sql.expression.Expression.TypeResolution;
 import org.elasticsearch.xpack.sql.expression.gen.pipeline.Pipe;
@@ -16,11 +15,13 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Locale;
+import java.util.StringJoiner;
 import java.util.function.Predicate;
 
 import static java.lang.String.format;
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.sql.type.DataType.BOOLEAN;
 
 public final class Expressions {
 
@@ -155,7 +156,7 @@ public static List<Pipe> pipe(List<Expression> expressions) {
     }
 
     public static TypeResolution typeMustBeBoolean(Expression e, String operationName, ParamOrdinal paramOrd) {
-        return typeMustBe(e, dt -> dt == DataType.BOOLEAN, operationName, paramOrd, "boolean");
+        return typeMustBe(e, dt -> dt == BOOLEAN, operationName, paramOrd, "boolean");
     }
 
     public static TypeResolution typeMustBeInteger(Expression e, String operationName, ParamOrdinal paramOrd) {
@@ -171,11 +172,11 @@ public static TypeResolution typeMustBeString(Expression e, String operationName
     }
 
     public static TypeResolution typeMustBeDate(Expression e, String operationName, ParamOrdinal paramOrd) {
-        return typeMustBe(e, dt -> dt == DataType.DATETIME, operationName, paramOrd, "date");
+        return typeMustBe(e, DataType::isDateBased, operationName, paramOrd, "date", "datetime");
     }
 
     public static TypeResolution typeMustBeNumericOrDate(Expression e, String operationName, ParamOrdinal paramOrd) {
-        return typeMustBe(e, dt -> dt.isNumeric() || dt == DataType.DATETIME, operationName, paramOrd, "numeric", "date");
+        return typeMustBe(e, dt -> dt.isNumeric() || dt.isDateBased(), operationName, paramOrd, "date", "datetime", "numeric");
     }
 
     public static TypeResolution typeMustBe(Expression e,
@@ -188,8 +189,20 @@ public static TypeResolution typeMustBe(Expression e,
             new TypeResolution(format(Locale.ROOT, "[%s]%s argument must be [%s], found value [%s] type [%s]",
                 operationName,
                 paramOrd == null || paramOrd == ParamOrdinal.DEFAULT ? "" : " " + paramOrd.name().toLowerCase(Locale.ROOT),
-                Strings.arrayToDelimitedString(acceptedTypes, " or "),
+                acceptedTypesForErrorMsg(acceptedTypes),
                 Expressions.name(e),
                 e.dataType().esType));
     }
+
+    private static String acceptedTypesForErrorMsg(String... acceptedTypes) {
+        StringJoiner sj = new StringJoiner(", ");
+        for (int i = 0; i < acceptedTypes.length - 1; i++) {
+            sj.add(acceptedTypes[i]);
+        }
+        if (acceptedTypes.length > 1) {
+            return sj.toString() + " or " + acceptedTypes[acceptedTypes.length - 1];
+        } else {
+            return acceptedTypes[0];
+        }
+    }
 }

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Max.java

@@ -47,4 +47,4 @@ public String innerName() {
     protected TypeResolution resolveType() {
         return Expressions.typeMustBeNumericOrDate(field(), sourceText(), ParamOrdinal.DEFAULT);
     }
-}
+}

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java

@@ -42,7 +42,7 @@ protected TypeResolution resolveType() {
         TypeResolution resolution = Expressions.typeMustBeNumericOrDate(field(), "HISTOGRAM", ParamOrdinal.FIRST);
         if (resolution == TypeResolution.TYPE_RESOLVED) {
             // interval must be Literal interval
-            if (field().dataType() == DataType.DATETIME) {
+            if (field().dataType().isDateBased()) {
                 resolution = Expressions.typeMustBe(interval, DataTypes::isInterval, "(Date) HISTOGRAM", ParamOrdinal.SECOND, "interval");
             } else {
                 resolution = Expressions.typeMustBeNumeric(interval, "(Numeric) HISTOGRAM", ParamOrdinal.SECOND);

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/BaseDateTimeFunction.java

@@ -74,4 +74,4 @@ public boolean equals(Object obj) {
     public int hashCode() {
         return Objects.hash(field(), zoneId());
     }
-}
+}

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java

@@ -357,11 +357,11 @@ private static Object asDateTime(Object dateTime, boolean lenient) {
             return ((JodaCompatibleZonedDateTime) dateTime).getZonedDateTime();
         }
         if (dateTime instanceof ZonedDateTime) {
-            return (ZonedDateTime) dateTime;
+            return dateTime;
         }
         if (false == lenient) {
             if (dateTime instanceof Number) {
-                return DateUtils.of(((Number) dateTime).longValue());
+                return DateUtils.asDateTime(((Number) dateTime).longValue());
             }
 
             throw new SqlIllegalArgumentException("Invalid date encountered [{}]", dateTime);

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/gen/script/ScriptWeaver.java

@@ -79,7 +79,7 @@ default ScriptTemplate scriptWithScalar(ScalarFunctionAttribute scalar) {
 
     default ScriptTemplate scriptWithAggregate(AggregateFunctionAttribute aggregate) {
         String template = "{}";
-        if (aggregate.dataType() == DataType.DATETIME) {
+        if (aggregate.dataType().isDateBased()) {
             template = "{sql}.asDateTime({})";
         }
         return new ScriptTemplate(processScript(template),
@@ -89,7 +89,7 @@ default ScriptTemplate scriptWithAggregate(AggregateFunctionAttribute aggregate)
 
     default ScriptTemplate scriptWithGrouping(GroupingFunctionAttribute grouping) {
         String template = "{}";
-        if (grouping.dataType() == DataType.DATETIME) {
+        if (grouping.dataType().isDateBased()) {
             template = "{sql}.asDateTime({})";
         }
         return new ScriptTemplate(processScript(template),