SQL: Introduce SQL DATE data type

docs/reference/sql/language/data-types.asciidoc

@@ -7,7 +7,10 @@ beta[]
 
 Most of {es} <<mapping-types, data types>> are available in {es-sql}, as indicated below.
 As one can see, all of {es} <<mapping-types, data types>> are mapped to the data type with the same
-name in {es-sql}, with the exception of **date** data type which is mapped to **datetime** in {es-sql}:
+name in {es-sql}, with the exception of **date** data type which is mapped to **datetime** in {es-sql}.
+This is to avoid confusion with the ANSI SQL date (date only) type, which is also supported by {es-sql}
+in queries (with the use of <<sql-functions-type-conversion-cast>>/<<sql-functions-type-conversion-convert>>),
+but doesn't correspond to an actual mapping in {es}.
 
 [cols="^,^m,^,^"]
 
@@ -33,6 +36,7 @@ s|SQL precision
 | <<text, `text`>>            | text          | VARCHAR     | 2,147,483,647
 | <<binary, `binary`>>        | binary        | VARBINARY   | 2,147,483,647
 | <<date, `date`>>            | datetime      | TIMESTAMP   | 24
+|                             | date          | DATE        | 24
 | <<ip, `ip`>>                | ip            | VARCHAR     | 39
 
 4+h| Complex types

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/EsType.java

@@ -28,6 +28,7 @@ public enum EsType implements SQLType {
     OBJECT(                   Types.STRUCT),
     NESTED(                   Types.STRUCT),
     BINARY(                   Types.VARBINARY),
+    DATE(                     Types.DATE),
     DATETIME(                 Types.TIMESTAMP),
     IP(                       Types.VARCHAR),
     INTERVAL_YEAR(            ExtraTypes.INTERVAL_YEAR),

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcDateUtils.java

@@ -12,6 +12,7 @@
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.DateTimeParseException;
 import java.util.Locale;
 import java.util.function.Function;
 
@@ -41,9 +42,25 @@ final class JdbcDateUtils {
         .appendFraction(MILLI_OF_SECOND, 3, 3, true)
         .appendOffsetId()
         .toFormatter(Locale.ROOT);
+
+    static final DateTimeFormatter ISO_WITH_MINUTES = new DateTimeFormatterBuilder()
+        .parseCaseInsensitive()
+        .append(ISO_LOCAL_DATE)
+        .appendLiteral('T')
+        .appendValue(HOUR_OF_DAY, 2)
+        .appendLiteral(':')
+        .appendValue(MINUTE_OF_HOUR, 2)
+        .appendOffsetId()
+        .toFormatter(Locale.ROOT);
 
     static long asMillisSinceEpoch(String date) {
-        ZonedDateTime zdt = ISO_WITH_MILLIS.parse(date, ZonedDateTime::from);
+        ZonedDateTime zdt;
+        try {
+            zdt = ISO_WITH_MILLIS.parse(date, ZonedDateTime::from);
+        } catch (DateTimeParseException e) {
+            // Case of a Group By with casting as DATE
+            zdt = ISO_WITH_MINUTES.parse(date, ZonedDateTime::from);
+        }
         return zdt.toInstant().toEpochMilli();
     }
 

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/JdbcResultSet.java

@@ -253,7 +253,10 @@ private Long dateTime(int columnIndex) throws SQLException {
                     return null;
                 }
                 return JdbcDateUtils.asDateTimeField(val, JdbcDateUtils::asMillisSinceEpoch, Function.identity());
-            };
+            }
+            if (EsType.DATE == type) {
+                return JdbcDateUtils.asDate((String) val).getTime();
+            }
             return val == null ? null : (Long) val;
         } catch (ClassCastException cce) {
             throw new SQLException(

x-pack/plugin/sql/jdbc/src/main/java/org/elasticsearch/xpack/sql/jdbc/TypeConverter.java

@@ -213,6 +213,8 @@ static Object convert(Object v, EsType columnType, String typeString) throws SQL
                 return doubleValue(v); // Double might be represented as string for infinity and NaN values
             case FLOAT:
                 return floatValue(v); // Float might be represented as string for infinity and NaN values
+            case DATE:
+                return JdbcDateUtils.asDate(v.toString());
             case DATETIME:
                 return JdbcDateUtils.asDateTimeField(v, JdbcDateUtils::asTimestamp, Timestamp::new);
             case INTERVAL_YEAR:

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/CsvSpecTestCase.java

@@ -36,6 +36,7 @@ public static List<Object[]> readScriptSpec() throws Exception {
         tests.addAll(readScriptSpec("/fulltext.csv-spec", parser));
         tests.addAll(readScriptSpec("/agg.csv-spec", parser));
         tests.addAll(readScriptSpec("/columns.csv-spec", parser));
+        tests.addAll(readScriptSpec("/date.csv-spec", parser));
         tests.addAll(readScriptSpec("/datetime.csv-spec", parser));
         tests.addAll(readScriptSpec("/alias.csv-spec", parser));
         tests.addAll(readScriptSpec("/null.csv-spec", parser));

x-pack/plugin/sql/qa/src/main/java/org/elasticsearch/xpack/sql/qa/jdbc/JdbcAssert.java

@@ -139,6 +139,7 @@ public static void assertResultSetMetadata(ResultSet expected, ResultSet actual,
             if (expectedType == Types.TIMESTAMP_WITH_TIMEZONE) {
                 expectedType = Types.TIMESTAMP;
             }
+
             // since csv doesn't support real, we use float instead.....
             if (expectedType == Types.FLOAT && expected instanceof CsvResultSet) {
                 expectedType = Types.REAL;
@@ -204,6 +205,9 @@ private static void doAssertResultSetData(ResultSet expected, ResultSet actual,
                         // fix for CSV which returns the shortName not fully-qualified name
                         if (!columnClassName.contains(".")) {
                             switch (columnClassName) {
+                                case "Date":
+                                    columnClassName = "java.sql.Date";
+                                    break;
                                 case "Timestamp":
                                     columnClassName = "java.sql.Timestamp";
                                     break;

x-pack/plugin/sql/qa/src/main/resources/date.csv-spec

@@ -0,0 +1,89 @@
+//
+// Date
+//
+
+dateExtractDateParts
+SELECT
+DAY(CAST(birth_date AS DATE)) d,
+DAY_OF_MONTH(CAST(birth_date AS DATE)) dm,
+DAY_OF_WEEK(CAST(birth_date AS DATE)) dw,
+DAY_OF_YEAR(CAST(birth_date AS DATE)) dy,
+ISO_DAY_OF_WEEK(CAST(birth_date AS DATE)) iso_dw,
+WEEK(CAST(birth_date AS DATE)) w,
+QUARTER(CAST(hire_date AS DATE)) q,
+YEAR(CAST(birth_date AS DATE)) y,
+birth_date, last_name l FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no;
+
+ d:i      | dm:i     | dw:i     | dy:i    | iso_dw:i  | w:i    | q:i    | y:i    | birth_date:ts        | l:s
+2         |2         |4         |245      |3          |36      |2       |1953    |1953-09-02T00:00:00Z  |Facello
+2         |2         |3         |154      |2          |23      |4       |1964    |1964-06-02T00:00:00Z  |Simmel
+3         |3         |5         |337      |4          |49      |3       |1959    |1959-12-03T00:00:00Z  |Bamford
+1         |1         |7         |121      |6          |18      |4       |1954    |1954-05-01T00:00:00Z  |Koblick
+21        |21        |6         |21       |5          |4       |3       |1955    |1955-01-21T00:00:00Z  |Maliniak
+20        |20        |2         |110      |1          |17      |2       |1953    |1953-04-20T00:00:00Z  |Preusig
+23        |23        |5         |143      |4          |21      |1       |1957    |1957-05-23T00:00:00Z  |Zielinski
+19        |19        |4         |50       |3          |8       |3       |1958    |1958-02-19T00:00:00Z  |Kalloufi
+19        |19        |7         |110      |6          |16      |1       |1952    |1952-04-19T00:00:00Z  |Peac
+;
+
+
+dateExtractTimePartsTimeSecond
+SELECT
+SECOND(CAST(birth_date AS DATE)) d,
+MINUTE(CAST(birth_date AS DATE)) m,
+HOUR(CAST(birth_date AS DATE)) h
+FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no;
+
+ d:i     | m:i      | h:i
+0        |0         |0
+0        |0         |0
+0        |0         |0
+0        |0         |0
+0        |0         |0
+0        |0         |0
+0        |0         |0
+0        |0         |0
+0        |0         |0
+;
+
+dateAsFilter
+SELECT birth_date, last_name FROM "test_emp" WHERE birth_date <= CAST('1955-01-21' AS DATE) ORDER BY emp_no LIMIT 5;
+
+ birth_date:ts        | last_name:s
+1953-09-02T00:00:00Z  |Facello
+1954-05-01T00:00:00Z  |Koblick
+1955-01-21T00:00:00Z  |Maliniak
+1953-04-20T00:00:00Z  |Preusig
+1952-04-19T00:00:00Z  |Peac
+;
+
+
+// The date return can differ due to the randomized timezones (it's transformed to the JDBC client's timezone)
+dateAsGroupingKey-Ignore
+SELECT CAST(birth_date AS DATE) AS d, CAST(SUM(emp_no) AS INT) s FROM test_emp GROUP BY d ORDER BY d LIMIT 5;
+
+ d:date     | s:i
+null        |100445
+1952-02-26  |10097
+1952-04-18  |10009
+1952-05-14  |10072
+1952-06-12  |10076
+;
+
+dateAndFunctionAsGroupingKey
+SELECT MONTH(CAST(birth_date AS DATE)) AS m, CAST(SUM(emp_no) AS INT) s FROM test_emp GROUP BY m ORDER BY m LIMIT 5;
+
+ m:i        | s:i
+null        |100445
+1           |60288
+2           |80388
+3           |20164
+4           |80401
+;
+
+dateAndInterval
+SELECT YEAR(CAST('2019-01-21' AS DATE) + INTERVAL '1-2' YEAR TO MONTH) AS y, MONTH(CAST('2019-01-21' AS DATE) + INTERVAL '1-2' YEAR TO MONTH) AS m;
+
+y:i       | m:i
+2020      | 3
+;

x-pack/plugin/sql/qa/src/main/resources/datetime.sql-spec

@@ -6,8 +6,9 @@
 // Time NOT IMPLEMENTED in H2 on TIMESTAMP WITH TIME ZONE - hence why these are moved to CSV
 //
 
-// WEEK_OF_YEAR moved to CSV tests, because H2 builds its Calendar with the local Locale, we consider ROOT as the default Locale
-// This has implications on the results, which could change given specific locales where the rules for determining the start of a year are different.
+// WEEK_OF_YEAR moved to CSV tests, because H2 builds its Calendar with the local Locale,
+// we consider ROOT as the default Locale. This has implications on the results, which could
+// change given specific locales where the rules for determining the start of a year are different.
 
 //
 // DateTime
@@ -31,10 +32,10 @@ SELECT MONTHNAME(CAST('2018-09-03' AS TIMESTAMP)) month FROM "test_emp" limit 1;
 dayNameFromStringDateTime
 SELECT DAYNAME(CAST('2018-09-03' AS TIMESTAMP)) day FROM "test_emp" limit 1;
 
-quarterSelect
+dateTimeQuarter
 SELECT QUARTER(hire_date) q, hire_date FROM test_emp ORDER BY hire_date LIMIT 15;
 
-dayOfWeek
+dateTimeDayOfWeek
 SELECT DAY_OF_WEEK(birth_date) day, birth_date FROM test_emp ORDER BY DAY_OF_WEEK(birth_date);
 
 //

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/CompositeKeyExtractor.java

@@ -95,7 +95,7 @@ public Object extract(Bucket bucket) {
             if (object == null) {
                 return object;
             } else if (object instanceof Long) {
-                object = DateUtils.of(((Long) object).longValue(), zoneId);
+                object = DateUtils.asDateTime(((Long) object).longValue(), zoneId);
             } else {
                 throw new SqlIllegalArgumentException("Invalid date key returned: {}", object);
             }
@@ -129,4 +129,4 @@ public boolean equals(Object obj) {
     public String toString() {
         return "|" + key + "|";
     }
-}
+}

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/execution/search/extractor/FieldHitExtractor.java

@@ -130,11 +130,11 @@ private Object unwrapMultiValue(Object values) {
         }
         if (dataType == DataType.DATETIME) {
             if (values instanceof String) {
-                return DateUtils.of(Long.parseLong(values.toString()));
+                return DateUtils.asDateTime(Long.parseLong(values.toString()));
             }
             // returned by nested types...
             if (values instanceof DateTime) {
-                return DateUtils.of((DateTime) values);
+                return DateUtils.asDateTime((DateTime) values);
             }
         }
         if (values instanceof Long || values instanceof Double || values instanceof String || values instanceof Boolean) {

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Expressions.java

@@ -21,6 +21,9 @@
 import static java.lang.String.format;
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
+import static org.elasticsearch.xpack.sql.type.DataType.BOOLEAN;
+import static org.elasticsearch.xpack.sql.type.DataType.DATE;
+import static org.elasticsearch.xpack.sql.type.DataType.DATETIME;
 
 public final class Expressions {
 
@@ -155,7 +158,7 @@ public static List<Pipe> pipe(List<Expression> expressions) {
     }
 
     public static TypeResolution typeMustBeBoolean(Expression e, String operationName, ParamOrdinal paramOrd) {
-        return typeMustBe(e, dt -> dt == DataType.BOOLEAN, operationName, paramOrd, "boolean");
+        return typeMustBe(e, dt -> dt == BOOLEAN, operationName, paramOrd, "boolean");
     }
 
     public static TypeResolution typeMustBeInteger(Expression e, String operationName, ParamOrdinal paramOrd) {
@@ -170,12 +173,12 @@ public static TypeResolution typeMustBeString(Expression e, String operationName
         return typeMustBe(e, DataType::isString, operationName, paramOrd, "string");
     }
 
-    public static TypeResolution typeMustBeDate(Expression e, String operationName, ParamOrdinal paramOrd) {
-        return typeMustBe(e, dt -> dt == DataType.DATETIME, operationName, paramOrd, "date");
+    public static TypeResolution typeMustBeDateOrDateTime(Expression e, String operationName, ParamOrdinal paramOrd) {
+        return typeMustBe(e, dt -> dt == DATETIME || dt == DATE, operationName, paramOrd, "datetime, date");
     }
 
-    public static TypeResolution typeMustBeNumericOrDate(Expression e, String operationName, ParamOrdinal paramOrd) {
-        return typeMustBe(e, dt -> dt.isNumeric() || dt == DataType.DATETIME, operationName, paramOrd, "numeric", "date");
+    public static TypeResolution typeMustBeNumericOrDateTimeOrDate(Expression e, String operationName, ParamOrdinal paramOrd) {
+        return typeMustBe(e, dt -> dt.isNumeric() || dt == DATETIME, operationName, paramOrd, "numeric", "datetime", "date");
     }
 
     public static TypeResolution typeMustBe(Expression e,

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/Literal.java

@@ -38,7 +38,11 @@ public Literal(Source source, Object value, DataType dataType) {
     public Literal(Source source, String name, Object value, DataType dataType) {
         super(source, name == null ? source.text() : name, emptyList(), null);
         this.dataType = dataType;
-        this.value = DataTypeConversion.convert(value, dataType);
+//        if (dataType == DATE) {
+//            this.value = value;
+//        } else {
+            this.value = DataTypeConversion.convert(value, dataType);
+//        }
     }
 
     @Override

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Max.java

@@ -45,6 +45,6 @@ public String innerName() {
 
     @Override
     protected TypeResolution resolveType() {
-        return Expressions.typeMustBeNumericOrDate(field(), sourceText(), ParamOrdinal.DEFAULT);
+        return Expressions.typeMustBeNumericOrDateTimeOrDate(field(), sourceText(), ParamOrdinal.DEFAULT);
     }
-}
+}

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/aggregate/Min.java

@@ -48,6 +48,6 @@ public String innerName() {
 
     @Override
     protected TypeResolution resolveType() {
-        return Expressions.typeMustBeNumericOrDate(field(), sourceText(), ParamOrdinal.DEFAULT);
+        return Expressions.typeMustBeNumericOrDateTimeOrDate(field(), sourceText(), ParamOrdinal.DEFAULT);
     }
 }

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/grouping/Histogram.java

@@ -39,7 +39,7 @@ public ZoneId zoneId() {
 
     @Override
     protected TypeResolution resolveType() {
-        TypeResolution resolution = Expressions.typeMustBeNumericOrDate(field(), "HISTOGRAM", ParamOrdinal.FIRST);
+        TypeResolution resolution = Expressions.typeMustBeNumericOrDateTimeOrDate(field(), "HISTOGRAM", ParamOrdinal.FIRST);
         if (resolution == TypeResolution.TYPE_RESOLVED) {
             // interval must be Literal interval
             if (field().dataType() == DataType.DATETIME) {

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/BaseDateTimeFunction.java

@@ -35,7 +35,7 @@ protected final NodeInfo<BaseDateTimeFunction> info() {
 
     @Override
     protected TypeResolution resolveType() {
-        return Expressions.typeMustBeDate(field(), sourceText(), ParamOrdinal.DEFAULT);
+        return Expressions.typeMustBeDateOrDateTime(field(), sourceText(), ParamOrdinal.DEFAULT);
     }
 
     public ZoneId zoneId() {
@@ -74,4 +74,4 @@ public boolean equals(Object obj) {
     public int hashCode() {
         return Objects.hash(field(), zoneId());
     }
-}
+}

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java

@@ -361,7 +361,7 @@ private static Object asDateTime(Object dateTime, boolean lenient) {
         }
         if (false == lenient) {
             if (dateTime instanceof Number) {
-                return DateUtils.of(((Number) dateTime).longValue());
+                return DateUtils.asDateTime(((Number) dateTime).longValue());
             }
 
             throw new SqlIllegalArgumentException("Invalid date encountered [{}]", dateTime);

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java

@@ -411,6 +411,8 @@ public DataType visitPrimitiveDataType(PrimitiveDataTypeContext ctx) {
             case "float":
             case "double":
                 return DataType.DOUBLE;
+            case "date":
+                return DataType.DATE;
             case "datetime":
             case "timestamp":
                 return DataType.DATETIME;
@@ -793,7 +795,7 @@ public Literal visitDateEscapedLiteral(DateEscapedLiteralContext ctx) {
         } catch(IllegalArgumentException ex) {
             throw new ParsingException(source, "Invalid date received; {}", ex.getMessage());
         }
-        return new Literal(source, DateUtils.of(dt), DataType.DATETIME);
+        return new Literal(source, DateUtils.asDate(dt), DataType.DATE);
     }
 
     @Override
@@ -829,7 +831,7 @@ public Literal visitTimestampEscapedLiteral(TimestampEscapedLiteralContext ctx)
         } catch (IllegalArgumentException ex) {
             throw new ParsingException(source, "Invalid timestamp received; {}", ex.getMessage());
         }
-        return new Literal(source, DateUtils.of(dt), DataType.DATETIME);
+        return new Literal(source, DateUtils.asDateTime(dt), DataType.DATETIME);
     }
 
     @Override

x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java

@@ -45,6 +45,7 @@ public enum DataType {
     // the precision is 23 (number of chars in ISO8601 with millis) + Z (the UTC timezone)
     // see https://github.com/elastic/elasticsearch/issues/30386#issuecomment-386807288
     DATETIME(              JDBCType.TIMESTAMP, Long.BYTES,        24,                24, false, false, true),
+    DATE(                  JDBCType.DATE,      Long.BYTES,        10,                10, false, false, true),
     //
     // specialized types
     //
@@ -102,7 +103,7 @@ public enum DataType {
         odbcToEs.put("SQL_LONGVARBINARY", BINARY);
 
         // Date
-        odbcToEs.put("SQL_DATE", DATETIME);
+        odbcToEs.put("SQL_DATE", DATE);
         odbcToEs.put("SQL_TIME", DATETIME);
         odbcToEs.put("SQL_TIMESTAMP", DATETIME);