DATE_FORMAT function (#764)

* Bug fix, support long type for aggregation (#522) * Bug fix, support long type for aggregation * change to datetime to JDBC format * Opendistro Release 1.9.0 (#532) * prepare odfe 1.9 * Fix all ES 7.8 compile and build errors * Revert changes as Lombok is working now * Update CustomExternalTestCluster.java * Fix license headers check * Use splitFieldsByMetadata to separate fields when calling SearchHit constructor * More fixes for ODFE 1.9 * Remove todo statement * Add ODFE 1.9.0 release notes * Rename release notes to use 4 digit versions (#547) * Revert changes ahead of develop branch in master (#551) * Revert "Rename release notes to use 4 digit versions (#547)" This reverts commit 33c6d3e. * Revert "Opendistro Release 1.9.0 (#532)" This reverts commit 254f2e0. * Revert "Bug fix, support long type for aggregation (#522)" This reverts commit fb2ed91. * Merge all SQL repos and adjust workflows (#549) (#554) * merge all sql repos * fix test and build workflows * fix workbench and odbc path * fix workbench and odbc path * restructure workbench dir and fix workflows * fix workbench workflow * fix workbench workflow * fix workbench workflow * fix workbench workflow * fix workbench workflow * revert workbench directory structure * fix workbench workflow * fix workbench workflow * fix workbench workflow * fix workbench workflow * update workbench workflow for release * Delete .github/ in sql-workbench directory * Add cypress to sql-workbench * Sync latest ODBC commits * Sync latest workbench commits (will add cypress in separate PR) * Add ignored ODBC libs * add date and time support (#560) * add date and time support * update doc * update doc * Revert "add date and time support (#560)" (#567) This reverts commit 4b33a2f. * add error details for all server communication errors (#645) - add null check to avoid crashing if details not initialized * Revert "add error details for all server communication errors (#645)" (#653) This reverts commit c11125d. * Fix download link in package description (#729) * add functions day, month, quarter, year * fix build error * fix doctest error * fix doctest build error * fix doctest * add dayofmonth() * add dayofyear() * add dayofweek() * fix dayofweek logic & add unit test * fix doctest for dayofweek() * add dayname * add monthname * fix checkstyle build error * fix build error * fix doctest for monthname * add hour() * add minute() * add second * add microsecond * fix datetime & timestamp issue for microsecond * add time_to_sec * add subdate & date_sub * fix doctest error * fix build error * add KeywordsCanBeId for dayofweek * add to_days * add from_days() * arrange by alphabetical order * add manual IT * add string input for date functions * fix microsecond * update doc * add date_add * add week * address PR comments * update tests & doc * fix doc format * update tests for adddate * move string conversion to ExprStringValue * add string type in doc * edge case * fix case 5 & 7 * add IT * update doc * fix table * rename * add string type * nit: add newline * fix type in comment * nit * add test cases for datetime function in ExpeStringValue * removing implicit def for keyword in parser * add dayofweek * [1] Merged rupals week branch in * add unit tests for null, missing values * nit * [1] Added integration tests. * [1] Working on ppl integration tests. * [1] Updated for integration tests * [1] Fixed schema verification * [1] Adding documentation. * [1] Fixing documentation * [1] Simplified a bunch of logic * [1] Reducing changes that are from spacing * [1] Removed extra merge line * address PR comment * [1] Updates * [1] Removed some unwanted changes. * [1] Minor whitespace adjustements * [1] Updating based on code review Co-authored-by: Peng Huo <penghuo@gmail.com> Co-authored-by: Joshua <joshuali925@gmail.com> Co-authored-by: Joshua Li <lijshu@amazon.com> Co-authored-by: Jordan Wilson <37088125+jordanw-bq@users.noreply.github.com> Co-authored-by: Chloe <chloezh1102@gmail.com> Co-authored-by: chloe-zh <fizhang@amazon.com> Co-authored-by: Sayali Gaikawad <61760125+gaiksaya@users.noreply.github.com> Co-authored-by: Rupal Mahajan <>
opendistro-for-elasticsearch · Oct 2, 2020 · f33c0e6 · f33c0e6
1 parent d75b7f1
commit f33c0e6
Show file tree

Hide file tree

Showing 13 changed files with 450 additions and 20 deletions.
diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java
@@ -21,9 +21,7 @@
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.LocalTime;
-import java.time.format.DateTimeParseException;
 import java.util.Objects;
-import lombok.EqualsAndHashCode;
 import lombok.RequiredArgsConstructor;
 
 /**

diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java
@@ -329,6 +329,10 @@ public FunctionExpression timestamp(Expression... expressions) {
  return function(BuiltinFunctionName.TIMESTAMP, expressions);
  }
 
+ public FunctionExpression date_format(Expression... expressions) {
+ return function(BuiltinFunctionName.DATE_FORMAT, expressions);
+ }
+
  public FunctionExpression to_days(Expression... expressions) {
  return function(BuiltinFunctionName.TO_DAYS, expressions);
  }

diff --git a/...in/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/CalendarLookup.java b/...in/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/CalendarLookup.java
@@ -47,9 +47,9 @@ private static Calendar getCalendar(int mode, LocalDate date) {
 
  /**
  * Set first day of week, minimal days in first week and date in calendar.
- * @param firstDayOfWeek  the given first day of the week.
+ * @param firstDayOfWeek the given first day of the week.
  * @param minimalDaysInWeek the given minimal days required in the first week of the year.
- * @param date  the given date.
+ * @param date the given date.
  */
  private static Calendar getCalendar(int firstDayOfWeek, int minimalDaysInWeek, LocalDate date) {
  Calendar calendar = Calendar.getInstance();
@@ -74,4 +74,19 @@ static int getWeekNumber(int mode, LocalDate date) {
  }
  return weekNumber;
  }
-}
+
+ /**
+ * Returns year for date according to mode.
+ * @param mode Integer for mode. Valid mode values are 0 to 7.
+ * @param date LocalDate for date.
+ */
+ static int getYearNumber(int mode, LocalDate date) {
+ Calendar calendar = getCalendar(mode, date);
+ int weekNumber = getWeekNumber(mode, date);
+ int yearNumber = calendar.get(Calendar.YEAR);
+ if ((weekNumber > 51) && (calendar.get(Calendar.DAY_OF_MONTH) < 7)) {
+ yearNumber--;
+ }
+ return yearNumber;
+ }
+}
diff --git a/.../com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFormatterUtil.java b/.../com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFormatterUtil.java
@@ -0,0 +1,120 @@
+package com.amazon.opendistroforelasticsearch.sql.expression.datetime;
+
+import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue;
+import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
+import com.google.common.collect.ImmutableMap;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * This class converts a SQL style DATE_FORMAT format specifier and converts it to a
+ * Java SimpleDateTime format.
+ */
+class DateTimeFormatterUtil {
+ private static final int SUFFIX_SPECIAL_START_TH = 11;
+ private static final int SUFFIX_SPECIAL_END_TH = 13;
+ private static final String SUFFIX_SPECIAL_TH = "th";
+ private static final Map<Integer, String> SUFFIX_CONVERTER =
+ ImmutableMap.<Integer, String>builder()
+ .put(1, "st").put(2, "nd").put(3, "rd").build();
+
+ // The following have special cases that need handling outside of the format options provided
+ // by the DateTimeFormatter class.
+ interface DateTimeFormatHandler {
+ String getFormat(LocalDateTime date);
+ }
+
+ private static final Map<String, DateTimeFormatHandler> HANDLERS =
+ ImmutableMap.<String, DateTimeFormatHandler>builder()
+ .put("%a", (date) -> "EEE") // %a => EEE - Abbreviated weekday name (Sun..Sat)
+ .put("%b", (date) -> "LLL") // %b => LLL - Abbreviated month name (Jan..Dec)
+ .put("%c", (date) -> "MM") // %c => MM - Month, numeric (0..12)
+ .put("%d", (date) -> "dd") // %d => dd - Day of the month, numeric (00..31)
+ .put("%e", (date) -> "d") // %e => d - Day of the month, numeric (0..31)
+ .put("%H", (date) -> "HH") // %H => HH - (00..23)
+ .put("%h", (date) -> "hh") // %h => hh - (01..12)
+ .put("%I", (date) -> "hh") // %I => hh - (01..12)
+ .put("%i", (date) -> "mm") // %i => mm - Minutes, numeric (00..59)
+ .put("%j", (date) -> "DDD") // %j => DDD - (001..366)
+ .put("%k", (date) -> "H") // %k => H - (0..23)
+ .put("%l", (date) -> "h") // %l => h - (1..12)
+ .put("%p", (date) -> "a") // %p => a - AM or PM
+ .put("%M", (date) -> "LLLL") // %M => LLLL - Month name (January..December)
+ .put("%m", (date) -> "MM") // %m => MM - Month, numeric (00..12)
+ .put("%r", (date) -> "hh:mm:ss a") // %r => hh:mm:ss a - hh:mm:ss followed by AM or PM
+ .put("%S", (date) -> "ss") // %S => ss - Seconds (00..59)
+ .put("%s", (date) -> "ss") // %s => ss - Seconds (00..59)
+ .put("%T", (date) -> "HH:mm:ss") // %T => HH:mm:ss
+ .put("%W", (date) -> "EEEE") // %W => EEEE - Weekday name (Sunday..Saturday)
+ .put("%Y", (date) -> "yyyy") // %Y => yyyy - Year, numeric, 4 digits
+ .put("%y", (date) -> "yy") // %y => yy - Year, numeric, 2 digits
+ // The following are not directly supported by DateTimeFormatter.
+ .put("%D", (date) -> // %w - Day of month with English suffix
+ String.format("'%d%s'", date.getDayOfMonth(), getSuffix(date.getDayOfMonth())))
+ .put("%f", (date) -> // %f - Microseconds
+ String.format("'%d'", (date.getNano() / 1000)))
+ .put("%w", (date) -> // %w - Day of week (0 indexed)
+ String.format("'%d'", date.getDayOfWeek().getValue()))
+ .put("%U", (date) -> // %U Week where Sunday is the first day - WEEK() mode 0
+ String.format("'%d'", CalendarLookup.getWeekNumber(0, date.toLocalDate())))
+ .put("%u", (date) -> // %u Week where Monday is the first day - WEEK() mode 1
+ String.format("'%d'", CalendarLookup.getWeekNumber(1, date.toLocalDate())))
+ .put("%V", (date) -> // %V Week where Sunday is the first day - WEEK() mode 2 used with %X
+ String.format("'%d'", CalendarLookup.getWeekNumber(2, date.toLocalDate())))
+ .put("%v", (date) -> // %v Week where Monday is the first day - WEEK() mode 3 used with %x
+ String.format("'%d'", CalendarLookup.getWeekNumber(3, date.toLocalDate())))
+ .put("%X", (date) -> // %X Year for week where Sunday is the first day, 4 digits used with %V
+ String.format("'%d'", CalendarLookup.getYearNumber(2, date.toLocalDate())))
+ .put("%x", (date) -> // %x Year for week where Monday is the first day, 4 digits used with %v
+ String.format("'%d'", CalendarLookup.getYearNumber(3, date.toLocalDate())))
+ .build();
+
+ private static final Pattern pattern = Pattern.compile("%.");
+ private static final String MOD_LITERAL = "%";
+
+ private DateTimeFormatterUtil() {
+ }
+
+ /**
+ * Format the date using the date format String.
+ * @param dateExpr the date ExprValue of Date/Datetime/Timestamp/String type.
+ * @param formatExpr the format ExprValue of String type.
+ * @return Date formatted using format and returned as a String.
+ */
+ static ExprValue getFormattedDate(ExprValue dateExpr, ExprValue formatExpr) {
+ final LocalDateTime date = dateExpr.datetimeValue();
+ final Matcher matcher = pattern.matcher(formatExpr.stringValue());
+ final StringBuffer format = new StringBuffer();
+ while (matcher.find()) {
+ matcher.appendReplacement(format,
+ HANDLERS.getOrDefault(matcher.group(), (d) ->
+ String.format("'%s'", matcher.group().replaceFirst(MOD_LITERAL, "")))
+ .getFormat(date));
+ }
+ matcher.appendTail(format);
+
+ // English Locale matches SQL requirements.
+ // 'AM'/'PM' instead of 'a.m.'/'p.m.'
+ // 'Sat' instead of 'Sat.' etc
+ return new ExprStringValue(date.format(
+ DateTimeFormatter.ofPattern(format.toString(), Locale.ENGLISH)));
+ }
+
+ /**
+ * Returns English suffix of incoming value.
+ * @param val Incoming value.
+ * @return English suffix as String (st, nd, rd, th)
+ */
+ private static String getSuffix(int val) {
+ // The numbers 11, 12, and 13 do not follow general suffix rules.
+ if ((SUFFIX_SPECIAL_START_TH <= val) && (val <= SUFFIX_SPECIAL_END_TH)) {
+ return SUFFIX_SPECIAL_TH;
+ }
+ return SUFFIX_CONVERTER.getOrDefault(val % 10, SUFFIX_SPECIAL_TH);
+ }
+}
diff --git a/.../java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java b/.../java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java
@@ -85,6 +85,7 @@ public void register(BuiltinFunctionRepository repository) {
  repository.register(time());
  repository.register(time_to_sec());
  repository.register(timestamp());
+ repository.register(date_format());
  repository.register(to_days());
  repository.register(week());
  repository.register(year());
@@ -400,6 +401,27 @@ private FunctionResolver year() {
  );
  }
 
+ /**
+ * Formats date according to format specifier. First argument is date, second is format.
+ * Detailed supported signatures:
+ * (STRING, STRING) -> STRING
+ * (DATE, STRING) -> STRING
+ * (DATETIME, STRING) -> STRING
+ * (TIMESTAMP, STRING) -> STRING
+ */
+ private FunctionResolver date_format() {
+ return define(BuiltinFunctionName.DATE_FORMAT.getName(),
+ impl(nullMissingHandling(DateTimeFormatterUtil::getFormattedDate),
+ STRING, STRING, STRING),
+ impl(nullMissingHandling(DateTimeFormatterUtil::getFormattedDate),
+ STRING, DATE, STRING),
+ impl(nullMissingHandling(DateTimeFormatterUtil::getFormattedDate),
+ STRING, DATETIME, STRING),
+ impl(nullMissingHandling(DateTimeFormatterUtil::getFormattedDate),
+ STRING, TIMESTAMP, STRING)
+ );
+ }
+
  /**
  * ADDDATE function implementation for ExprValue.
  *

diff --git a/...va/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java b/...va/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java
@@ -72,6 +72,7 @@ public enum BuiltinFunctionName {
  TIME(FunctionName.of("time")),
  TIME_TO_SEC(FunctionName.of("time_to_sec")),
  TIMESTAMP(FunctionName.of("timestamp")),
+ DATE_FORMAT(FunctionName.of("date_format")),
  TO_DAYS(FunctionName.of("to_days")),
  WEEK(FunctionName.of("week")),
  YEAR(FunctionName.of("year")),

diff --git a/...a/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunctionTest.java b/...a/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunctionTest.java
@@ -47,6 +47,9 @@
 import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionTestBase;
 import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression;
 import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment;
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import lombok.AllArgsConstructor;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -71,6 +74,89 @@ public void setup() {
  when(missingRef.valueOf(env)).thenReturn(missingValue());
  }
 
+ final List<DateFormatTester> dateFormatTesters = ImmutableList.of(
+ new DateFormatTester("1998-01-31 13:14:15.012345",
+ ImmutableList.of("%H","%I","%k","%l","%i","%p","%r","%S","%T"," %M",
+ "%W","%D","%Y","%y","%a","%b","%j","%m","%d","%h","%s","%w","%f",
+ "%q","%"),
+ ImmutableList.of("13","01","13","1","14","PM","01:14:15 PM","15","13:14:15"," January",
+ "Saturday","31st","1998","98","Sat","Jan","031","01","31","01","15","6","12345",
+ "q","%")
+ ),
+ new DateFormatTester("1999-12-01",
+ ImmutableList.of("%D"),
+ ImmutableList.of("1st")
+ ),
+ new DateFormatTester("1999-12-02",
+ ImmutableList.of("%D"),
+ ImmutableList.of("2nd")
+ ),
+ new DateFormatTester("1999-12-03",
+ ImmutableList.of("%D"),
+ ImmutableList.of("3rd")
+ ),
+ new DateFormatTester("1999-12-04",
+ ImmutableList.of("%D"),
+ ImmutableList.of("4th")
+ ),
+ new DateFormatTester("1999-12-11",
+ ImmutableList.of("%D"),
+ ImmutableList.of("11th")
+ ),
+ new DateFormatTester("1999-12-12",
+ ImmutableList.of("%D"),
+ ImmutableList.of("12th")
+ ),
+ new DateFormatTester("1999-12-13",
+ ImmutableList.of("%D"),
+ ImmutableList.of("13th")
+ ),
+ new DateFormatTester("1999-12-31",
+ ImmutableList.of("%x","%v","%X","%V","%u","%U"),
+ ImmutableList.of("1999", "52", "1999", "52", "52", "52")
+ ),
+ new DateFormatTester("2000-01-01",
+ ImmutableList.of("%x","%v","%X","%V","%u","%U"),
+ ImmutableList.of("1999", "52", "1999", "52", "0", "0")
+ ),
+ new DateFormatTester("1998-12-31",
+ ImmutableList.of("%x","%v","%X","%V","%u","%U"),
+ ImmutableList.of("1998", "52", "1998", "52", "52", "52")
+ ),
+ new DateFormatTester("1999-01-01",
+ ImmutableList.of("%x","%v","%X","%V","%u","%U"),
+ ImmutableList.of("1998", "52", "1998", "52", "0", "0")
+ ),
+ new DateFormatTester("2020-01-04",
+ ImmutableList.of("%x","%X"),
+ ImmutableList.of("2020", "2019")
+ ),
+ new DateFormatTester("2008-12-31",
+ ImmutableList.of("%v","%V","%u","%U"),
+ ImmutableList.of("53","52","53","52")
+ )
+ );
+
+ @AllArgsConstructor
+ private class DateFormatTester {
+ private final String date;
+ private final List<String> formatterList;
+ private final List<String> formattedList;
+ private static final String DELIMITER = "|";
+
+ String getFormatter() {
+ return String.join(DELIMITER, formatterList);
+ }
+
+ String getFormatted() {
+ return String.join(DELIMITER, formattedList);
+ }
+
+ FunctionExpression getDateFormatExpression() {
+ return dsl.date_format(DSL.literal(date), DSL.literal(getFormatter()));
+ }
+ }
+
  @Test
  public void adddate() {
  FunctionExpression expr = dsl.adddate(dsl.date(DSL.literal("2020-08-26")), DSL.literal(7));
@@ -872,6 +958,48 @@ public void year() {
  assertEquals(integerValue(2020), eval(expression));
  }
 
+ @Test
+ public void date_format() {
+ dateFormatTesters.forEach(this::testDateFormat);
+ String timestamp = "1998-01-31 13:14:15.012345";
+ String timestampFormat = "%a %b %c %D %d %e %f %H %h %I %i %j %k %l %M "
+ + "%m %p %r %S %s %T %% %P";
+ String timestampFormatted = "Sat Jan 01 31st 31 31 12345 13 01 01 14 031 13 1 "
+ + "January 01 PM 01:14:15 PM 15 15 13:14:15 % P";
+
+ FunctionExpression expr = dsl.date_format(DSL.literal(timestamp), DSL.literal(timestampFormat));
+ assertEquals(STRING, expr.type());
+ assertEquals(timestampFormatted, eval(expr).stringValue());
+
+ when(nullRef.type()).thenReturn(DATE);
+ when(missingRef.type()).thenReturn(DATE);
+ assertEquals(nullValue(), eval(dsl.date_format(nullRef, DSL.literal(""))));
+ assertEquals(missingValue(), eval(dsl.date_format(missingRef, DSL.literal(""))));
+
+ when(nullRef.type()).thenReturn(DATETIME);
+ when(missingRef.type()).thenReturn(DATETIME);
+ assertEquals(nullValue(), eval(dsl.date_format(nullRef, DSL.literal(""))));
+ assertEquals(missingValue(), eval(dsl.date_format(missingRef, DSL.literal(""))));
+
+ when(nullRef.type()).thenReturn(TIMESTAMP);
+ when(missingRef.type()).thenReturn(TIMESTAMP);
+ assertEquals(nullValue(), eval(dsl.date_format(nullRef, DSL.literal(""))));
+ assertEquals(missingValue(), eval(dsl.date_format(missingRef, DSL.literal(""))));
+
+ when(nullRef.type()).thenReturn(STRING);
+ when(missingRef.type()).thenReturn(STRING);
+ assertEquals(nullValue(), eval(dsl.date_format(nullRef, DSL.literal(""))));
+ assertEquals(missingValue(), eval(dsl.date_format(missingRef, DSL.literal(""))));
+ assertEquals(nullValue(), eval(dsl.date_format(DSL.literal(""), nullRef)));
+ assertEquals(missingValue(), eval(dsl.date_format(DSL.literal(""), missingRef)));
+ }
+
+ void testDateFormat(DateFormatTester dft) {
+ FunctionExpression expr = dft.getDateFormatExpression();
+ assertEquals(STRING, expr.type());
+ assertEquals(dft.getFormatted(), eval(expr).stringValue());
+ }
+
  private ExprValue eval(Expression expression) {
  return expression.valueOf(env);
  }