DATE_FORMAT function (#764)

* Bug fix, support long type for aggregation (#522)

* Bug fix, support long type for aggregation

* change to datetime to JDBC format

* Opendistro Release 1.9.0 (#532)

* prepare odfe 1.9

* Fix all ES 7.8 compile and build errors

* Revert changes as Lombok is working now

* Update CustomExternalTestCluster.java

* Fix license headers check

* Use splitFieldsByMetadata to separate fields when calling SearchHit constructor

* More fixes for ODFE 1.9

* Remove todo statement

* Add ODFE 1.9.0 release notes

* Rename release notes to use 4 digit versions (#547)

* Revert changes ahead of develop branch in master (#551)

* Revert "Rename release notes to use 4 digit versions (#547)"

This reverts commit 33c6d3e.

* Revert "Opendistro Release 1.9.0 (#532)"

This reverts commit 254f2e0.

* Revert "Bug fix, support long type for aggregation (#522)"

This reverts commit fb2ed91.

* Merge all SQL repos and adjust workflows (#549) (#554)

* merge all sql repos

* fix test and build workflows

* fix workbench and odbc path

* fix workbench and odbc path

* restructure workbench dir and fix workflows

* fix workbench workflow

* fix workbench workflow

* fix workbench workflow

* fix workbench workflow

* fix workbench workflow

* revert workbench directory structure

* fix workbench workflow

* fix workbench workflow

* fix workbench workflow

* fix workbench workflow

* update workbench workflow for release

* Delete .github/ in sql-workbench directory

* Add cypress to sql-workbench

* Sync latest ODBC commits

* Sync latest workbench commits (will add cypress in separate PR)

* Add ignored ODBC libs

* add date and time support (#560)

* add date and time support

* update doc

* update doc

* Revert "add date and time support (#560)" (#567)

This reverts commit 4b33a2f.

* add error details for all server communication errors (#645)

- add null check to avoid crashing if details not initialized

* Revert "add error details for all server communication errors (#645)" (#653)

This reverts commit c11125d.

* Fix download link in package description (#729)

* add functions day, month, quarter, year

* fix build error

* fix doctest error

* fix doctest build error

* fix doctest

* add dayofmonth()

* add  dayofyear()

* add dayofweek()

* fix dayofweek logic & add unit test

* fix doctest for dayofweek()

* add dayname

* add monthname

* fix checkstyle build error

* fix build error

* fix doctest for monthname

* add hour()

* add minute()

* add second

* add microsecond

* fix datetime & timestamp issue for microsecond

* add time_to_sec

* add subdate & date_sub

* fix doctest error

* fix build error

* add KeywordsCanBeId for dayofweek

* add to_days

* add from_days()

* arrange by alphabetical order

* add manual IT

* add string input for date functions

* fix microsecond

* update doc

* add date_add

* add week

* address PR comments

* update tests & doc

* fix doc format

* update tests for adddate

* move string conversion to ExprStringValue

* add string type in doc

* edge case

* fix case 5 & 7

* add IT

* update doc

* fix table

* rename

* add string type

* nit: add newline

* fix type in comment

* nit

* add test cases for datetime function in ExpeStringValue

* removing implicit def for keyword in parser

* add dayofweek

* [1] Merged rupals week branch in

* add unit tests for null, missing values

* nit

* [1] Added integration tests.

* [1] Working on ppl integration tests.

* [1] Updated for integration tests

* [1] Fixed schema verification

* [1] Adding documentation.

* [1] Fixing documentation

* [1] Simplified a bunch of logic

* [1] Reducing changes that are from spacing

* [1] Removed extra merge line

* address PR comment

* [1] Updates

* [1] Removed some unwanted changes.

* [1] Minor whitespace adjustements

* [1] Updating based on code review

Co-authored-by: Peng Huo <penghuo@gmail.com>
Co-authored-by: Joshua <joshuali925@gmail.com>
Co-authored-by: Joshua Li <lijshu@amazon.com>
Co-authored-by: Jordan Wilson <37088125+jordanw-bq@users.noreply.github.com>
Co-authored-by: Chloe <chloezh1102@gmail.com>
Co-authored-by: chloe-zh <fizhang@amazon.com>
Co-authored-by: Sayali Gaikawad <61760125+gaiksaya@users.noreply.github.com>
Co-authored-by: Rupal Mahajan <>

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java

@@ -21,9 +21,7 @@
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.LocalTime;
-import java.time.format.DateTimeParseException;
 import java.util.Objects;
-import lombok.EqualsAndHashCode;
 import lombok.RequiredArgsConstructor;
 
 /**

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java

@@ -329,6 +329,10 @@ public FunctionExpression timestamp(Expression... expressions) {
     return function(BuiltinFunctionName.TIMESTAMP, expressions);
   }
 
+  public FunctionExpression date_format(Expression... expressions) {
+    return function(BuiltinFunctionName.DATE_FORMAT, expressions);
+  }
+
   public FunctionExpression to_days(Expression... expressions) {
     return function(BuiltinFunctionName.TO_DAYS, expressions);
   }

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/CalendarLookup.java

@@ -47,9 +47,9 @@ private static Calendar getCalendar(int mode, LocalDate date) {
 
   /**
    * Set first day of week, minimal days in first week and date in calendar.
-   * @param firstDayOfWeek    the given first day of the week.
+   * @param firstDayOfWeek the given first day of the week.
    * @param minimalDaysInWeek the given minimal days required in the first week of the year.
-   * @param date              the given date.
+   * @param date the given date.
    */
   private static Calendar getCalendar(int firstDayOfWeek, int minimalDaysInWeek, LocalDate date) {
     Calendar calendar = Calendar.getInstance();
@@ -74,4 +74,19 @@ static int getWeekNumber(int mode, LocalDate date) {
     }
     return weekNumber;
   }
-}
+
+  /**
+   * Returns year for date according to mode.
+   * @param mode Integer for mode. Valid mode values are 0 to 7.
+   * @param date LocalDate for date.
+   */
+  static int getYearNumber(int mode, LocalDate date) {
+    Calendar calendar = getCalendar(mode, date);
+    int weekNumber = getWeekNumber(mode, date);
+    int yearNumber = calendar.get(Calendar.YEAR);
+    if ((weekNumber > 51) && (calendar.get(Calendar.DAY_OF_MONTH) < 7)) {
+      yearNumber--;
+    }
+    return yearNumber;
+  }
+}

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFormatterUtil.java

@@ -0,0 +1,120 @@
+package com.amazon.opendistroforelasticsearch.sql.expression.datetime;
+
+import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue;
+import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
+import com.google.common.collect.ImmutableMap;
+
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * This class converts a SQL style DATE_FORMAT format specifier and converts it to a
+ * Java SimpleDateTime format.
+ */
+class DateTimeFormatterUtil {
+  private static final int SUFFIX_SPECIAL_START_TH = 11;
+  private static final int SUFFIX_SPECIAL_END_TH = 13;
+  private static final String SUFFIX_SPECIAL_TH = "th";
+  private static final Map<Integer, String> SUFFIX_CONVERTER =
+      ImmutableMap.<Integer, String>builder()
+      .put(1, "st").put(2, "nd").put(3, "rd").build();
+
+  // The following have special cases that need handling outside of the format options provided
+  // by the DateTimeFormatter class.
+  interface DateTimeFormatHandler {
+    String getFormat(LocalDateTime date);
+  }
+
+  private static final Map<String, DateTimeFormatHandler> HANDLERS =
+      ImmutableMap.<String, DateTimeFormatHandler>builder()
+      .put("%a", (date) -> "EEE") // %a => EEE - Abbreviated weekday name (Sun..Sat)
+      .put("%b", (date) -> "LLL") // %b => LLL - Abbreviated month name (Jan..Dec)
+      .put("%c", (date) -> "MM") // %c => MM - Month, numeric (0..12)
+      .put("%d", (date) -> "dd") // %d => dd - Day of the month, numeric (00..31)
+      .put("%e", (date) -> "d") // %e => d - Day of the month, numeric (0..31)
+      .put("%H", (date) -> "HH") // %H => HH - (00..23)
+      .put("%h", (date) -> "hh") // %h => hh - (01..12)
+      .put("%I", (date) -> "hh") // %I => hh - (01..12)
+      .put("%i", (date) -> "mm") // %i => mm - Minutes, numeric (00..59)
+      .put("%j", (date) -> "DDD") // %j => DDD - (001..366)
+      .put("%k", (date) -> "H") // %k => H - (0..23)
+      .put("%l", (date) -> "h") // %l => h - (1..12)
+      .put("%p", (date) -> "a") // %p => a - AM or PM
+      .put("%M", (date) -> "LLLL") // %M => LLLL - Month name (January..December)
+      .put("%m", (date) -> "MM") // %m => MM - Month, numeric (00..12)
+      .put("%r", (date) -> "hh:mm:ss a") // %r => hh:mm:ss a - hh:mm:ss followed by AM or PM
+      .put("%S", (date) -> "ss") // %S => ss - Seconds (00..59)
+      .put("%s", (date) -> "ss") // %s => ss - Seconds (00..59)
+      .put("%T", (date) -> "HH:mm:ss") // %T => HH:mm:ss
+      .put("%W", (date) -> "EEEE") // %W => EEEE - Weekday name (Sunday..Saturday)
+      .put("%Y", (date) -> "yyyy") // %Y => yyyy - Year, numeric, 4 digits
+      .put("%y", (date) -> "yy") // %y => yy - Year, numeric, 2 digits
+      // The following are not directly supported by DateTimeFormatter.
+      .put("%D", (date) -> // %w - Day of month with English suffix
+          String.format("'%d%s'", date.getDayOfMonth(), getSuffix(date.getDayOfMonth())))
+      .put("%f", (date) -> // %f - Microseconds
+          String.format("'%d'", (date.getNano() / 1000)))
+      .put("%w", (date) -> // %w - Day of week (0 indexed)
+          String.format("'%d'", date.getDayOfWeek().getValue()))
+      .put("%U", (date) -> // %U Week where Sunday is the first day - WEEK() mode 0
+          String.format("'%d'", CalendarLookup.getWeekNumber(0, date.toLocalDate())))
+      .put("%u", (date) -> // %u Week where Monday is the first day - WEEK() mode 1
+          String.format("'%d'", CalendarLookup.getWeekNumber(1, date.toLocalDate())))
+      .put("%V", (date) -> // %V Week where Sunday is the first day - WEEK() mode 2 used with %X
+          String.format("'%d'", CalendarLookup.getWeekNumber(2, date.toLocalDate())))
+      .put("%v", (date) -> // %v Week where Monday is the first day - WEEK() mode 3 used with %x
+          String.format("'%d'", CalendarLookup.getWeekNumber(3, date.toLocalDate())))
+      .put("%X", (date) -> // %X Year for week where Sunday is the first day, 4 digits used with %V
+          String.format("'%d'", CalendarLookup.getYearNumber(2, date.toLocalDate())))
+      .put("%x", (date) -> // %x Year for week where Monday is the first day, 4 digits used with %v
+          String.format("'%d'", CalendarLookup.getYearNumber(3, date.toLocalDate())))
+      .build();
+
+  private static final Pattern pattern = Pattern.compile("%.");
+  private static final String MOD_LITERAL = "%";
+
+  private DateTimeFormatterUtil() {
+  }
+
+  /**
+   * Format the date using the date format String.
+   * @param dateExpr the date ExprValue of Date/Datetime/Timestamp/String type.
+   * @param formatExpr the format ExprValue of String type.
+   * @return Date formatted using format and returned as a String.
+   */
+  static ExprValue getFormattedDate(ExprValue dateExpr, ExprValue formatExpr) {
+    final LocalDateTime date = dateExpr.datetimeValue();
+    final Matcher matcher = pattern.matcher(formatExpr.stringValue());
+    final StringBuffer format = new StringBuffer();
+    while (matcher.find()) {
+      matcher.appendReplacement(format,
+          HANDLERS.getOrDefault(matcher.group(), (d) ->
+              String.format("'%s'", matcher.group().replaceFirst(MOD_LITERAL, "")))
+              .getFormat(date));
+    }
+    matcher.appendTail(format);
+
+    // English Locale matches SQL requirements.
+    // 'AM'/'PM' instead of 'a.m.'/'p.m.'
+    // 'Sat' instead of 'Sat.' etc
+    return new ExprStringValue(date.format(
+        DateTimeFormatter.ofPattern(format.toString(), Locale.ENGLISH)));
+  }
+
+  /**
+   * Returns English suffix of incoming value.
+   * @param val Incoming value.
+   * @return English suffix as String (st, nd, rd, th)
+   */
+  private static String getSuffix(int val) {
+    // The numbers 11, 12, and 13 do not follow general suffix rules.
+    if ((SUFFIX_SPECIAL_START_TH <= val) && (val <= SUFFIX_SPECIAL_END_TH)) {
+      return SUFFIX_SPECIAL_TH;
+    }
+    return SUFFIX_CONVERTER.getOrDefault(val % 10, SUFFIX_SPECIAL_TH);
+  }
+}

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java

@@ -85,6 +85,7 @@ public void register(BuiltinFunctionRepository repository) {
     repository.register(time());
     repository.register(time_to_sec());
     repository.register(timestamp());
+    repository.register(date_format());
     repository.register(to_days());
     repository.register(week());
     repository.register(year());
@@ -400,6 +401,27 @@ private FunctionResolver year() {
     );
   }
 
+  /**
+   * Formats date according to format specifier. First argument is date, second is format.
+   * Detailed supported signatures:
+   * (STRING, STRING) -> STRING
+   * (DATE, STRING) -> STRING
+   * (DATETIME, STRING) -> STRING
+   * (TIMESTAMP, STRING) -> STRING
+   */
+  private FunctionResolver date_format() {
+    return define(BuiltinFunctionName.DATE_FORMAT.getName(),
+        impl(nullMissingHandling(DateTimeFormatterUtil::getFormattedDate),
+            STRING, STRING, STRING),
+        impl(nullMissingHandling(DateTimeFormatterUtil::getFormattedDate),
+            STRING, DATE, STRING),
+        impl(nullMissingHandling(DateTimeFormatterUtil::getFormattedDate),
+            STRING, DATETIME, STRING),
+        impl(nullMissingHandling(DateTimeFormatterUtil::getFormattedDate),
+            STRING, TIMESTAMP, STRING)
+    );
+  }
+
   /**
    * ADDDATE function implementation for ExprValue.
    *

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java

@@ -72,6 +72,7 @@ public enum BuiltinFunctionName {
   TIME(FunctionName.of("time")),
   TIME_TO_SEC(FunctionName.of("time_to_sec")),
   TIMESTAMP(FunctionName.of("timestamp")),
+  DATE_FORMAT(FunctionName.of("date_format")),
   TO_DAYS(FunctionName.of("to_days")),
   WEEK(FunctionName.of("week")),
   YEAR(FunctionName.of("year")),

core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunctionTest.java

@@ -47,6 +47,9 @@
 import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionTestBase;
 import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression;
 import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment;
+import com.google.common.collect.ImmutableList;
+import java.util.List;
+import lombok.AllArgsConstructor;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
@@ -71,6 +74,89 @@ public void setup() {
     when(missingRef.valueOf(env)).thenReturn(missingValue());
   }
 
+  final List<DateFormatTester> dateFormatTesters = ImmutableList.of(
+      new DateFormatTester("1998-01-31 13:14:15.012345",
+          ImmutableList.of("%H","%I","%k","%l","%i","%p","%r","%S","%T"," %M",
+              "%W","%D","%Y","%y","%a","%b","%j","%m","%d","%h","%s","%w","%f",
+              "%q","%"),
+          ImmutableList.of("13","01","13","1","14","PM","01:14:15 PM","15","13:14:15"," January",
+              "Saturday","31st","1998","98","Sat","Jan","031","01","31","01","15","6","12345",
+              "q","%")
+      ),
+      new DateFormatTester("1999-12-01",
+          ImmutableList.of("%D"),
+          ImmutableList.of("1st")
+      ),
+      new DateFormatTester("1999-12-02",
+          ImmutableList.of("%D"),
+          ImmutableList.of("2nd")
+      ),
+      new DateFormatTester("1999-12-03",
+          ImmutableList.of("%D"),
+          ImmutableList.of("3rd")
+      ),
+      new DateFormatTester("1999-12-04",
+          ImmutableList.of("%D"),
+          ImmutableList.of("4th")
+      ),
+      new DateFormatTester("1999-12-11",
+          ImmutableList.of("%D"),
+          ImmutableList.of("11th")
+      ),
+      new DateFormatTester("1999-12-12",
+          ImmutableList.of("%D"),
+          ImmutableList.of("12th")
+      ),
+      new DateFormatTester("1999-12-13",
+          ImmutableList.of("%D"),
+          ImmutableList.of("13th")
+      ),
+      new DateFormatTester("1999-12-31",
+          ImmutableList.of("%x","%v","%X","%V","%u","%U"),
+          ImmutableList.of("1999", "52", "1999", "52", "52", "52")
+      ),
+      new DateFormatTester("2000-01-01",
+          ImmutableList.of("%x","%v","%X","%V","%u","%U"),
+          ImmutableList.of("1999", "52", "1999", "52", "0", "0")
+      ),
+      new DateFormatTester("1998-12-31",
+          ImmutableList.of("%x","%v","%X","%V","%u","%U"),
+          ImmutableList.of("1998", "52", "1998", "52", "52", "52")
+      ),
+      new DateFormatTester("1999-01-01",
+          ImmutableList.of("%x","%v","%X","%V","%u","%U"),
+          ImmutableList.of("1998", "52", "1998", "52", "0", "0")
+      ),
+      new DateFormatTester("2020-01-04",
+          ImmutableList.of("%x","%X"),
+          ImmutableList.of("2020", "2019")
+      ),
+      new DateFormatTester("2008-12-31",
+          ImmutableList.of("%v","%V","%u","%U"),
+          ImmutableList.of("53","52","53","52")
+      )
+  );
+
+  @AllArgsConstructor
+  private class DateFormatTester {
+    private final String date;
+    private final List<String> formatterList;
+    private final List<String> formattedList;
+    private static final String DELIMITER = "|";
+
+    String getFormatter() {
+      return String.join(DELIMITER, formatterList);
+    }
+
+    String getFormatted() {
+      return String.join(DELIMITER, formattedList);
+    }
+
+    FunctionExpression getDateFormatExpression() {
+      return dsl.date_format(DSL.literal(date), DSL.literal(getFormatter()));
+    }
+  }
+
   @Test
   public void adddate() {
     FunctionExpression expr = dsl.adddate(dsl.date(DSL.literal("2020-08-26")), DSL.literal(7));
@@ -872,6 +958,48 @@ public void year() {
     assertEquals(integerValue(2020), eval(expression));
   }
 
+  @Test
+  public void date_format() {
+    dateFormatTesters.forEach(this::testDateFormat);
+    String timestamp = "1998-01-31 13:14:15.012345";
+    String timestampFormat = "%a %b %c %D %d %e %f %H %h %I %i %j %k %l %M "
+        + "%m %p %r %S %s %T %% %P";
+    String timestampFormatted = "Sat Jan 01 31st 31 31 12345 13 01 01 14 031 13 1 "
+        + "January 01 PM 01:14:15 PM 15 15 13:14:15 % P";
+
+    FunctionExpression expr = dsl.date_format(DSL.literal(timestamp), DSL.literal(timestampFormat));
+    assertEquals(STRING, expr.type());
+    assertEquals(timestampFormatted, eval(expr).stringValue());
+
+    when(nullRef.type()).thenReturn(DATE);
+    when(missingRef.type()).thenReturn(DATE);
+    assertEquals(nullValue(), eval(dsl.date_format(nullRef, DSL.literal(""))));
+    assertEquals(missingValue(), eval(dsl.date_format(missingRef, DSL.literal(""))));
+
+    when(nullRef.type()).thenReturn(DATETIME);
+    when(missingRef.type()).thenReturn(DATETIME);
+    assertEquals(nullValue(), eval(dsl.date_format(nullRef, DSL.literal(""))));
+    assertEquals(missingValue(), eval(dsl.date_format(missingRef, DSL.literal(""))));
+
+    when(nullRef.type()).thenReturn(TIMESTAMP);
+    when(missingRef.type()).thenReturn(TIMESTAMP);
+    assertEquals(nullValue(), eval(dsl.date_format(nullRef, DSL.literal(""))));
+    assertEquals(missingValue(), eval(dsl.date_format(missingRef, DSL.literal(""))));
+
+    when(nullRef.type()).thenReturn(STRING);
+    when(missingRef.type()).thenReturn(STRING);
+    assertEquals(nullValue(), eval(dsl.date_format(nullRef, DSL.literal(""))));
+    assertEquals(missingValue(), eval(dsl.date_format(missingRef, DSL.literal(""))));
+    assertEquals(nullValue(), eval(dsl.date_format(DSL.literal(""), nullRef)));
+    assertEquals(missingValue(), eval(dsl.date_format(DSL.literal(""), missingRef)));
+  }
+
+  void testDateFormat(DateFormatTester dft) {
+    FunctionExpression expr = dft.getDateFormatExpression();
+    assertEquals(STRING, expr.type());
+    assertEquals(dft.getFormatted(), eval(expr).stringValue());
+  }
+
   private ExprValue eval(Expression expression) {
     return expression.valueOf(env);
   }