diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java
index 200c300f3b..3f8c2c971a 100644
--- a/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java
+++ b/integ-test/src/test/java/org/opensearch/sql/legacy/PrettyFormatResponseIT.java
@@ -27,6 +27,7 @@
import org.json.JSONObject;
import org.junit.Ignore;
import org.junit.Test;
+import org.junit.jupiter.api.Disabled;
import org.opensearch.client.Request;
/**
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java
index 35ae5d3675..901f198b14 100644
--- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java
+++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java
@@ -634,6 +634,10 @@ public enum Index {
"calcs",
getMappingFile("calcs_index_mappings.json"),
"src/test/resources/calcs.json"),
+ DATE_FORMATS(TestsConstants.TEST_INDEX_DATE_FORMATS,
+ "date_formats",
+ getMappingFile("date_formats_index_mapping.json"),
+ "src/test/resources/date_formats.json"),
WILDCARD(TestsConstants.TEST_INDEX_WILDCARD,
"wildcard",
getMappingFile("wildcard_index_mappings.json"),
diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java
index c3af98b794..338be25a0c 100644
--- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java
+++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java
@@ -56,6 +56,7 @@ public class TestsConstants {
public final static String TEST_INDEX_BEER = TEST_INDEX + "_beer";
public final static String TEST_INDEX_NULL_MISSING = TEST_INDEX + "_null_missing";
public final static String TEST_INDEX_CALCS = TEST_INDEX + "_calcs";
+ public final static String TEST_INDEX_DATE_FORMATS = TEST_INDEX + "_date_formats";
public final static String TEST_INDEX_WILDCARD = TEST_INDEX + "_wildcard";
public final static String TEST_INDEX_MULTI_NESTED_TYPE = TEST_INDEX + "_multi_nested";
public final static String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls";
diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java
index b5677b04a7..c70ca1dc2c 100644
--- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java
+++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java
@@ -8,6 +8,7 @@
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK;
import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS;
import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT;
import static org.opensearch.sql.util.MatcherUtils.rows;
import static org.opensearch.sql.util.MatcherUtils.schema;
@@ -40,7 +41,7 @@ public void init() throws Exception {
loadIndex(Index.BANK);
loadIndex(Index.CALCS);
loadIndex(Index.PEOPLE2);
- loadIndex(Index.CALCS);
+ loadIndex(Index.DATE_FORMATS);
}
// Integration test framework sets for OpenSearch instance a random timezone.
@@ -1278,6 +1279,99 @@ public void testTimeFormat() throws IOException {
verifyTimeFormat(timestamp, "timestamp", timestampFormat, timestampFormatted);
}
+ @Test
+ public void testReadingDateFormats() throws IOException {
+ String query = String.format("SELECT * FROM %s LIMIT 1", TEST_INDEX_DATE_FORMATS);
+ JSONObject result = executeQuery(query);
+ verifyDataRows(result,
+ rows("1984-04-12 00:00:00", // date
+ "1984-04-12 09:07:42", // date_optional_time
+ "1984-04-12 09:07:42", // basic_week_date_time
+ "1970-01-01 09:07:42", // basic_ordinal_date_time TODO: fix bug
+ "1970-01-01 09:07:42", // strict_time_no_millis
+ "1984-04-12 09:07:42", // date_time
+ "1984-04-12 09:07:42", // strict_basic_week_date_time
+ "1984-04-12 00:00:00", // strict_basic_week_date
+ "1984-04-12 00:00:00", // strict_ordinal_date
+ "1970-01-01 09:07:42", // basic_time_no_millis
+ "1984-04-12 09:07:42.000123456", // strict_date_optional_time_nanos
+ "1984-04-12 00:00:00", // year_month_day
+ "1970-01-01 09:00:00", // strict_hour
+ "1984-04-12 00:00:00", // basic_week_date
+ "1984-04-12 00:00:00", // strict_week_date
+ "1970-01-01 09:07:42", // basic_t_time_no_millis
+ "1970-01-01 09:07:00", // strict_date_hour_minute
+ "1984-04-12 09:07:42", // strict_week_date_time_no_millis
+ "1984-04-12 09:07:42", // basic_date_time_no_millis
+ "1984-04-12 09:07:42", // strict_week_date_time
+ "1984-04-12 09:07:42.000123456", // epoch_second
+ "1984-04-12 09:07:42", // basic_ordinal_date_time_no_millis
+ "1984-04-12 09:07:42.000123456", // ordinal_date_time
+ "1984-04-12 09:07:42", // strict_date_optional_time
+ "1970-01-01 09:07:42", // hour_minute_second
+ "1970-01-01 09:07:42", // basic_time
+ "1970-01-01 09:07:42", // HH:mm:ss
+ "1970-01-01 09:07:42", // time_no_millis
+ "1970-01-01 09:07:42", // hour_minute_second_fraction
+ "1970-01-01 09:07:42", // strict_date_hour_minute_second_fraction
+ "1970-01-01 09:07:42", // strict_hour_minute_second_fraction
+ "1970-01-01 09:07:42", // strict_time
+ "1970-01-01 09:07:42", // strict_t_time_no_millis
+ "1970-01-01 09:07:00", // date_hour_minute
+ "1984-04-12 09:07:42.000123456", // strict_ordinal_date_time
+ "1970-01-01 09:07:42", // strict_t_time
+ "1984-04-12 09:07:42", // week_date_time
+ "1970-01-01 09:07:42", // date_hour_minute_second
+ "1984-04-12 09:07:42", // date_time_no_millis
+ "1970-01-01 09:00:00", // hour
+ "1970-01-01 09:07:42", // strict_date_hour_minute_second_millis TODO: fix bug
+ "1984-04-12 00:00:00", // yyyy-MM-dd_OR_epoch_millis
+ "1984-04-12 00:00:00", // strict_date
+ "1984-04-12 09:07:42", // week_date_time_no_millis
+ "1970-01-01 09:07:42", // t_time
+ "1970-01-01 09:07:42", // hour_minute_second_millis
+ "1984-04-12 00:00:00", // basic_ordinal_date
+ "1970-01-01 09:07:42", // date_hour_minute_second_millis TODO: fix bug
+ "1984-04-12 09:07:42", // strict_ordinal_date_time_no_millis
+ "1970-01-01 09:07:42", // hour_minute_second_OR_t_time
+ "1984-04-12 00:00:00", // strict_year_month_day
+ "1984-04-12 09:07:42", // ordinal_date_time_no_millis
+ "1984-04-12 00:00:00", // week_date
+ "1984-04-12 09:07:42.000123456", // epoch_millis
+ "1970-01-01 09:07:00", // strict_hour_minute
+ "1984-04-12 00:00:00", // basic_date
+ "1970-01-01 09:07:42", // t_time_no_millis
+ "1984-04-12 00:00:00", // weekyear_week_day
+ "1984-04-12 00:00:00", // ordinal_date
+ "1970-01-01 09:07:42", // strict_hour_minute_second_millis
+ "1984-04-12 09:07:42", // basic_date_time
+ "1970-01-01 09:00:00", // date_hour TODO: fix bug
+ "1970-01-01 09:00:00", // strict_date_hour TODO: fix bug
+ "1984-04-12 00:00:00", // strict_weekyear_week_day
+ "1970-01-01 09:07:42", // date_hour_minute_second_fraction TODO: fix bug
+ "1984-04-12 09:07:42", // strict_date_time_no_millis
+ "1970-01-01 09:07:42", // basic_t_time
+ "1970-01-01 09:07:00", // hour_minute
+ "1984-04-12 09:07:42", // basic_week_date_time_no_millis
+ "1984-04-12 00:00:00", // yyyy-MM-dd
+ "1984-04-12 09:07:42.000123456", // strict_date_time
+ "1970-01-01 09:07:42", // time
+ "1970-01-01 09:07:42", // strict_date_hour_minute_second TODO: fix bug
+ "1970-01-01 09:07:42", // strict_hour_minute_second
+ "1984-04-12 09:07:42" // strict_basic_week_date_time_no_millis
+ ));
+ }
+
+ @Test
+ public void testDateFormatsWithOr() throws IOException {
+ String query = String.format("SELECT yyyy-MM-dd_OR_epoch_millis, hour_minute_second_OR_t_time"
+ + " FROM %s", TEST_INDEX_DATE_FORMATS);
+ JSONObject result = executeQuery(query);
+ verifyDataRows(result,
+ rows("1984-04-12 00:00:00", "1970-01-01 09:07:42"),
+ rows("1984-04-12 09:07:42.000123456", "1970-01-01 11:27:25"));
+ }
+
protected JSONObject executeQuery(String query) throws IOException {
Request request = new Request("POST", QUERY_API_ENDPOINT);
request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query));
diff --git a/integ-test/src/test/resources/date_formats.json b/integ-test/src/test/resources/date_formats.json
new file mode 100644
index 0000000000..a3a4affc0b
--- /dev/null
+++ b/integ-test/src/test/resources/date_formats.json
@@ -0,0 +1,4 @@
+{"index": {}}
+{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "HH:mm:ss": "09:07:42", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42"}
+{"index": {}}
+{"epoch_millis": "450608862000", "epoch_second": "450608862", "date_optional_time": "2023-05-03T11:27:25.000Z", "strict_date_optional_time": "2023-05-03T11:27:25.000Z", "strict_date_optional_time_nanos": "2023-05-03T11:27:25.000123456Z", "basic_date": "20230503", "basic_date_time": "20230503T112725.000Z", "basic_date_time_no_millis": "20230503T112725Z", "basic_ordinal_date": "2023123", "basic_ordinal_date_time": "2023123T112725.000Z", "basic_ordinal_date_time_no_millis": "2023123T112725Z", "basic_time": "112725.000Z", "basic_time_no_millis": "112725Z", "basic_t_time": "T112725.000Z", "basic_t_time_no_millis": "T112725Z", "basic_week_date": "2023W183", "strict_basic_week_date": "2023W183", "basic_week_date_time": "2023W183T112725.000Z", "strict_basic_week_date_time": "2023W183T112725.000Z", "basic_week_date_time_no_millis": "2023W183T112725Z", "strict_basic_week_date_time_no_millis": "2023W183T112725Z", "date": "2023-05-03", "strict_date": "2023-05-03", "date_hour": "2023-05-03T11", "strict_date_hour": "2023-05-03T11", "date_hour_minute": "2023-05-03T11:27", "strict_date_hour_minute": "2023-05-03T11:27", "date_hour_minute_second": "2023-05-03T11:27:25", "strict_date_hour_minute_second": "2023-05-03T11:27:25", "date_hour_minute_second_fraction": "2023-05-03T11:27:25.000", "strict_date_hour_minute_second_fraction": "2023-05-03T11:27:25.000", "date_hour_minute_second_millis": "2023-05-03T11:27:25.000", "strict_date_hour_minute_second_millis": "2023-05-03T11:27:25.000", "date_time": "2023-05-03T11:27:25.000Z", "strict_date_time": "2023-05-03T11:27:25.000123456Z", "date_time_no_millis": "2023-05-03T11:27:25Z", "strict_date_time_no_millis": "2023-05-03T11:27:25Z", "hour": "11", "strict_hour": "11", "hour_minute": "11:27", "strict_hour_minute": "11:27", "hour_minute_second": "11:27:25", "strict_hour_minute_second": "11:27:25", "hour_minute_second_fraction": "11:27:25.000", "strict_hour_minute_second_fraction": "11:27:25.000", "hour_minute_second_millis": "11:27:25.000", "strict_hour_minute_second_millis": "11:27:25.000", "ordinal_date": "2023-123", "strict_ordinal_date": "2023-123", "ordinal_date_time": "2023-123T11:27:25.000123456Z", "strict_ordinal_date_time": "2023-123T11:27:25.000123456Z", "ordinal_date_time_no_millis": "2023-123T11:27:25Z", "strict_ordinal_date_time_no_millis": "2023-123T11:27:25Z", "time": "11:27:25.000Z", "strict_time": "11:27:25.000Z", "time_no_millis": "11:27:25Z", "strict_time_no_millis": "11:27:25Z", "t_time": "T11:27:25.000Z", "strict_t_time": "T11:27:25.000Z", "t_time_no_millis": "T11:27:25Z", "strict_t_time_no_millis": "T11:27:25Z", "week_date": "2023-W18-3", "strict_week_date": "2023-W18-3", "week_date_time": "2023-W18-3T11:27:25.000Z", "strict_week_date_time": "2023-W18-3T11:27:25.000Z", "week_date_time_no_millis": "2023-W18-3T11:27:25Z", "strict_week_date_time_no_millis": "2023-W18-3T11:27:25Z", "weekyear_week_day": "2023-W18-3", "strict_weekyear_week_day": "2023-W18-3", "year_month_day": "2023-05-03", "strict_year_month_day": "2023-05-03", "yyyy-MM-dd": "2023-05-03", "HH:mm:ss": "11:27:25", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T11:27:25.000Z"}
diff --git a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json
new file mode 100644
index 0000000000..2c46555653
--- /dev/null
+++ b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json
@@ -0,0 +1,306 @@
+{
+ "mappings" : {
+ "properties" : {
+ "epoch_millis" : {
+ "type" : "date",
+ "format" : "epoch_millis"
+ },
+ "epoch_second" : {
+ "type" : "date",
+ "format" : "epoch_second"
+ },
+ "date_optional_time" : {
+ "type" : "date",
+ "format" : "date_optional_time"
+ },
+ "strict_date_optional_time" : {
+ "type" : "date",
+ "format" : "strict_date_optional_time"
+ },
+ "strict_date_optional_time_nanos" : {
+ "type" : "date",
+ "format" : "strict_date_optional_time_nanos"
+ },
+ "basic_date" : {
+ "type" : "date",
+ "format" : "basic_date"
+ },
+ "basic_date_time" : {
+ "type" : "date",
+ "format" : "basic_date_time"
+ },
+ "basic_date_time_no_millis" : {
+ "type" : "date",
+ "format" : "basic_date_time_no_millis"
+ },
+ "basic_ordinal_date" : {
+ "type" : "date",
+ "format" : "basic_ordinal_date"
+ },
+ "basic_ordinal_date_time" : {
+ "type" : "date",
+ "format" : "basic_ordinal_date_time"
+ },
+ "basic_ordinal_date_time_no_millis" : {
+ "type" : "date",
+ "format" : "basic_ordinal_date_time_no_millis"
+ },
+ "basic_time" : {
+ "type" : "date",
+ "format" : "basic_time"
+ },
+ "basic_time_no_millis" : {
+ "type" : "date",
+ "format" : "basic_time_no_millis"
+ },
+ "basic_t_time" : {
+ "type" : "date",
+ "format" : "basic_t_time"
+ },
+ "basic_t_time_no_millis" : {
+ "type" : "date",
+ "format" : "basic_t_time_no_millis"
+ },
+ "basic_week_date" : {
+ "type" : "date",
+ "format" : "basic_week_date"
+ },
+ "strict_basic_week_date" : {
+ "type" : "date",
+ "format" : "strict_basic_week_date"
+ },
+ "basic_week_date_time" : {
+ "type" : "date",
+ "format" : "basic_week_date_time"
+ },
+ "strict_basic_week_date_time" : {
+ "type" : "date",
+ "format" : "strict_basic_week_date_time"
+ },
+ "basic_week_date_time_no_millis" : {
+ "type" : "date",
+ "format" : "basic_week_date_time_no_millis"
+ },
+ "strict_basic_week_date_time_no_millis" : {
+ "type" : "date",
+ "format" : "strict_basic_week_date_time_no_millis"
+ },
+ "date" : {
+ "type" : "date",
+ "format" : "date"
+ },
+ "strict_date" : {
+ "type" : "date",
+ "format" : "strict_date"
+ },
+ "date_hour" : {
+ "type" : "date",
+ "format" : "date_hour"
+ },
+ "strict_date_hour" : {
+ "type" : "date",
+ "format" : "strict_date_hour"
+ },
+ "date_hour_minute" : {
+ "type" : "date",
+ "format" : "date_hour_minute"
+ },
+ "strict_date_hour_minute" : {
+ "type" : "date",
+ "format" : "strict_date_hour_minute"
+ },
+ "date_hour_minute_second" : {
+ "type" : "date",
+ "format" : "date_hour_minute_second"
+ },
+ "strict_date_hour_minute_second" : {
+ "type" : "date",
+ "format" : "strict_date_hour_minute_second"
+ },
+ "date_hour_minute_second_fraction" : {
+ "type" : "date",
+ "format" : "date_hour_minute_second_fraction"
+ },
+ "strict_date_hour_minute_second_fraction" : {
+ "type" : "date",
+ "format" : "strict_date_hour_minute_second_fraction"
+ },
+ "date_hour_minute_second_millis" : {
+ "type" : "date",
+ "format" : "date_hour_minute_second_millis"
+ },
+ "strict_date_hour_minute_second_millis" : {
+ "type" : "date",
+ "format" : "strict_date_hour_minute_second_millis"
+ },
+ "date_time" : {
+ "type" : "date",
+ "format" : "date_time"
+ },
+ "strict_date_time" : {
+ "type" : "date",
+ "format" : "strict_date_time"
+ },
+ "date_time_no_millis" : {
+ "type" : "date",
+ "format" : "date_time_no_millis"
+ },
+ "strict_date_time_no_millis" : {
+ "type" : "date",
+ "format" : "strict_date_time_no_millis"
+ },
+ "hour" : {
+ "type" : "date",
+ "format" : "hour"
+ },
+ "strict_hour" : {
+ "type" : "date",
+ "format" : "strict_hour"
+ },
+ "hour_minute" : {
+ "type" : "date",
+ "format" : "hour_minute"
+ },
+ "strict_hour_minute" : {
+ "type" : "date",
+ "format" : "strict_hour_minute"
+ },
+ "hour_minute_second" : {
+ "type" : "date",
+ "format" : "hour_minute_second"
+ },
+ "strict_hour_minute_second" : {
+ "type" : "date",
+ "format" : "strict_hour_minute_second"
+ },
+ "hour_minute_second_fraction" : {
+ "type" : "date",
+ "format" : "hour_minute_second_fraction"
+ },
+ "strict_hour_minute_second_fraction" : {
+ "type" : "date",
+ "format" : "strict_hour_minute_second_fraction"
+ },
+ "hour_minute_second_millis" : {
+ "type" : "date",
+ "format" : "hour_minute_second_millis"
+ },
+ "strict_hour_minute_second_millis" : {
+ "type" : "date",
+ "format" : "strict_hour_minute_second_millis"
+ },
+ "ordinal_date" : {
+ "type" : "date",
+ "format" : "ordinal_date"
+ },
+ "strict_ordinal_date" : {
+ "type" : "date",
+ "format" : "strict_ordinal_date"
+ },
+ "ordinal_date_time" : {
+ "type" : "date",
+ "format" : "ordinal_date_time"
+ },
+ "strict_ordinal_date_time" : {
+ "type" : "date",
+ "format" : "strict_ordinal_date_time"
+ },
+ "ordinal_date_time_no_millis" : {
+ "type" : "date",
+ "format" : "ordinal_date_time_no_millis"
+ },
+ "strict_ordinal_date_time_no_millis" : {
+ "type" : "date",
+ "format" : "strict_ordinal_date_time_no_millis"
+ },
+ "time" : {
+ "type" : "date",
+ "format" : "time"
+ },
+ "strict_time" : {
+ "type" : "date",
+ "format" : "strict_time"
+ },
+ "time_no_millis" : {
+ "type" : "date",
+ "format" : "time_no_millis"
+ },
+ "strict_time_no_millis" : {
+ "type" : "date",
+ "format" : "strict_time_no_millis"
+ },
+ "t_time" : {
+ "type" : "date",
+ "format" : "t_time"
+ },
+ "strict_t_time" : {
+ "type" : "date",
+ "format" : "strict_t_time"
+ },
+ "t_time_no_millis" : {
+ "type" : "date",
+ "format" : "t_time_no_millis"
+ },
+ "strict_t_time_no_millis" : {
+ "type" : "date",
+ "format" : "strict_t_time_no_millis"
+ },
+ "week_date" : {
+ "type" : "date",
+ "format" : "week_date"
+ },
+ "strict_week_date" : {
+ "type" : "date",
+ "format" : "strict_week_date"
+ },
+ "week_date_time" : {
+ "type" : "date",
+ "format" : "week_date_time"
+ },
+ "strict_week_date_time" : {
+ "type" : "date",
+ "format" : "strict_week_date_time"
+ },
+ "week_date_time_no_millis" : {
+ "type" : "date",
+ "format" : "week_date_time_no_millis"
+ },
+ "strict_week_date_time_no_millis" : {
+ "type" : "date",
+ "format" : "strict_week_date_time_no_millis"
+ },
+ "weekyear_week_day" : {
+ "type" : "date",
+ "format" : "weekyear_week_day"
+ },
+ "strict_weekyear_week_day" : {
+ "type" : "date",
+ "format" : "strict_weekyear_week_day"
+ },
+ "year_month_day" : {
+ "type" : "date",
+ "format" : "year_month_day"
+ },
+ "strict_year_month_day" : {
+ "type" : "date",
+ "format" : "strict_year_month_day"
+ },
+ "yyyy-MM-dd" : {
+ "type" : "date",
+ "format": "yyyy-MM-dd"
+ },
+ "HH:mm:ss" : {
+ "type" : "date",
+ "format": "HH:mm:ss"
+ },
+ "yyyy-MM-dd_OR_epoch_millis" : {
+ "type" : "date",
+ "format": "yyyy-MM-dd||epoch_millis"
+ },
+ "hour_minute_second_OR_t_time" : {
+ "type" : "date",
+ "format": "hour_minute_second||t_time"
+ }
+ }
+ }
+}
diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java b/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java
index ebd9648d5e..1adb0e745e 100644
--- a/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java
+++ b/legacy/src/main/java/org/opensearch/sql/legacy/executor/GetIndexRequestRestListener.java
@@ -6,16 +6,13 @@
package org.opensearch.sql.legacy.executor;
-import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import java.io.IOException;
import java.util.List;
-import java.util.Map;
import org.opensearch.action.admin.indices.get.GetIndexRequest;
import org.opensearch.action.admin.indices.get.GetIndexResponse;
import org.opensearch.cluster.metadata.AliasMetadata;
import org.opensearch.cluster.metadata.MappingMetadata;
-import org.opensearch.common.collect.ImmutableOpenMap;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
@@ -25,7 +22,6 @@
import org.opensearch.rest.RestStatus;
import org.opensearch.rest.action.RestBuilderListener;
import org.opensearch.sql.legacy.antlr.semantic.SemanticAnalysisException;
-import org.opensearch.sql.legacy.domain.Field;
/**
* Created by Eliran on 6/10/2015.
diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java b/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java
index 44baa8050b..595b6987a7 100644
--- a/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java
+++ b/legacy/src/test/java/org/opensearch/sql/legacy/util/CheckScriptContents.java
@@ -37,10 +37,8 @@
import org.opensearch.cluster.ClusterState;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
-import org.opensearch.cluster.metadata.MappingMetadata;
import org.opensearch.cluster.metadata.Metadata;
import org.opensearch.cluster.service.ClusterService;
-import org.opensearch.common.collect.ImmutableOpenMap;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.xcontent.DeprecationHandler;
import org.opensearch.core.xcontent.NamedXContentRegistry;
diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java
index 1c1498feb5..b26680b3ba 100644
--- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java
+++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java
@@ -112,11 +112,11 @@ public Map<String, Integer> getIndexMaxResultWindows(String... indexExpression)
GetSettingsResponse settingsResponse =
client.admin().indices().prepareGetSettings(indexExpression).setLocal(true).get();
ImmutableMap.Builder<String, Integer> result = ImmutableMap.builder();
- for (ObjectObjectCursor<String, Settings> indexToSetting :
- settingsResponse.getIndexToSettings()) {
- Settings settings = indexToSetting.value;
+ for (Map.Entry<String, Settings> indexToSetting :
+ settingsResponse.getIndexToSettings().entrySet()) {
+ Settings settings = indexToSetting.getValue();
result.put(
- indexToSetting.key,
+ indexToSetting.getKey(),
settings.getAsInt(
IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(),
IndexSettings.MAX_RESULT_WINDOW_SETTING.getDefault(settings)));
diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java
index d9f9dbbe5d..66cc067541 100644
--- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java
+++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java
@@ -30,7 +30,6 @@
import org.opensearch.client.indices.GetMappingsResponse;
import org.opensearch.client.node.NodeClient;
import org.opensearch.cluster.metadata.AliasMetadata;
-import org.opensearch.common.collect.ImmutableOpenMap;
import org.opensearch.common.settings.Settings;
import org.opensearch.sql.opensearch.mapping.IndexMapping;
import org.opensearch.sql.opensearch.request.OpenSearchRequest;
@@ -85,21 +84,21 @@ public Map<String, Integer> getIndexMaxResultWindows(String... indexExpression)
.indices(indexExpression).includeDefaults(true);
try {
GetSettingsResponse response = client.indices().getSettings(request, RequestOptions.DEFAULT);
- ImmutableOpenMap<String, Settings> settings = response.getIndexToSettings();
- ImmutableOpenMap<String, Settings> defaultSettings = response.getIndexToDefaultSettings();
+ Map<String, Settings> settings = response.getIndexToSettings();
+ Map<String, Settings> defaultSettings = response.getIndexToDefaultSettings();
Map<String, Integer> result = new HashMap<>();
- defaultSettings.forEach(entry -> {
- Integer maxResultWindow = entry.value.getAsInt("index.max_result_window", null);
+ defaultSettings.forEach((key, value) -> {
+ Integer maxResultWindow = value.getAsInt("index.max_result_window", null);
if (maxResultWindow != null) {
- result.put(entry.key, maxResultWindow);
+ result.put(key, maxResultWindow);
}
});
- settings.forEach(entry -> {
- Integer maxResultWindow = entry.value.getAsInt("index.max_result_window", null);
+ settings.forEach((key, value) -> {
+ Integer maxResultWindow = value.getAsInt("index.max_result_window", null);
if (maxResultWindow != null) {
- result.put(entry.key, maxResultWindow);
+ result.put(key, maxResultWindow);
}
});
diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java
index 2fda12a567..242f351f81 100644
--- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java
+++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java
@@ -65,6 +65,7 @@ public String toString() {
}
@EqualsAndHashCode.Exclude
+ @Getter
protected MappingType mappingType;
// resolved ExprCoreType
@@ -97,56 +98,81 @@ public ExprType getExprType() {
instances.put(t.toString(), OpenSearchDataType.of(t)));
}
+ /**
+ * Parses index mapping and maps it to a Data type in the SQL plugin.
+ * @param indexMapping An input with keys and objects that need to be mapped to a data type.
+ * @return The mapping.
+ */
+ public static Map<String, OpenSearchDataType> parseMapping(Map<String, Object> indexMapping) {
+ Map<String, OpenSearchDataType> result = new LinkedHashMap<>();
+ if (indexMapping != null) {
+ indexMapping.forEach((k, v) -> {
+ var innerMap = (Map<String, Object>)v;
+ // by default, the type is treated as an Object if "type" is not provided
+ var type = ((String) innerMap
+ .getOrDefault(
+ "type",
+ "object"))
+ .replace("_", "");
+ if (!EnumUtils.isValidEnumIgnoreCase(OpenSearchDataType.MappingType.class, type)) {
+ // unknown type, e.g. `alias`
+ // TODO resolve alias reference
+ return;
+ }
+ // create OpenSearchDataType
+ result.put(k, OpenSearchDataType.of(
+ EnumUtils.getEnumIgnoreCase(OpenSearchDataType.MappingType.class, type),
+ innerMap)
+ );
+ });
+ }
+ return result;
+ }
+
/**
* A constructor function which builds proper `OpenSearchDataType` for given mapping `Type`.
* @param mappingType A mapping type.
* @return An instance or inheritor of `OpenSearchDataType`.
*/
- public static OpenSearchDataType of(MappingType mappingType) {
- var res = instances.getOrDefault(mappingType.toString(), null);
- if (res != null) {
- return res;
- }
- ExprCoreType exprCoreType = mappingType.getExprCoreType();
- if (exprCoreType == ExprCoreType.UNKNOWN) {
- switch (mappingType) {
+ public static OpenSearchDataType of(MappingType mappingType, Map<String, Object> innerMap) {
+ OpenSearchDataType res = instances.getOrDefault(mappingType.toString(),
+ new OpenSearchDataType(mappingType)
+ );
+ switch (mappingType) {
+ case Object:
+ case Nested:
+ if (innerMap.isEmpty()) {
+ return res;
+ }
+ Map<String, OpenSearchDataType> properties =
+ parseMapping((Map<String, Object>) innerMap.getOrDefault("properties", Map.of()));
+ OpenSearchDataType objectDataType = res.cloneEmpty();
+ objectDataType.properties = properties;
+ return objectDataType;
+ case Text:
// TODO update these 2 below #1038 https://github.com/opensearch-project/sql/issues/1038
- case Text: return OpenSearchTextType.of();
- case GeoPoint: return OpenSearchGeoPointType.of();
- case Binary: return OpenSearchBinaryType.of();
- case Ip: return OpenSearchIpType.of();
- default:
- throw new IllegalArgumentException(mappingType.toString());
- }
+ Map<String, OpenSearchDataType> fields =
+ parseMapping((Map<String, Object>) innerMap.getOrDefault("fields", Map.of()));
+ return (!fields.isEmpty()) ? OpenSearchTextType.of(fields) : OpenSearchTextType.of();
+ case GeoPoint: return OpenSearchGeoPointType.of();
+ case Binary: return OpenSearchBinaryType.of();
+ case Ip: return OpenSearchIpType.of();
+ case Date:
+ return OpenSearchDateType.create(
+ (String) innerMap.getOrDefault("format", ""));
+ default:
+ return res;
}
- res = new OpenSearchDataType(mappingType);
- res.exprCoreType = exprCoreType;
- return res;
}
/**
* A constructor function which builds proper `OpenSearchDataType` for given mapping `Type`.
* Designed to be called by the mapping parser only (and tests).
* @param mappingType A mapping type.
- * @param properties Properties to set.
- * @param fields Fields to set.
* @return An instance or inheritor of `OpenSearchDataType`.
*/
- public static OpenSearchDataType of(MappingType mappingType,
- Map<String, OpenSearchDataType> properties,
- Map<String, OpenSearchDataType> fields) {
- var res = of(mappingType);
- if (!properties.isEmpty() || !fields.isEmpty()) {
- // Clone to avoid changing the singleton instance.
- res = res.cloneEmpty();
- res.properties = ImmutableMap.copyOf(properties);
- res.fields = ImmutableMap.copyOf(fields);
- }
- return res;
- }
-
- protected OpenSearchDataType(MappingType mappingType) {
- this.mappingType = mappingType;
+ public static OpenSearchDataType of(MappingType mappingType) {
+ return of(mappingType, Map.of());
}
/**
@@ -165,11 +191,13 @@ public static OpenSearchDataType of(ExprType type) {
return new OpenSearchDataType((ExprCoreType) type);
}
- protected OpenSearchDataType(ExprCoreType type) {
- this.exprCoreType = type;
+ protected OpenSearchDataType(MappingType mappingType) {
+ this.mappingType = mappingType;
+ this.exprCoreType = mappingType.getExprCoreType();
}
- protected OpenSearchDataType() {
+ protected OpenSearchDataType(ExprCoreType type) {
+ this.exprCoreType = type;
}
// For datatypes with properties (example: object and nested types)
@@ -178,11 +206,6 @@ protected OpenSearchDataType() {
@EqualsAndHashCode.Exclude
Map<String, OpenSearchDataType> properties = ImmutableMap.of();
- // text could have fields
- // a read-only collection
- @EqualsAndHashCode.Exclude
- Map<String, OpenSearchDataType> fields = ImmutableMap.of();
-
@Override
// Called when building TypeEnvironment and when serializing PPL response
public String typeName() {
@@ -209,16 +232,16 @@ public String legacyTypeName() {
* @return A cloned object.
*/
protected OpenSearchDataType cloneEmpty() {
- var copy = new OpenSearchDataType();
- copy.mappingType = mappingType;
- copy.exprCoreType = exprCoreType;
- return copy;
+ if (this.mappingType == null) {
+ return new OpenSearchDataType(this.exprCoreType);
+ }
+ return new OpenSearchDataType(this.mappingType);
}
/**
* Flattens mapping tree into a single layer list of objects (pairs of name-types actually),
* which don't have nested types.
- * See {@link OpenSearchDataTypeTest#traverseAndFlatten() test} for example.
+ * See OpenSearchDataTypeTest#traverseAndFlatten() test for example.
* @param tree A list of `OpenSearchDataType`s - map between field name and its type.
* @return A list of all `OpenSearchDataType`s from given map on the same nesting level (1).
* Nested object names are prefixed by names of their host.
diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java
new file mode 100644
index 0000000000..0379d63275
--- /dev/null
+++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDateType.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.opensearch.data.type;
+
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import lombok.EqualsAndHashCode;
+import org.opensearch.common.time.DateFormatter;
+import org.opensearch.sql.data.type.ExprType;
+
+/**
+ * Date type with support for predefined and custom formats read from the index mapping.
+ */
+@EqualsAndHashCode(callSuper = true)
+public class OpenSearchDateType extends OpenSearchDataType {
+
+ private static final OpenSearchDateType instance = new OpenSearchDateType();
+
+ private static final String FORMAT_DELIMITER = "\\|\\|";
+
+ @EqualsAndHashCode.Exclude
+ String formatString;
+
+ private OpenSearchDateType() {
+ super(MappingType.Date);
+ this.formatString = "";
+ }
+
+ private OpenSearchDateType(String formatStringArg) {
+ super(MappingType.Date);
+ this.formatString = formatStringArg;
+ }
+
+ /**
+ * Retrieves and splits a user defined format string from the mapping into a list of formats.
+ * @return A list of format names and user defined formats.
+ */
+ public List<String> getFormatList() {
+ return Arrays.stream(formatString.split(FORMAT_DELIMITER))
+ .map(String::trim)
+ .collect(Collectors.toList());
+ }
+
+
+ /**
+ * Retrieves named formatters defined by OpenSearch.
+ * @return a list of DateFormatters that can be used to parse a Date/Time/Timestamp.
+ */
+ public List<DateFormatter> getNamedFormatters() {
+ return getFormatList().stream().filter(f -> {
+ try {
+ DateTimeFormatter.ofPattern(f);
+ //TODO: Filter off of a constant list of formats
+ return false;
+ } catch (IllegalArgumentException e) {
+ return true;
+ }
+ }).map(DateFormatter::forPattern).collect(Collectors.toList());
+ }
+
+ /**
+ * Create a Date type which has a LinkedHashMap defining all formats.
+ * @return A new type object.
+ */
+ public static OpenSearchDateType create(String format) {
+ return new OpenSearchDateType(format);
+ }
+
+ public static OpenSearchDateType of() {
+ return OpenSearchDateType.instance;
+ }
+
+ @Override
+ public boolean shouldCast(ExprType other) {
+ return false;
+ }
+
+ @Override
+ protected OpenSearchDataType cloneEmpty() {
+ return OpenSearchDateType.create(this.formatString);
+ }
+}
diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java
index 1098662e65..b27193757d 100644
--- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java
+++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchTextType.java
@@ -23,19 +23,24 @@ public class OpenSearchTextType extends OpenSearchDataType {
private static final OpenSearchTextType instance = new OpenSearchTextType();
+ // text could have fields
+ // a read-only collection
+ @EqualsAndHashCode.Exclude
+ Map<String, OpenSearchDataType> fields = ImmutableMap.of();
+
private OpenSearchTextType() {
super(MappingType.Text);
exprCoreType = UNKNOWN;
}
/**
- * Create a Text type which has fields.
- * @param fields Fields to set for the new type.
- * @return A new type object.
+ * Constructs a Text Type using the passed in fields argument.
+ * @param fields The fields to be used to construct the text type.
+ * @return A new OpenSeachTextTypeObject
*/
public static OpenSearchTextType of(Map<String, OpenSearchDataType> fields) {
var res = new OpenSearchTextType();
- res.fields = ImmutableMap.copyOf(fields);
+ res.fields = fields;
return res;
}
@@ -59,7 +64,7 @@ public Map<String, OpenSearchDataType> getFields() {
@Override
protected OpenSearchDataType cloneEmpty() {
- return OpenSearchTextType.of(fields);
+ return OpenSearchTextType.of(ImmutableMap.copyOf(this.fields));
}
/**
diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java
index 0c4548a368..8cdfd0b2ac 100644
--- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java
+++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java
@@ -6,12 +6,8 @@
package org.opensearch.sql.opensearch.data.value;
-import static org.opensearch.sql.data.type.ExprCoreType.DATE;
-import static org.opensearch.sql.data.type.ExprCoreType.DATETIME;
import static org.opensearch.sql.data.type.ExprCoreType.STRING;
import static org.opensearch.sql.data.type.ExprCoreType.STRUCT;
-import static org.opensearch.sql.data.type.ExprCoreType.TIME;
-import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP;
import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -19,16 +15,23 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterators;
+import java.time.DateTimeException;
import java.time.Instant;
+import java.time.LocalDate;
+import java.time.LocalTime;
+import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
+import java.time.temporal.TemporalAccessor;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.function.Function;
+import java.util.function.BiFunction;
import lombok.Getter;
import lombok.Setter;
+import org.apache.logging.log4j.LogManager;
+import org.opensearch.common.time.DateFormatter;
import org.opensearch.common.time.DateFormatters;
import org.opensearch.sql.data.model.ExprBooleanValue;
import org.opensearch.sql.data.model.ExprByteValue;
@@ -46,8 +49,11 @@
import org.opensearch.sql.data.model.ExprTimestampValue;
import org.opensearch.sql.data.model.ExprTupleValue;
import org.opensearch.sql.data.model.ExprValue;
+import org.opensearch.sql.data.type.ExprCoreType;
import org.opensearch.sql.data.type.ExprType;
+import org.opensearch.sql.expression.function.FunctionProperties;
import org.opensearch.sql.opensearch.data.type.OpenSearchDataType;
+import org.opensearch.sql.opensearch.data.type.OpenSearchDateType;
import org.opensearch.sql.opensearch.data.utils.Content;
import org.opensearch.sql.opensearch.data.utils.ObjectContent;
import org.opensearch.sql.opensearch.data.utils.OpenSearchJsonContent;
@@ -85,40 +91,44 @@ public void extendTypeMapping(Map<String, OpenSearchDataType> typeMapping) {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- private final Map<ExprType, Function<Content, ExprValue>> typeActionMap =
- new ImmutableMap.Builder<ExprType, Function<Content, ExprValue>>()
+ private final Map<ExprType, BiFunction<Content, ExprType, ExprValue>> typeActionMap =
+ new ImmutableMap.Builder<ExprType, BiFunction<Content, ExprType, ExprValue>>()
.put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Integer),
- c -> new ExprIntegerValue(c.intValue()))
+ (c, dt) -> new ExprIntegerValue(c.intValue()))
.put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Long),
- c -> new ExprLongValue(c.longValue()))
+ (c, dt) -> new ExprLongValue(c.longValue()))
.put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Short),
- c -> new ExprShortValue(c.shortValue()))
+ (c, dt) -> new ExprShortValue(c.shortValue()))
.put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Byte),
- c -> new ExprByteValue(c.byteValue()))
+ (c, dt) -> new ExprByteValue(c.byteValue()))
.put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Float),
- c -> new ExprFloatValue(c.floatValue()))
+ (c, dt) -> new ExprFloatValue(c.floatValue()))
.put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Double),
- c -> new ExprDoubleValue(c.doubleValue()))
+ (c, dt) -> new ExprDoubleValue(c.doubleValue()))
.put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Text),
- c -> new OpenSearchExprTextValue(c.stringValue()))
+ (c, dt) -> new OpenSearchExprTextValue(c.stringValue()))
.put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword),
- c -> new ExprStringValue(c.stringValue()))
+ (c, dt) -> new ExprStringValue(c.stringValue()))
.put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Boolean),
- c -> ExprBooleanValue.of(c.booleanValue()))
- .put(OpenSearchDataType.of(TIMESTAMP), this::parseTimestamp)
- .put(OpenSearchDataType.of(DATE),
- c -> new ExprDateValue(parseTimestamp(c).dateValue().toString()))
- .put(OpenSearchDataType.of(TIME),
- c -> new ExprTimeValue(parseTimestamp(c).timeValue().toString()))
- .put(OpenSearchDataType.of(DATETIME),
- c -> new ExprDatetimeValue(parseTimestamp(c).datetimeValue()))
+ (c, dt) -> ExprBooleanValue.of(c.booleanValue()))
+ //Handles the creation of DATE, TIME, TIMESTAMP
+ .put(OpenSearchDataType.of(ExprCoreType.TIMESTAMP),
+ (c, dt) -> parseTimestamp(c, dt))
+ .put(OpenSearchDataType.of(ExprCoreType.TIME),
+ (c, dt) -> parseTimestamp(c, dt))
+ .put(OpenSearchDataType.of(ExprCoreType.DATETIME),
+ (c, dt) -> parseTimestamp(c, dt))
+ .put(OpenSearchDataType.of(ExprCoreType.DATE),
+ (c, dt) -> parseTimestamp(c, dt))
+ .put(OpenSearchDateType.create(""),
+ (c, dt) -> parseTimestamp(c, dt))
.put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Ip),
- c -> new OpenSearchExprIpValue(c.stringValue()))
+ (c, dt) -> new OpenSearchExprIpValue(c.stringValue()))
.put(OpenSearchDataType.of(OpenSearchDataType.MappingType.GeoPoint),
- c -> new OpenSearchExprGeoPointValue(c.geoValue().getLeft(),
+ (c, dt) -> new OpenSearchExprGeoPointValue(c.geoValue().getLeft(),
c.geoValue().getRight()))
.put(OpenSearchDataType.of(OpenSearchDataType.MappingType.Binary),
- c -> new OpenSearchExprBinaryValue(c.stringValue()))
+ (c, dt) -> new OpenSearchExprBinaryValue(c.stringValue()))
.build();
/**
@@ -173,7 +183,7 @@ private ExprValue parse(Content content, String field, Optional<ExprType> fieldT
return parseArray(content, field);
} else {
if (typeActionMap.containsKey(type)) {
- return typeActionMap.get(type).apply(content);
+ return typeActionMap.get(type).apply(content, type);
} else {
throw new IllegalStateException(
String.format(
@@ -210,14 +220,88 @@ private ExprValue constructTimestamp(String value) {
}
}
- private ExprValue parseTimestamp(Content value) {
- if (value.isNumber()) {
- return new ExprTimestampValue(Instant.ofEpochMilli(value.longValue()));
- } else if (value.isString()) {
- return constructTimestamp(value.stringValue());
+ private TemporalAccessor parseTimestampString(String value, OpenSearchDateType dt) {
+ for (DateFormatter formatter : dt.getNamedFormatters()) {
+ try {
+ return formatter.parse(value);
+ } catch (IllegalArgumentException ignored) {
+ // nothing to do, try another format
+ }
+ }
+ return null;
+ }
+
+ private ExprValue formatReturn(ExprType formatType, ExprTimestampValue unformatted) {
+ if (formatType.equals(ExprCoreType.DATE)) {
+ return new ExprDateValue(unformatted.dateValue());
+ }
+ if (formatType.equals(ExprCoreType.DATETIME)) {
+ return new ExprDatetimeValue(unformatted.datetimeValue());
+ }
+ if (formatType.equals(ExprCoreType.TIME)) {
+ return new ExprTimeValue(unformatted.timeValue().toString());
+ }
+ return unformatted;
+ }
+
+ private ExprValue parseTimestamp(Content value, ExprType type) {
+ OpenSearchDateType dt;
+ ExprType returnFormat;
+ if (type instanceof OpenSearchDateType) {
+ //Case when an OpenSearchDateType is passed in
+ dt = (OpenSearchDateType) type;
+ returnFormat = dt.getExprType();
} else {
- return new ExprTimestampValue((Instant) value.objectValue());
+ //Case when an OpenSearchDataType.of(<ExprCoreType>) is passed in
+ dt = OpenSearchDateType.of();
+ returnFormat = ((OpenSearchDataType) type).getExprType();
+ }
+
+ if (value.isNumber()) {
+ return formatReturn(
+ returnFormat,
+ new ExprTimestampValue(Instant.ofEpochMilli(value.longValue())));
+ }
+
+ if (value.isString()) {
+ TemporalAccessor parsed = parseTimestampString(value.stringValue(),dt);
+ if (parsed == null) { // failed to parse or no formats given
+ return formatReturn(
+ returnFormat,
+ (ExprTimestampValue)constructTimestamp(value.stringValue()));
+ }
+ // Try Timestamp
+ try {
+ return formatReturn(returnFormat, new ExprTimestampValue(Instant.from(parsed)));
+ } catch (DateTimeException ignored) {
+ // nothing to do, try another type
+ }
+ //Try Time
+ try {
+ return formatReturn(
+ returnFormat,
+ new ExprTimestampValue(
+ new ExprTimeValue(LocalTime.from(parsed))
+ .timestampValue(new FunctionProperties(Instant.EPOCH, ZoneOffset.UTC))));
+ } catch (DateTimeException ignored) {
+ // nothing to do, try another type
+ }
+ //Try Date
+ try {
+ return formatReturn(
+ returnFormat,
+ new ExprTimestampValue(new ExprDateValue(LocalDate.from(parsed)).timestampValue()));
+ } catch (DateTimeException ignored) {
+ LogManager.getLogger(OpenSearchExprValueFactory.class).error(
+ String.format("Can't recognize parsed value: %s, %s", parsed, parsed.getClass()));
+ throw new IllegalStateException(
+ String.format(
+ "Construct ExprTimestampValue from \"%s\" failed, unsupported date format.",
+ value.stringValue()),
+ ignored);
+ }
}
+ return formatReturn(returnFormat, new ExprTimestampValue((Instant) value.objectValue()));
}
private ExprValue parseStruct(Content content, String prefix) {
diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/mapping/IndexMapping.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/mapping/IndexMapping.java
index 4fdcf0c637..0185ca95b6 100644
--- a/opensearch/src/main/java/org/opensearch/sql/opensearch/mapping/IndexMapping.java
+++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/mapping/IndexMapping.java
@@ -27,10 +27,15 @@ public class IndexMapping {
@Getter
private final Map<String, OpenSearchDataType> fieldMappings;
+ /**
+ * Maps each column in the index definition to an OpenSearchSQL datatype.
+ * @param metaData The metadata retrieved from the index mapping defined by the user.
+ */
@SuppressWarnings("unchecked")
public IndexMapping(MappingMetadata metaData) {
- this.fieldMappings = parseMapping((Map<String, Object>) metaData.getSourceAsMap()
- .getOrDefault("properties", null));
+ this.fieldMappings = OpenSearchDataType.parseMapping(
+ (Map<String, Object>) metaData.getSourceAsMap().getOrDefault("properties", null)
+ );
}
/**
@@ -41,28 +46,4 @@ public IndexMapping(MappingMetadata metaData) {
public int size() {
return fieldMappings.size();
}
-
- @SuppressWarnings("unchecked")
- private Map<String, OpenSearchDataType> parseMapping(Map<String, Object> indexMapping) {
- Map<String, OpenSearchDataType> result = new LinkedHashMap<>();
- if (indexMapping != null) {
- indexMapping.forEach((k, v) -> {
- var innerMap = (Map<String, Object>)v;
- // TODO: confirm that only `object` mappings can omit `type` field.
- var type = ((String) innerMap.getOrDefault("type", "object")).replace("_", "");
- if (!EnumUtils.isValidEnumIgnoreCase(OpenSearchDataType.MappingType.class, type)) {
- // unknown type, e.g. `alias`
- // TODO resolve alias reference
- return;
- }
- // TODO read formats for date type
- result.put(k, OpenSearchDataType.of(
- EnumUtils.getEnumIgnoreCase(OpenSearchDataType.MappingType.class, type),
- parseMapping((Map<String, Object>) innerMap.getOrDefault("properties", null)),
- parseMapping((Map<String, Object>) innerMap.getOrDefault("fields", null))
- ));
- });
- }
- return result;
- }
}
diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java
index 352635bfc3..8af9a4bbfa 100644
--- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java
+++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java
@@ -55,7 +55,6 @@
import org.opensearch.cluster.metadata.AliasMetadata;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.MappingMetadata;
-import org.opensearch.common.collect.ImmutableOpenMap;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.util.concurrent.ThreadContext;
import org.opensearch.common.xcontent.XContentType;
@@ -425,10 +424,8 @@ private void mockNodeClientSettings(String indexName, String indexMetadata)
GetSettingsResponse mockResponse = mock(GetSettingsResponse.class);
when(nodeClient.admin().indices().prepareGetSettings(any()).setLocal(anyBoolean()).get())
.thenReturn(mockResponse);
- ImmutableOpenMap<String, Settings> metadata =
- new ImmutableOpenMap.Builder<String, Settings>()
- .fPut(indexName, IndexMetadata.fromXContent(createParser(indexMetadata)).getSettings())
- .build();
+ Map<String, Settings> metadata = Map.of(indexName,
+ IndexMetadata.fromXContent(createParser(indexMetadata)).getSettings());
when(mockResponse.getIndexToSettings()).thenReturn(metadata);
}
diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java
index dd5bfd4e6f..141e21c38a 100644
--- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java
+++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java
@@ -49,7 +49,6 @@
import org.opensearch.client.indices.GetMappingsResponse;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.cluster.metadata.MappingMetadata;
-import org.opensearch.common.collect.ImmutableOpenMap;
import org.opensearch.common.settings.Settings;
import org.opensearch.common.xcontent.XContentType;
import org.opensearch.core.xcontent.DeprecationHandler;
@@ -232,9 +231,9 @@ void getIndexMaxResultWindowsSettings() throws IOException {
.put("index.max_result_window", maxResultWindow)
.build();
Settings emptySettings = Settings.builder().build();
- ImmutableOpenMap<String, Settings> indexToSettings =
+ Map<String, Settings> indexToSettings =
mockSettings(indexName, maxResultWindowSettings);
- ImmutableOpenMap<String, Settings> indexToDefaultSettings =
+ Map<String, Settings> indexToDefaultSettings =
mockSettings(indexName, emptySettings);
when(response.getIndexToSettings()).thenReturn(indexToSettings);
when(response.getIndexToDefaultSettings()).thenReturn(indexToDefaultSettings);
@@ -256,9 +255,9 @@ void getIndexMaxResultWindowsDefaultSettings() throws IOException {
.put("index.max_result_window", maxResultWindow)
.build();
Settings emptySettings = Settings.builder().build();
- ImmutableOpenMap<String, Settings> indexToSettings =
+ Map<String, Settings> indexToSettings =
mockSettings(indexName, emptySettings);
- ImmutableOpenMap<String, Settings> indexToDefaultSettings =
+ Map<String, Settings> indexToDefaultSettings =
mockSettings(indexName, maxResultWindowSettings);
when(response.getIndexToSettings()).thenReturn(indexToSettings);
when(response.getIndexToDefaultSettings()).thenReturn(indexToDefaultSettings);
@@ -427,10 +426,8 @@ private Map<String, MappingMetadata> mockFieldMappings(String indexName, String
return ImmutableMap.of(indexName, IndexMetadata.fromXContent(createParser(mappings)).mapping());
}
- private ImmutableOpenMap<String, Settings> mockSettings(String indexName, Settings settings) {
- ImmutableOpenMap.Builder<String, Settings> indexToSettingsBuilder = ImmutableOpenMap.builder();
- indexToSettingsBuilder.put(indexName, settings);
- return indexToSettingsBuilder.build();
+ private Map<String, Settings> mockSettings(String indexName, Settings settings) {
+ return Map.of(indexName, settings);
}
private XContentParser createParser(String mappings) throws IOException {
diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java
index 5cd76b1962..fe2865ec21 100644
--- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java
+++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java
@@ -9,6 +9,7 @@
import static org.junit.jupiter.api.Assertions.assertAll;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
@@ -18,6 +19,7 @@
import static org.opensearch.sql.data.type.ExprCoreType.ARRAY;
import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN;
import static org.opensearch.sql.data.type.ExprCoreType.BYTE;
+import static org.opensearch.sql.data.type.ExprCoreType.DATE;
import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE;
import static org.opensearch.sql.data.type.ExprCoreType.FLOAT;
import static org.opensearch.sql.data.type.ExprCoreType.INTEGER;
@@ -29,6 +31,8 @@
import static org.opensearch.sql.data.type.ExprCoreType.UNKNOWN;
import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.MappingType;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import org.apache.commons.lang3.reflect.FieldUtils;
@@ -39,6 +43,7 @@
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.EnumSource;
import org.junit.jupiter.params.provider.MethodSource;
+import org.opensearch.common.time.DateFormatter;
import org.opensearch.sql.data.type.ExprCoreType;
import org.opensearch.sql.data.type.ExprType;
@@ -49,6 +54,10 @@ class OpenSearchDataTypeTest {
private static final OpenSearchDataType textKeywordType =
OpenSearchTextType.of(Map.of("words", OpenSearchTextType.of(MappingType.Keyword)));
+ private static final String formatString = "epoch_millis || yyyyMMDD";
+
+ private static final OpenSearchDateType dateType = OpenSearchDateType.create(formatString);
+
@Test
public void isCompatible() {
assertTrue(STRING.isCompatible(textType));
@@ -157,15 +166,20 @@ public void of_OpenSearchDataType_from_MappingType(OpenSearchDataType.MappingTyp
@Test
// All types without `fields` and `properties` are singletones unless cloned.
- public void types_but_clones_are_singletones_and_cached() {
+ public void types_but_clones_are_singletons_and_cached() {
var type = OpenSearchDataType.of(MappingType.Object);
var alsoType = OpenSearchDataType.of(MappingType.Object);
- var typeWithProperties = OpenSearchDataType.of(MappingType.Object,
- Map.of("subfield", OpenSearchDataType.of(INTEGER)), Map.of());
- var typeWithFields = OpenSearchDataType.of(MappingType.Text,
- Map.of(), Map.of("subfield", OpenSearchDataType.of(INTEGER)));
-
+ Map<String, Object> properties = Map.of(
+ "properties",
+ Map.of("number", Map.of("type", "integer")));
+ var typeWithProperties = OpenSearchDataType.of(
+ MappingType.Object,
+ properties);
+ var typeWithFields = OpenSearchDataType.of(
+ MappingType.Text,
+ Map.of());
var cloneType = type.cloneEmpty();
+
assertAll(
() -> assertSame(type, alsoType),
() -> assertNotSame(type, cloneType),
@@ -173,6 +187,7 @@ public void types_but_clones_are_singletones_and_cached() {
() -> assertNotSame(type, typeWithFields),
() -> assertNotSame(typeWithProperties, typeWithProperties.cloneEmpty()),
() -> assertNotSame(typeWithFields, typeWithFields.cloneEmpty()),
+ () -> assertNotSame(dateType, dateType.cloneEmpty()),
() -> assertSame(OpenSearchDataType.of(MappingType.Text),
OpenSearchTextType.of()),
() -> assertSame(OpenSearchDataType.of(MappingType.Binary),
@@ -182,7 +197,7 @@ public void types_but_clones_are_singletones_and_cached() {
() -> assertSame(OpenSearchDataType.of(MappingType.Ip),
OpenSearchIpType.of()),
() -> assertNotSame(OpenSearchTextType.of(),
- OpenSearchTextType.of(Map.of("subfield", OpenSearchDataType.of(INTEGER)))),
+ OpenSearchTextType.of(Map.of("properties", OpenSearchDataType.of(INTEGER)))),
() -> assertSame(OpenSearchDataType.of(INTEGER), OpenSearchDataType.of(INTEGER)),
() -> assertSame(OpenSearchDataType.of(STRING), OpenSearchDataType.of(STRING)),
() -> assertSame(OpenSearchDataType.of(STRUCT), OpenSearchDataType.of(STRUCT)),
@@ -213,26 +228,23 @@ public void fields_and_properties_are_readonly() {
@Test
// Test and type added for coverage only
public void of_null_MappingType() {
- assertThrows(IllegalArgumentException.class, () -> OpenSearchDataType.of(MappingType.Invalid));
+ assertNotNull(OpenSearchDataType.of(MappingType.Invalid));
}
@Test
// cloneEmpty doesn't clone properties and fields.
// Fields are cloned by OpenSearchTextType::cloneEmpty, because it is used in that type only.
public void cloneEmpty() {
- var type = OpenSearchDataType.of(MappingType.Object,
- Map.of("val", OpenSearchDataType.of(INTEGER)),
- Map.of("words", OpenSearchDataType.of(STRING)));
+ var type = OpenSearchDataType.of(
+ MappingType.Object,
+ Map.of("val", OpenSearchDataType.of(INTEGER))
+ );
var clone = type.cloneEmpty();
var textClone = textKeywordType.cloneEmpty();
assertAll(
// can compare because `properties` and `fields` are marked as @EqualsAndHashCode.Exclude
() -> assertEquals(type, clone),
- // read private field `fields`
- () -> assertTrue(
- ((Map<String, OpenSearchDataType>) FieldUtils.readField(clone, "fields", true))
- .isEmpty()),
() -> assertTrue(clone.getProperties().isEmpty()),
() -> assertEquals(textKeywordType, textClone),
() -> assertEquals(FieldUtils.readField(textKeywordType, "fields", true),
@@ -261,17 +273,17 @@ public void cloneEmpty() {
// =================
// as
// =================
- // type : Object
- // type.subtype : Object
- // type.subtype.subsubtype : Object
- // type.subtype.subsubtype.textWithKeywordType : Text
+ // mapping : Object
+ // mapping.submapping : Object
+ // mapping.submapping.subsubmapping : Object
+ // mapping.submapping.subsubmapping.textWithKeywordType : Text
// |- keyword : Keyword
- // type.subtype.subsubtype.INTEGER : INTEGER
- // type.subtype.geo_point : GeoPoint
- // type.subtype.textWithFieldsType: Text
+ // mapping.submapping.subsubmapping.INTEGER : INTEGER
+ // mapping.submapping.geo_point : GeoPoint
+ // mapping.submapping.textWithFieldsType: Text
// |- words : Keyword
- // type.text : Text
- // type.keyword : Keyword
+ // mapping.text : Text
+ // mapping.keyword : Keyword
// ==================
// Objects are flattened by OpenSearch, but Texts aren't
// TODO Arrays
@@ -281,28 +293,27 @@ public void traverseAndFlatten() {
var objectType = OpenSearchDataType.of(MappingType.Object);
assertAll(
() -> assertEquals(9, flattened.size()),
- () -> assertTrue(flattened.get("type").getProperties().isEmpty()),
- () -> assertTrue(flattened.get("type.subtype").getProperties().isEmpty()),
- () -> assertTrue(flattened.get("type.subtype.subsubtype").getProperties().isEmpty()),
+ () -> assertTrue(flattened.get("mapping").getProperties().isEmpty()),
+ () -> assertTrue(flattened.get("mapping.submapping").getProperties().isEmpty()),
+ () -> assertTrue(
+ flattened.get("mapping.submapping.subsubmapping").getProperties().isEmpty()),
- () -> assertEquals(objectType, flattened.get("type")),
- () -> assertEquals(objectType, flattened.get("type.subtype")),
- () -> assertEquals(objectType, flattened.get("type.subtype.subsubtype")),
+ () -> assertEquals(objectType, flattened.get("mapping")),
+ () -> assertEquals(objectType, flattened.get("mapping.submapping")),
+ () -> assertEquals(objectType, flattened.get("mapping.submapping.subsubmapping")),
() -> assertEquals(OpenSearchDataType.of(MappingType.Keyword),
- flattened.get("type.keyword")),
+ flattened.get("mapping.keyword")),
() -> assertEquals(OpenSearchDataType.of(MappingType.Text),
- flattened.get("type.text")),
-
+ flattened.get("mapping.text")),
() -> assertEquals(OpenSearchGeoPointType.of(),
- flattened.get("type.subtype.geo_point")),
+ flattened.get("mapping.submapping.geo_point")),
() -> assertEquals(OpenSearchTextType.of(),
- flattened.get("type.subtype.textWithFieldsType")),
-
+ flattened.get("mapping.submapping.textWithFieldsType")),
() -> assertEquals(OpenSearchTextType.of(),
- flattened.get("type.subtype.subsubtype.textWithKeywordType")),
+ flattened.get("mapping.submapping.subsubmapping.texttype")),
() -> assertEquals(OpenSearchDataType.of(INTEGER),
- flattened.get("type.subtype.subsubtype.INTEGER"))
+ flattened.get("mapping.submapping.subsubmapping.INTEGER"))
);
}
@@ -313,13 +324,13 @@ public void resolve() {
assertAll(
() -> assertNull(OpenSearchDataType.resolve(mapping, "incorrect")),
() -> assertEquals(OpenSearchDataType.of(MappingType.Object),
- OpenSearchDataType.resolve(mapping, "type")),
+ OpenSearchDataType.resolve(mapping, "mapping")),
() -> assertEquals(OpenSearchDataType.of(MappingType.Object),
- OpenSearchDataType.resolve(mapping, "subtype")),
+ OpenSearchDataType.resolve(mapping, "submapping")),
() -> assertEquals(OpenSearchDataType.of(MappingType.Object),
- OpenSearchDataType.resolve(mapping, "subsubtype")),
+ OpenSearchDataType.resolve(mapping, "subsubmapping")),
() -> assertEquals(OpenSearchDataType.of(MappingType.Text),
- OpenSearchDataType.resolve(mapping, "textWithKeywordType")),
+ OpenSearchDataType.resolve(mapping, "texttype")),
() -> assertEquals(OpenSearchDataType.of(MappingType.Text),
OpenSearchDataType.resolve(mapping, "textWithFieldsType")),
() -> assertEquals(OpenSearchDataType.of(MappingType.Text),
@@ -358,28 +369,31 @@ public void text_type_with_fields_ctor() {
}
private Map<String, OpenSearchDataType> getSampleMapping() {
- var textWithKeywordType = OpenSearchTextType.of(Map.of("keyword",
- OpenSearchDataType.of(MappingType.Keyword)));
+ Map<String, Object> subsubmapping = Map.of(
+ "properties", Map.of(
+ "texttype", Map.of("type", "text"),
+ "INTEGER", Map.of("type", "integer")
+ )
+ );
+
+ Map<String, Object> submapping = Map.of(
+ "properties", Map.of(
+ "subsubmapping", subsubmapping,
+ "textWithFieldsType", Map.of("type", "text", "fieldsType", true),
+ "geo_point", Map.of("type", "geo_point")
+ )
+ );
- var subsubsubtypes = Map.of(
- "textWithKeywordType", textWithKeywordType,
- "INTEGER", OpenSearchDataType.of(INTEGER));
-
- var subsubtypes = Map.of(
- "subsubtype", OpenSearchDataType.of(MappingType.Object,
- subsubsubtypes, Map.of()),
- "textWithFieldsType", OpenSearchDataType.of(MappingType.Text, Map.of(),
- Map.of("words", OpenSearchDataType.of(MappingType.Keyword))),
- "geo_point", OpenSearchGeoPointType.of());
-
- var subtypes = Map.of(
- "subtype", OpenSearchDataType.of(MappingType.Object,
- subsubtypes, Map.of()),
- "keyword", OpenSearchDataType.of(MappingType.Keyword),
- "text", OpenSearchDataType.of(MappingType.Text));
-
- var type = OpenSearchDataType.of(MappingType.Object, subtypes, Map.of());
- return Map.of("type", type);
+ Map<String, Object> types = Map.of(
+ "properties", Map.of(
+ "submapping", submapping,
+ "keyword", Map.of("type", "keyword"),
+ "text", Map.of("type", "text")
+ )
+ );
+
+ var mapping = OpenSearchDataType.of(MappingType.Object, types);
+ return Map.of("mapping", mapping);
}
@Test
@@ -392,4 +406,15 @@ public void test_getExprType() {
assertEquals(DOUBLE, OpenSearchDataType.of(MappingType.ScaledFloat).getExprType());
assertEquals(TIMESTAMP, OpenSearchDataType.of(MappingType.Date).getExprType());
}
+
+ @Test
+ public void test_getRegularFormatters() {
+ List<DateFormatter> definedFormatters = dateType.getNamedFormatters();
+ assertEquals(definedFormatters.get(0), DateFormatter.forPattern("epoch_millis"));
+ }
+
+ @Test
+ public void test_shouldCastFunction() {
+ assertFalse(dateType.shouldCast(DATE));
+ }
}
diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java
index 2dfa5de93a..b60402e746 100644
--- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java
+++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprTextValueTest.java
@@ -55,9 +55,9 @@ void non_text_types_arent_converted() {
@Test
void non_text_types_with_nested_objects_arent_converted() {
var objectType = OpenSearchDataType.of(OpenSearchDataType.MappingType.Object,
- Map.of("subfield", OpenSearchDataType.of(STRING)), Map.of());
+ Map.of("subfield", OpenSearchDataType.of(STRING)));
var arrayType = OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested,
- Map.of("subfield", OpenSearchDataType.of(STRING)), Map.of());
+ Map.of("subfield", OpenSearchDataType.of(STRING)));
assertAll(
() -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", objectType)),
() -> assertEquals("field", OpenSearchTextType.convertTextToKeyword("field", arrayType))
diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java
index 8f2c954f65..fb23394ff6 100644
--- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java
+++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java
@@ -53,6 +53,7 @@
import org.opensearch.sql.data.model.ExprTupleValue;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.opensearch.data.type.OpenSearchDataType;
+import org.opensearch.sql.opensearch.data.type.OpenSearchDateType;
import org.opensearch.sql.opensearch.data.type.OpenSearchTextType;
import org.opensearch.sql.opensearch.data.utils.OpenSearchJsonContent;
@@ -71,6 +72,12 @@ class OpenSearchExprValueFactoryTest {
.put("datetimeV", OpenSearchDataType.of(DATETIME))
.put("timeV", OpenSearchDataType.of(TIME))
.put("timestampV", OpenSearchDataType.of(TIMESTAMP))
+ .put("hourV", OpenSearchDateType.create("hour"))
+ .put("epochSecondV", OpenSearchDateType.create("epoch_second"))
+ .put("dateStringV", OpenSearchDateType.create("date"))
+ .put("epochMillisV", OpenSearchDateType.create("epoch_millis"))
+ .put("dateOrEpochMillisV", OpenSearchDateType.create("date_time_no_millis||epoch_millis"))
+ .put("badDateFormatV", OpenSearchDateType.create("MM,DD"))
.put("boolV", OpenSearchDataType.of(BOOLEAN))
.put("structV", OpenSearchDataType.of(STRUCT))
.put("structV.id", OpenSearchDataType.of(INTEGER))
@@ -218,6 +225,9 @@ public void constructDate() {
assertEquals(
new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)),
constructFromObject("timestampV", Instant.ofEpochMilli(1420070400001L)));
+ assertEquals(
+ new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)),
+ constructFromObject("epochMillisV", "1420070400001"));
assertEquals(
new ExprTimestampValue("2015-01-01 12:10:30"),
constructFromObject("timestampV", "2015-01-01 12:10:30"));
@@ -230,6 +240,15 @@ public void constructDate() {
assertEquals(
new ExprDatetimeValue("2015-01-01 12:10:30"),
constructFromObject("datetimeV", "2015-01-01 12:10:30"));
+ assertEquals(
+ new ExprDatetimeValue("1970-01-01 09:00:00"),
+ constructFromObject("hourV", "09"));
+ assertEquals(
+ new ExprDatetimeValue("1984-04-12 00:00:00"),
+ constructFromObject("dateStringV", "1984-04-12"));
+ assertEquals(
+ new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)),
+ constructFromObject("dateOrEpochMillisV", "1420070400001"));
}
@Test
@@ -241,6 +260,19 @@ public void constructDateFromUnsupportedFormatThrowException() {
"Construct ExprTimestampValue from \"2015-01-01 12:10\" failed, "
+ "unsupported date format.",
exception.getMessage());
+
+ exception = assertThrows(
+ IllegalStateException.class, () -> tupleValue("{\"badDateFormatV\":\"11,22\"}"));
+ assertEquals(
+ "Construct ExprTimestampValue from \"11,22\" failed, "
+ + "unsupported date format.",
+ exception.getMessage());
+ exception = assertThrows(
+ IllegalStateException.class, () -> tupleValue("{\"dateStringV\":\"2023-11\"}"));
+ assertEquals(
+ "Construct ExprTimestampValue from \"2023-11\" failed, "
+ + "unsupported date format.",
+ exception.getMessage());
}
@Test
@@ -431,7 +463,7 @@ private ExprValue constructFromObject(String fieldName, Object value) {
private static class TestType extends OpenSearchDataType {
public TestType() {
- mappingType = null;
+ super(MappingType.Invalid);
}
@Override