From 752da2154d337c74ce60752fe8e47b97ea182bb3 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Mon, 21 Aug 2023 09:40:29 -0700 Subject: [PATCH] Add support for `date_nanos` and tests. (#337) (#1976) * Add support for `date_nanos` and tests. (#337) * Add support for `date_nanos` and tests. Signed-off-by: Yury-Fridlyand * Add more IT. Signed-off-by: Yury-Fridlyand --------- Signed-off-by: Yury-Fridlyand * Typo fix in IT. Signed-off-by: Yury-Fridlyand * Address PR feedback. Signed-off-by: Yury-Fridlyand * Spotless Signed-off-by: Yury-Fridlyand --------- Signed-off-by: Yury-Fridlyand --- .../org/opensearch/sql/ppl/DataTypeIT.java | 1 + .../opensearch/sql/ppl/SystemFunctionIT.java | 25 ++++-- .../opensearch/sql/sql/DateTimeFormatsIT.java | 90 +++++++++++++++++++ .../opensearch/sql/sql/SystemFunctionIT.java | 18 +++- integ-test/src/test/resources/datatypes.json | 2 +- .../src/test/resources/date_formats.json | 2 +- .../datatypes_index_mapping.json | 5 +- .../date_formats_index_mapping.json | 2 +- .../data/type/OpenSearchDataType.java | 3 + .../data/type/OpenSearchDataTypeTest.java | 1 + 10 files changed, 133 insertions(+), 16 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java index 8b5a6d498e..fe5c2ff270 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DataTypeIT.java @@ -49,6 +49,7 @@ public void test_nonnumeric_data_types() throws IOException { schema("text_value", "string"), schema("binary_value", "binary"), schema("date_value", "timestamp"), + schema("date_nanos_value", "timestamp"), schema("ip_value", "ip"), schema("object_value", "struct"), schema("nested_value", "array"), diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java index d2cd140e99..1c23935f81 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SystemFunctionIT.java @@ -70,19 +70,28 @@ public void typeof_opensearch_types() throws IOException { response = executeQuery( String.format( - "source=%s | eval " - + "`text` = typeof(text_value), `date` = typeof(date_value)," - + "`boolean` = typeof(boolean_value), `object` = typeof(object_value)," - + "`keyword` = typeof(keyword_value), `ip` = typeof(ip_value)," - + "`binary` = typeof(binary_value), `geo_point` = typeof(geo_point_value)" + "source=%s | eval `text` = typeof(text_value), `date` = typeof(date_value)," + + " `date_nanos` = typeof(date_nanos_value),`boolean` = typeof(boolean_value)," + + " `object` = typeof(object_value),`keyword` = typeof(keyword_value), `ip` =" + + " typeof(ip_value),`binary` = typeof(binary_value), `geo_point` =" + + " typeof(geo_point_value)" // TODO activate this test once `ARRAY` type supported, see // ExpressionAnalyzer::isTypeNotSupported // + ", `nested` = typeof(nested_value)" - + " | fields `text`, `date`, `boolean`, `object`, `keyword`, `ip`, `binary`," - + " `geo_point`", + + " | fields `text`, `date`, `date_nanos`, `boolean`, `object`, `keyword`," + + " `ip`, `binary`, `geo_point`", TEST_INDEX_DATATYPE_NONNUMERIC)); verifyDataRows( response, - rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD", "IP", "BINARY", "GEO_POINT")); + rows( + "TEXT", + "TIMESTAMP", + "TIMESTAMP", + "BOOLEAN", + "OBJECT", + "KEYWORD", + "IP", + "BINARY", + "GEO_POINT")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java index d6f2d2c7f4..13c2eecd56 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFormatsIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.sql; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE_FORMATS; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; import static org.opensearch.sql.util.MatcherUtils.rows; @@ -29,6 +30,7 @@ public class DateTimeFormatsIT extends SQLIntegTestCase { public void init() throws Exception { super.init(); loadIndex(Index.DATE_FORMATS); + loadIndex(Index.DATA_TYPE_NONNUMERIC); } @Test @@ -139,6 +141,94 @@ public void testNumericFormats() { rows("1970-01-02 03:55:00", "1970-01-01 00:01:40.5")); } + @Test + @SneakyThrows + public void testDateNanosWithFormats() { + String query = + String.format("SELECT hour_minute_second_OR_t_time" + " FROM %s", TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifyDataRows(result, rows("09:07:42"), rows("07:07:42.123456789")); + } + + @Test + @SneakyThrows + public void testDateNanosWithFunctions() { + // in memory funcs + String query = + String.format( + "SELECT" + + " hour_minute_second_OR_t_time > TIME '08:07:00'," + + " hour_minute_second_OR_t_time < TIME '08:07:00'," + + " hour_minute_second_OR_t_time = t_time_no_millis," + + " hour_minute_second_OR_t_time <> strict_t_time," + + " hour_minute_second_OR_t_time >= t_time" + + " FROM %s", + TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema( + result, + schema("hour_minute_second_OR_t_time > TIME '08:07:00'", null, "boolean"), + schema("hour_minute_second_OR_t_time < TIME '08:07:00'", null, "boolean"), + schema("hour_minute_second_OR_t_time = t_time_no_millis", null, "boolean"), + schema("hour_minute_second_OR_t_time <> strict_t_time", null, "boolean"), + schema("hour_minute_second_OR_t_time >= t_time", null, "boolean")); + verifyDataRows( + result, rows(true, false, true, false, true), rows(false, true, false, true, false)); + // push down + query = + String.format( + "SELECT hour_minute_second_OR_t_time" + + " FROM %s WHERE hour_minute_second_OR_t_time > TIME '08:07:00'", + TEST_INDEX_DATE_FORMATS); + result = executeQuery(query); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifyDataRows(result, rows("09:07:42")); + query = + String.format( + "SELECT hour_minute_second_OR_t_time" + + " FROM %s WHERE hour_minute_second_OR_t_time < TIME '08:07:00'", + TEST_INDEX_DATE_FORMATS); + result = executeQuery(query); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifyDataRows(result, rows("07:07:42.123456789")); + } + + @Test + @SneakyThrows + public void testDateNanosOrderBy() { + String query = + String.format( + "SELECT hour_minute_second_OR_t_time" + + " FROM %s ORDER BY hour_minute_second_OR_t_time ASC", + TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, schema("hour_minute_second_OR_t_time", null, "time")); + verifyDataRows(result, rows("07:07:42.123456789"), rows("09:07:42")); + } + + @Test + @SneakyThrows + public void testDateNanosGroupBy() { + String query = + String.format( + "SELECT count(*)" + " FROM %s GROUP BY hour_minute_second_OR_t_time", + TEST_INDEX_DATE_FORMATS); + JSONObject result = executeQuery(query); + verifySchema(result, schema("count(*)", null, "integer")); + verifyDataRows(result, rows(1), rows(1)); + } + + @Test + @SneakyThrows + public void testDateNanosWithNanos() { + String query = + String.format("SELECT date_nanos_value" + " FROM %s", TEST_INDEX_DATATYPE_NONNUMERIC); + JSONObject result = executeQuery(query); + verifySchema(result, schema("date_nanos_value", null, "timestamp")); + verifyDataRows(result, rows("2019-03-24 01:34:46.123456789")); + } + protected JSONObject executeQuery(String query) throws IOException { Request request = new Request("POST", QUERY_API_ENDPOINT); request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java index 4b39e2925c..d2798728a1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/SystemFunctionIT.java @@ -55,9 +55,10 @@ public void typeof_opensearch_types() { response = executeJdbcRequest( String.format( - "SELECT typeof(text_value),typeof(date_value), typeof(boolean_value)," - + " typeof(object_value), typeof(keyword_value),typeof(ip_value)," - + " typeof(binary_value), typeof(geo_point_value)" + "SELECT typeof(text_value),typeof(date_value), typeof(date_nanos_value)," + + " typeof(boolean_value), typeof(object_value)," + + " typeof(keyword_value),typeof(ip_value), typeof(binary_value)," + + " typeof(geo_point_value)" // TODO activate this test once `ARRAY` type supported, see // ExpressionAnalyzer::isTypeNotSupported // + ", typeof(nested_value)" @@ -65,6 +66,15 @@ public void typeof_opensearch_types() { TEST_INDEX_DATATYPE_NONNUMERIC)); verifyDataRows( response, - rows("TEXT", "TIMESTAMP", "BOOLEAN", "OBJECT", "KEYWORD", "IP", "BINARY", "GEO_POINT")); + rows( + "TEXT", + "TIMESTAMP", + "TIMESTAMP", + "BOOLEAN", + "OBJECT", + "KEYWORD", + "IP", + "BINARY", + "GEO_POINT")); } } diff --git a/integ-test/src/test/resources/datatypes.json b/integ-test/src/test/resources/datatypes.json index ea3290ee64..70ddd28763 100644 --- a/integ-test/src/test/resources/datatypes.json +++ b/integ-test/src/test/resources/datatypes.json @@ -1,2 +1,2 @@ {"index":{"_id":"1"}} -{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" : "Smith"}, {"first" : "Alice", "last" : "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }} +{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "date_nanos_value": "2019-03-23T21:34:46.123456789-04:00", "ip_value": "127.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" : "Smith"}, {"first" : "Alice", "last" : "White"}], "geo_point_value": { "lat": 40.71, "lon": 74.00 }} diff --git a/integ-test/src/test/resources/date_formats.json b/integ-test/src/test/resources/date_formats.json index 13d46a0e8c..2ff0c867a3 100644 --- a/integ-test/src/test/resources/date_formats.json +++ b/integ-test/src/test/resources/date_formats.json @@ -1,4 +1,4 @@ {"index": {}} {"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 AM", "yyyy-MM-dd_OR_epoch_millis": "1984-04-12", "hour_minute_second_OR_t_time": "09:07:42", "custom_timestamp": "1984-04-12 09:07:42 ---- AM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time": "1961-04-12", "custom_time_parser_check": "85476321", "incomplete_1" : 1984, "incomplete_2": null, "incomplete_custom_date": 1999, "incomplete_custom_time" : 10, "incorrect" : null, "epoch_sec" : 42, "epoch_milli" : 42, "custom_no_delimiter_date" : "19841020", "custom_no_delimiter_time" : "102030", "custom_no_delimiter_ts" : "19841020153548"} {"index": {}} -{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 PM", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T09:07:42.000Z", "custom_timestamp": "1984-04-12 10:07:42 ---- PM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time": "09:07:00", "custom_time_parser_check": "::: 9-32476542", "incomplete_1" : 2012, "incomplete_2": null, "incomplete_custom_date": 3021, "incomplete_custom_time" : 20, "incorrect" : null, "epoch_sec" : 100500, "epoch_milli" : 100500, "custom_no_delimiter_date" : "19610412", "custom_no_delimiter_time" : "090700", "custom_no_delimiter_ts" : "19610412090700"} +{"epoch_millis": "450608862000.123456", "epoch_second": "450608862.000123456", "date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time": "1984-04-12T09:07:42.000Z", "strict_date_optional_time_nanos": "1984-04-12T09:07:42.000123456Z", "basic_date": "19840412", "basic_date_time": "19840412T090742.000Z", "basic_date_time_no_millis": "19840412T090742Z", "basic_ordinal_date": "1984103", "basic_ordinal_date_time": "1984103T090742.000Z", "basic_ordinal_date_time_no_millis": "1984103T090742Z", "basic_time": "090742.000Z", "basic_time_no_millis": "090742Z", "basic_t_time": "T090742.000Z", "basic_t_time_no_millis": "T090742Z", "basic_week_date": "1984W154", "strict_basic_week_date": "1984W154", "basic_week_date_time": "1984W154T090742.000Z", "strict_basic_week_date_time": "1984W154T090742.000Z", "basic_week_date_time_no_millis": "1984W154T090742Z", "strict_basic_week_date_time_no_millis": "1984W154T090742Z", "date": "1984-04-12", "strict_date": "1984-04-12", "date_hour": "1984-04-12T09", "strict_date_hour": "1984-04-12T09", "date_hour_minute": "1984-04-12T09:07", "strict_date_hour_minute": "1984-04-12T09:07", "date_hour_minute_second": "1984-04-12T09:07:42", "strict_date_hour_minute_second": "1984-04-12T09:07:42", "date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_fraction": "1984-04-12T09:07:42.000", "date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "strict_date_hour_minute_second_millis": "1984-04-12T09:07:42.000", "date_time": "1984-04-12T09:07:42.000Z", "strict_date_time": "1984-04-12T09:07:42.000123456Z", "date_time_no_millis": "1984-04-12T09:07:42Z", "strict_date_time_no_millis": "1984-04-12T09:07:42Z", "hour": "09", "strict_hour": "09", "hour_minute": "09:07", "strict_hour_minute": "09:07", "hour_minute_second": "09:07:42", "strict_hour_minute_second": "09:07:42", "hour_minute_second_fraction": "09:07:42.000", "strict_hour_minute_second_fraction": "09:07:42.000", "hour_minute_second_millis": "09:07:42.000", "strict_hour_minute_second_millis": "09:07:42.000", "ordinal_date": "1984-103", "strict_ordinal_date": "1984-103", "ordinal_date_time": "1984-103T09:07:42.000123456Z", "strict_ordinal_date_time": "1984-103T09:07:42.000123456Z", "ordinal_date_time_no_millis": "1984-103T09:07:42Z", "strict_ordinal_date_time_no_millis": "1984-103T09:07:42Z", "time": "09:07:42.000Z", "strict_time": "09:07:42.000Z", "time_no_millis": "09:07:42Z", "strict_time_no_millis": "09:07:42Z", "t_time": "T09:07:42.000Z", "strict_t_time": "T09:07:42.000Z", "t_time_no_millis": "T09:07:42Z", "strict_t_time_no_millis": "T09:07:42Z", "week_date": "1984-W15-4", "strict_week_date": "1984-W15-4", "week_date_time": "1984-W15-4T09:07:42.000Z", "strict_week_date_time": "1984-W15-4T09:07:42.000Z", "week_date_time_no_millis": "1984-W15-4T09:07:42Z", "strict_week_date_time_no_millis": "1984-W15-4T09:07:42Z", "weekyear_week_day": "1984-W15-4", "strict_weekyear_week_day": "1984-W15-4", "year_month_day": "1984-04-12", "strict_year_month_day": "1984-04-12", "yyyy-MM-dd": "1984-04-12", "custom_time": "09:07:42 PM", "yyyy-MM-dd_OR_epoch_millis": "450608862000.123456", "hour_minute_second_OR_t_time": "T07:07:42.123456789Z", "custom_timestamp": "1984-04-12 10:07:42 ---- PM", "custom_date_or_date": "1984-04-12", "custom_date_or_custom_time": "09:07:00", "custom_time_parser_check": "::: 9-32476542", "incomplete_1" : 2012, "incomplete_2": null, "incomplete_custom_date": 3021, "incomplete_custom_time" : 20, "incorrect" : null, "epoch_sec" : 100500, "epoch_milli" : 100500, "custom_no_delimiter_date" : "19610412", "custom_no_delimiter_time" : "090700", "custom_no_delimiter_ts" : "19610412090700"} diff --git a/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json index 8c1759b369..5908114201 100644 --- a/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json @@ -13,10 +13,13 @@ "binary_value": { "type": "binary" }, - "date_value": { + "date_value": { "type" : "date", "format": "yyyy-MM-dd HH:mm:ss" }, + "date_nanos_value": { + "type" : "date_nanos" + }, "ip_value": { "type": "ip" }, diff --git a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json index 65811f8d9e..0b6daaacb4 100644 --- a/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/date_formats_index_mapping.json @@ -298,7 +298,7 @@ "format": "yyyy-MM-dd||epoch_millis" }, "hour_minute_second_OR_t_time" : { - "type" : "date", + "type" : "date_nanos", "format": "hour_minute_second||t_time" }, "custom_timestamp" : { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java index d276374539..ddbba61260 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataType.java @@ -30,6 +30,7 @@ public enum MappingType { GeoPoint("geo_point", ExprCoreType.UNKNOWN), Binary("binary", ExprCoreType.UNKNOWN), Date("date", ExprCoreType.TIMESTAMP), + DateNanos("date_nanos", ExprCoreType.TIMESTAMP), Object("object", ExprCoreType.STRUCT), Nested("nested", ExprCoreType.ARRAY), Byte("byte", ExprCoreType.BYTE), @@ -130,6 +131,7 @@ public static Map parseMapping(Map i * @param mappingType A mapping type. * @return An instance or inheritor of `OpenSearchDataType`. */ + @SuppressWarnings("unchecked") public static OpenSearchDataType of(MappingType mappingType, Map innerMap) { OpenSearchDataType res = instances.getOrDefault(mappingType.toString(), new OpenSearchDataType(mappingType)); @@ -157,6 +159,7 @@ public static OpenSearchDataType of(MappingType mappingType, Map case Ip: return OpenSearchIpType.of(); case Date: + case DateNanos: // Default date formatter is used when "" is passed as the second parameter String format = (String) innerMap.getOrDefault("format", ""); return OpenSearchDateType.of(format); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java index b0288dc9a7..82e6222dc4 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/type/OpenSearchDataTypeTest.java @@ -105,6 +105,7 @@ private static Stream getTestDataWithType() { Arguments.of(MappingType.Double, "double", DOUBLE), Arguments.of(MappingType.Boolean, "boolean", BOOLEAN), Arguments.of(MappingType.Date, "date", TIMESTAMP), + Arguments.of(MappingType.DateNanos, "date", TIMESTAMP), Arguments.of(MappingType.Object, "object", STRUCT), Arguments.of(MappingType.Nested, "nested", ARRAY), Arguments.of(MappingType.GeoPoint, "geo_point", OpenSearchGeoPointType.of()),