diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index c6b21e1605..ee5e83db7f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -89,6 +89,18 @@ public void testLimitPushDownExplain() throws Exception { + "| fields ageMinus")); } + @Test + public void testFillNullPushDownExplain() throws Exception { + String expected = loadFromFile("expectedOutput/ppl/explain_fillnull_push.json"); + + assertJsonEquals( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account" + + "| fields age, balance " + + "| fillnull with -1 in age,balance")); + } + String loadFromFile(String filename) throws Exception { URI uri = Resources.getResource(filename).toURI(); return new String(Files.readAllBytes(Paths.get(uri))); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/FillNullCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/FillNullCommandIT.java new file mode 100644 index 0000000000..5a1604ec05 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/FillNullCommandIT.java @@ -0,0 +1,130 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl; + +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; + +public class FillNullCommandIT extends PPLIntegTestCase { + @Override + public void init() throws IOException { + loadIndex(Index.CALCS); + } + + @Test + public void testFillNullSameValueOneField() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fields str2, num0 | fillnull with -1 in num0", TEST_INDEX_CALCS)); + verifyDataRows( + result, + rows("one", 12.3), + rows("two", -12.3), + rows("three", 15.7), + rows(null, -15.7), + rows("five", 3.5), + rows("six", -3.5), + rows(null, 0), + rows("eight", -1), + rows("nine", 10), + rows("ten", -1), + rows("eleven", -1), + rows("twelve", -1), + rows(null, -1), + rows("fourteen", -1), + rows("fifteen", -1), + rows("sixteen", -1), + rows(null, -1)); + } + + @Test + public void testFillNullSameValueTwoFields() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fields num0, num2 | fillnull with -1 in num0,num2", TEST_INDEX_CALCS)); + verifyDataRows( + result, + rows(12.3, 17.86), + rows(-12.3, 16.73), + rows(15.7, -1), + rows(-15.7, 8.51), + rows(3.5, 6.46), + rows(-3.5, 8.98), + rows(0, 11.69), + rows(-1, 17.25), + rows(10, -1), + rows(-1, 11.5), + rows(-1, 6.8), + rows(-1, 3.79), + rows(-1, -1), + rows(-1, 13.04), + rows(-1, -1), + rows(-1, 10.98), + rows(-1, 7.87)); + } + + @Test + public void testFillNullVariousValuesOneField() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fields str2, num0 | fillnull using num0 = -1", TEST_INDEX_CALCS)); + verifyDataRows( + result, + rows("one", 12.3), + rows("two", -12.3), + rows("three", 15.7), + rows(null, -15.7), + rows("five", 3.5), + rows("six", -3.5), + rows(null, 0), + rows("eight", -1), + rows("nine", 10), + rows("ten", -1), + rows("eleven", -1), + rows("twelve", -1), + rows(null, -1), + rows("fourteen", -1), + rows("fifteen", -1), + rows("sixteen", -1), + rows(null, -1)); + } + + @Test + public void testFillNullVariousValuesTwoFields() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | fields num0, num2 | fillnull using num0 = -1, num2 = -2", + TEST_INDEX_CALCS)); + verifyDataRows( + result, + rows(12.3, 17.86), + rows(-12.3, 16.73), + rows(15.7, -2), + rows(-15.7, 8.51), + rows(3.5, 6.46), + rows(-3.5, 8.98), + rows(0, 11.69), + rows(-1, 17.25), + rows(10, -2), + rows(-1, 11.5), + rows(-1, 6.8), + rows(-1, 3.79), + rows(-1, -2), + rows(-1, 13.04), + rows(-1, -2), + rows(-1, 10.98), + rows(-1, 7.87)); + } +} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.json new file mode 100644 index 0000000000..fe3a9cf820 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_fillnull_push.json @@ -0,0 +1,36 @@ +{ + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[age, balance]" + }, + "children": [ + { + "name":"EvalOperator", + "description": { + "expressions": { + "balance": "ifnull(balance, -1)", + "age":"ifnull(age, -1)" + } + }, + "children": [ + { + "name": "ProjectOperator", + "description": { + "fields": "[age, balance]" + }, + "children": [ + { + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_account, sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"balance\",\"age\"],\"excludes\":[]}}, needClean=true, searchDone=false, pitId=null, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + }, + "children": [] + } + ] + } + ] + } + ] + } +} \ No newline at end of file diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java index 943953d416..04f7d27b61 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -417,4 +417,16 @@ public void testCanParseTimestampdiffFunction() { new PPLSyntaxParser() .parse("SOURCE=test | eval k = TIMESTAMPDIFF(WEEK,'2003-01-02','2003-01-02')")); } + + @Test + public void testCanParseFillNullSameValue() { + assertNotNull(new PPLSyntaxParser().parse("SOURCE=test | fillnull with 0 in a")); + assertNotNull(new PPLSyntaxParser().parse("SOURCE=test | fillnull with 0 in a,b")); + } + + @Test + public void testCanParseFillNullVariousValues() { + assertNotNull(new PPLSyntaxParser().parse("SOURCE=test | fillnull using a = 0")); + assertNotNull(new PPLSyntaxParser().parse("SOURCE=test | fillnull using a = 0, b = 1")); + } }