[feature] support not equals and like and not like filter push down (a…

…pache#219)
bingquanzhao · Jul 29, 2024 · 3901055 · 3901055
1 parent 5a3177b
commit 3901055
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 3 deletions.
diff --git a/spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/Utils.scala b/spark-doris-connector/src/main/scala/org/apache/doris/spark/sql/Utils.scala
@@ -51,6 +51,7 @@ private[spark] object Utils {
   def compileFilter(filter: Filter, dialect: JdbcDialect, inValueLengthLimit: Int): Option[String] = {
     Option(filter match {
       case EqualTo(attribute, value) => s"${quote(attribute)} = ${compileValue(value)}"
+      case Not(EqualTo(attribute, value)) => s"${quote(attribute)} != ${compileValue(value)}"
       case GreaterThan(attribute, value) => s"${quote(attribute)} > ${compileValue(value)}"
       case GreaterThanOrEqual(attribute, value) => s"${quote(attribute)} >= ${compileValue(value)}"
       case LessThan(attribute, value) => s"${quote(attribute)} < ${compileValue(value)}"
@@ -83,6 +84,12 @@ private[spark] object Utils {
         } else {
           null
         }
+      case StringContains(attribute, value) => s"${quote(attribute)} like '%$value%'"
+      case Not(StringContains(attribute, value)) => s"${quote(attribute)} not like '%$value%'"
+      case StringEndsWith(attribute, value) => s"${quote(attribute)} like '%$value'"
+      case Not(StringEndsWith(attribute, value)) => s"${quote(attribute)} not like '%$value'"
+      case StringStartsWith(attribute, value) => s"${quote(attribute)} like '$value%'"
+      case Not(StringStartsWith(attribute, value)) => s"${quote(attribute)} not like '$value%'"
       case _ => null
     })
   }

diff --git a/spark-doris-connector/src/test/scala/org/apache/doris/spark/sql/TestUtils.scala b/spark-doris-connector/src/test/scala/org/apache/doris/spark/sql/TestUtils.scala
@@ -34,29 +34,38 @@ class TestUtils extends ExpectedExceptionTest {
     val inValueLengthLimit = 5
 
     val equalFilter = EqualTo("left", 5)
+    val notEqualFilter = Not(EqualTo("left", 5))
     val greaterThanFilter = GreaterThan("left", 5)
     val greaterThanOrEqualFilter = GreaterThanOrEqual("left", 5)
     val lessThanFilter = LessThan("left", 5)
     val lessThanOrEqualFilter = LessThanOrEqual("left", 5)
     val validInFilter = In("left", Array(1, 2, 3, 4))
     val emptyInFilter = In("left", Array.empty)
     val invalidInFilter = In("left", Array(1, 2, 3, 4, 5))
+    val notInFilter = Not(In("left", Array(1, 2, 3)))
     val isNullFilter = IsNull("left")
     val isNotNullFilter = IsNotNull("left")
-    val notSupportFilter = StringContains("left", "right")
     val validAndFilter = And(equalFilter, greaterThanFilter)
-    val invalidAndFilter = And(equalFilter, notSupportFilter)
+    val invalidAndFilter = And(equalFilter, invalidInFilter)
     val validOrFilter = Or(equalFilter, greaterThanFilter)
-    val invalidOrFilter = Or(equalFilter, notSupportFilter)
+    val invalidOrFilter = Or(equalFilter, invalidInFilter)
+    val stringContainsFilter = StringContains("left", "right")
+    val notStringContainsFilter = Not(StringContains("left", "right"))
+    val stringEndsWithFilter = StringEndsWith("left", "right")
+    val notStringEndsWithFilter = Not(StringEndsWith("left", "right"))
+    val stringStartsWithFilter = StringStartsWith("left", "right")
+    val notStringStartsWithFilter = Not(StringStartsWith("left", "right"))
 
     Assert.assertEquals("`left` = 5", Utils.compileFilter(equalFilter, dialect, inValueLengthLimit).get)
+    Assert.assertEquals("`left` != 5", Utils.compileFilter(notEqualFilter, dialect, inValueLengthLimit).get)
     Assert.assertEquals("`left` > 5", Utils.compileFilter(greaterThanFilter, dialect, inValueLengthLimit).get)
     Assert.assertEquals("`left` >= 5", Utils.compileFilter(greaterThanOrEqualFilter, dialect, inValueLengthLimit).get)
     Assert.assertEquals("`left` < 5", Utils.compileFilter(lessThanFilter, dialect, inValueLengthLimit).get)
     Assert.assertEquals("`left` <= 5", Utils.compileFilter(lessThanOrEqualFilter, dialect, inValueLengthLimit).get)
     Assert.assertEquals("`left` in (1, 2, 3, 4)", Utils.compileFilter(validInFilter, dialect, inValueLengthLimit).get)
     Assert.assertTrue(Utils.compileFilter(emptyInFilter, dialect, inValueLengthLimit).isEmpty)
     Assert.assertTrue(Utils.compileFilter(invalidInFilter, dialect, inValueLengthLimit).isEmpty)
+    Assert.assertEquals("`left` not in (1, 2, 3)", Utils.compileFilter(notInFilter, dialect, inValueLengthLimit).get)
     Assert.assertEquals("`left` is null", Utils.compileFilter(isNullFilter, dialect, inValueLengthLimit).get)
     Assert.assertEquals("`left` is not null", Utils.compileFilter(isNotNullFilter, dialect, inValueLengthLimit).get)
     Assert.assertEquals("(`left` = 5) and (`left` > 5)",
@@ -65,6 +74,12 @@ class TestUtils extends ExpectedExceptionTest {
     Assert.assertEquals("(`left` = 5) or (`left` > 5)",
       Utils.compileFilter(validOrFilter, dialect, inValueLengthLimit).get)
     Assert.assertTrue(Utils.compileFilter(invalidOrFilter, dialect, inValueLengthLimit).isEmpty)
+    Assert.assertEquals("`left` like '%right%'", Utils.compileFilter(stringContainsFilter, dialect, inValueLengthLimit).get)
+    Assert.assertEquals("`left` not like '%right%'", Utils.compileFilter(notStringContainsFilter, dialect, inValueLengthLimit).get)
+    Assert.assertEquals("`left` like '%right'", Utils.compileFilter(stringEndsWithFilter, dialect, inValueLengthLimit).get)
+    Assert.assertEquals("`left` not like '%right'", Utils.compileFilter(notStringEndsWithFilter, dialect, inValueLengthLimit).get)
+    Assert.assertEquals("`left` like 'right%'", Utils.compileFilter(stringStartsWithFilter, dialect, inValueLengthLimit).get)
+    Assert.assertEquals("`left` not like 'right%'", Utils.compileFilter(notStringStartsWithFilter, dialect, inValueLengthLimit).get)
   }
 
   @Test