feat(test): change action to open all optimizer and close optimizer

.github/actions/confWriter/action.yml

@@ -103,47 +103,53 @@ runs:
                 echo "$RUNNER_OS is not supported"
                 exit 1
               fi
-        - if: inputs.Push-Down=='true'
+        - if: inputs.Open-Optimizer=='true'
           name: Change push_down
           shell: bash
           run: |
               if [ "$RUNNER_OS" == "Linux" ]; then
-                sudo sed -i 's/enablePushDown=false/enablePushDown=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
-                sed -i 's/FilterPushDownRule=off/FilterPushDownRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                  sudo sed -i 's/enablePushDown=false/enablePushDown=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                  sed -i 's/Rule=off/Rule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
               elif [ "$RUNNER_OS" == "Windows" ]; then
-                sed -i 's/enablePushDown=false/enablePushDown=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
-                sed -i 's/FilterPushDownRule=off/FilterPushDownRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                  sed -i 's/enablePushDown=false/enablePushDown=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                  sed -i 's/Rule=off/Rule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
               elif [ "$RUNNER_OS" == "macOS" ]; then
                 sudo sed -i '' 's/enablePushDown=false/enablePushDown=true/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
-                sed -i '' 's/FilterPushDownRule=off/FilterPushDownRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                sed -i '' 's/Rule=off/Rule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
               else
                 echo "$RUNNER_OS is not supported"
                 exit 1
               fi
-        - if: inputs.Set-Key-Range-Test-Policy=='true'
-          name: Change KeyRangeTestPolicy
+        - if: inputs.Close-Optimizer=='true'
+          name: Close-All-Optimizer
           shell: bash
           run: |
               if [ "$RUNNER_OS" == "Linux" ]; then
-                  sudo sed -i 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                sudo sed -i 's/Rule=on/Rule=off/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                sudo sed -i 's/enablePushDown=true/enablePushDown=false/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                sudo sed -i 's/ColumnPruningRule=off/ColumnPruningRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
               elif [ "$RUNNER_OS" == "Windows" ]; then
-                sed -i 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                sed -i 's/Rule=on/Rule=off/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                sed -i 's/enablePushDown=true/enablePushDown=false/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                sed -i 's/ColumnPruningRule=off/ColumnPruningRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
               elif [ "$RUNNER_OS" == "macOS" ]; then
-                  sudo sed -i '' 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                sudo sed -i '' 's/Rule=on/Rule=off/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                sudo sed -i '' 's/enablePushDown=true/enablePushDown=false/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                sudo sed -i '' 's/ColumnPruningRule=off/ColumnPruningRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
               else
-                  echo "$RUNNER_OS is not supported"
-                  exit 1
+                echo "$RUNNER_OS is not supported"
+                exit 1
               fi
-        - if: inputs.Set-Filter-Fragment-OFF=='true'
-          name: Set FragmentPruningByFilterRule OFF
+        - if: inputs.Set-Key-Range-Test-Policy=='true'
+          name: Change KeyRangeTestPolicy
           shell: bash
           run: |
               if [ "$RUNNER_OS" == "Linux" ]; then
-                  sudo sed -i 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                  sudo sed -i 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
               elif [ "$RUNNER_OS" == "Windows" ]; then
-                sed -i 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                sed -i 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
               elif [ "$RUNNER_OS" == "macOS" ]; then
-                  sudo sed -i '' 's/ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on/ruleBasedOptimizer=NotFilterRemoveRule=on/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
+                  sudo sed -i '' 's/policyClassName=cn.edu.tsinghua.iginx.policy.naive.NaivePolicy/policyClassName=cn.edu.tsinghua.iginx.policy.test.KeyRangeTestPolicy/g' ${{ inputs.Root-Dir-Path }}/core/target/iginx-core-${VERSION}/conf/config.properties
               else
                   echo "$RUNNER_OS is not supported"
                   exit 1

.github/workflows/DB-CE.yml

@@ -91,7 +91,6 @@ jobs:
               shell: bash
               run: |
                   mvn test -q -Dtest=${FUNCTEST} -DfailIfNoTests=false -P-format
-
             # 第 2 阶段测试开始==========================================
             - name: Prepare CapExp environment oriNoDataExpNoData
               uses: ./.github/actions/capacityExpansionUnionTest

.github/workflows/full-test-suite.yml

@@ -19,7 +19,7 @@ jobs:
     standalone-test:
         uses: ./.github/workflows/standalone-test.yml
     standalone-test-pushdown:
-        uses: ./.github/workflows/standalone-test-pushdown.yml
+        uses: ./.github/workflows/standalone-test-no-optimizer.yml
     db-ce:
         uses: ./.github/workflows/DB-CE.yml
     remote-test:

.github/workflows/standalone-test-pushdown.yml → .github/workflows/standalone-test-no-optimizer.yml

@@ -1,4 +1,4 @@
-name: "Union Database Test With Push Down"
+name: "Union Database Test No Optimizer"
 
 on:
     workflow_call:
@@ -72,8 +72,7 @@ jobs:
               uses: ./.github/actions/confWriter
               with:
                   DB-name: ${{ matrix.DB-name }}
-                  Push-Down: "true"
-                  Set-Filter-Fragment-OFF: "true"
+                  Close-Optimizer: "true"
                   Metadata: ${{ matrix.metadata }}
 
             - name: Start IGinX

.github/workflows/standard-test-suite.yml

@@ -23,8 +23,8 @@ jobs:
         uses: ./.github/workflows/standalone-test.yml
         with:
             metadata-matrix: '["zookeeper"]'
-    standalone-test-pushdown:
-        uses: ./.github/workflows/standalone-test-pushdown.yml
+    standalone-test-no-optimizer:
+        uses: ./.github/workflows/standalone-test-no-optimizer.yml
         with:
             metadata-matrix: '["zookeeper"]'
     db-ce:
@@ -42,8 +42,8 @@ jobs:
         with:
             os-matrix: '["ubuntu-latest"]'
             metadata-matrix: '["zookeeper"]'
-    tpc-h-regression-test-pushdown:
-        uses: ./.github/workflows/tpc-h-pushdown.yml
+    tpc-h-regression-test-no-optimizer:
+        uses: ./.github/workflows/tpc-h-no-optimizer.yml
         with:
             os-matrix: '["ubuntu-latest"]'
             metadata-matrix: '["zookeeper"]'

.github/workflows/tpc-h-pushdown.yml → .github/workflows/tpc-h-no-optimizer.yml

@@ -1,4 +1,4 @@
-name: "TPC-H Regression Test With Push Down"
+name: "TPC-H Regression Test Without Optimizer"
 
 on:
     workflow_call:
@@ -30,7 +30,7 @@ on:
                 default: '["IoTDB12", "InfluxDB", "FileSystem", "PostgreSQL", "Redis", "MongoDB", "MySQL"]'
 
 jobs:
-    TPC-H-Test-Push_Down:
+    TPC-H-Test-Without-Optimizer:
         timeout-minutes: 35
         strategy:
             fail-fast: false
@@ -118,8 +118,7 @@ jobs:
               uses: ./.github/actions/confWriter
               with:
                   DB-name: ${{ matrix.DB-name }}
-                  Push-Down: "true"
-                  Set-Filter-Fragment-OFF: "true"
+                  Close-Optimizer: "true"
                   Metadata: ${{ matrix.metadata }}
                   Root-Dir-Path: "IGinX"
 
@@ -131,8 +130,7 @@ jobs:
               uses: ./.github/actions/confWriter
               with:
                   DB-name: ${{ matrix.DB-name }}
-                  Push-Down: "true"
-                  Set-Filter-Fragment-OFF: "true"
+                  Close-Optimizer: "true"
                   Metadata: ${{ matrix.metadata }}
 
             - name: Insert Data into DB

.github/workflows/tpc-h.yml

@@ -27,7 +27,7 @@ on:
                 description: "The database to run the test on"
                 type: string
                 required: false
-                default: '["IoTDB12", "InfluxDB", "FileSystem", "PostgreSQL", "Redis", "MongoDB", "MySQL"]'
+                default: '["InfluxDB", "FileSystem", "PostgreSQL", "Redis", "MongoDB", "MySQL"]'
 
 jobs:
     TPC-H-Test:
@@ -118,21 +118,14 @@ jobs:
               uses: ./.github/actions/confWriter
               with:
                   DB-name: ${{ matrix.DB-name }}
-                  Set-Filter-Fragment-OFF: "true"
+                  Open-Optimizer: "true"
                   Metadata: ${{ matrix.metadata }}
                   Root-Dir-Path: "IGinX"
 
             - name: Install New IGinX with Maven
               shell: bash
               run: mvn clean package -DskipTests -P-format -q
 
-            - name: Change New IGinX Config
-              uses: ./.github/actions/confWriter
-              with:
-                  DB-name: ${{ matrix.DB-name }}
-                  Set-Filter-Fragment-OFF: "true"
-                  Metadata: ${{ matrix.metadata }}
-
             - name: Insert Data into DB
               uses: ./.github/actions/tpchDataWriter
               with:

conf/config.properties

@@ -68,8 +68,9 @@ maxCachedPhysicalTaskPerStorage=500
 queryOptimizer=rbo
 
 # 优化器规则
-ruleBasedOptimizer=NotFilterRemoveRule=on,FragmentPruningByFilterRule=on,ColumnPruningRule=on,ConstantPropagationRule=on,DistinctEliminateRule=on,\
-  ConstantFoldingRule=on,FilterPushDownRule=off,JoinFactorizationRule=on,SetTransformPushDownPathUnionJoinRule=off
+ruleBasedOptimizer=NotFilterRemoveRule=on,ColumnPruningRule=on,ConstantPropagationRule=on,DistinctEliminateRule=on,\
+  ConstantFoldingRule=on,FilterPushDownRule=on,JoinFactorizationRule=on,SetTransformPushDownPathUnionJoinRule=on,InFilterTransformRule=on,\
+  OuterJoinEliminateRule=on
 
 # ParallelFilter触发行数
 parallelFilterThreshold=10000
@@ -175,7 +176,7 @@ fragmentCacheThreshold=131072
 ### 执行层配置
 ##########################
 
-enablePushDown=false
+enablePushDown=true
 
 useStreamExecutor=false
 

dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/service/rpc/client/ClientObjectMappingUtils.java

@@ -38,6 +38,7 @@
 import cn.edu.tsinghua.iginx.utils.Pair;
 import java.nio.ByteBuffer;
 import java.util.*;
+import java.util.stream.Collectors;
 import javax.annotation.Nullable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -84,11 +85,24 @@ public static RawFilter toRawFilter(Filter filter) {
         return toRawFilter((BoolFilter) filter);
       case Path:
         return toRawFilter((PathFilter) filter);
+      case In:
+        return toRawFilter((InFilter) filter);
       default:
         throw new UnsupportedOperationException("unsupported filter type: " + filter.getType());
     }
   }
 
+  private static RawFilter toRawFilter(InFilter filter) {
+    RawFilter raw = new RawFilter(RawFilterType.In);
+    raw.setPath(filter.getPath());
+    raw.setArray(
+        filter.getValues().stream()
+            .map(ClientObjectMappingUtils::toRawValue)
+            .collect(Collectors.toSet()));
+    raw.setInOp(toRawFilterInOp(filter.getInOp()));
+    return raw;
+  }
+
   private static RawFilter toRawFilter(AndFilter filter) {
     RawFilter raw = new RawFilter(RawFilterType.And);
     if (filter.getChildren().isEmpty()) {
@@ -146,6 +160,21 @@ private static RawFilter toRawFilter(BoolFilter filter) {
     return raw;
   }
 
+  private static RawFilterInOp toRawFilterInOp(InFilter.InOp inOp) {
+    switch (inOp) {
+      case IN_OR:
+        return RawFilterInOp.IN;
+      case NOT_IN_OR:
+        return RawFilterInOp.NOT_IN;
+      case IN_AND:
+        return RawFilterInOp.IN_AND;
+      case NOT_IN_AND:
+        return RawFilterInOp.NOT_IN_AND;
+      default:
+        throw new UnsupportedOperationException("unsupported in op: " + inOp);
+    }
+  }
+
   private static RawFilterOp toRawFilterOp(Op op) {
     switch (op) {
       case L:

dataSource/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/service/rpc/server/ServerObjectMappingUtils.java

@@ -19,6 +19,7 @@
  */
 package cn.edu.tsinghua.iginx.filesystem.service.rpc.server;
 
+import static cn.edu.tsinghua.iginx.engine.shared.operator.filter.FilterType.In;
 import static cn.edu.tsinghua.iginx.engine.shared.operator.filter.Op.*;
 
 import cn.edu.tsinghua.iginx.engine.physical.exception.PhysicalException;
@@ -67,11 +68,22 @@ public static Filter resolveRawFilter(RawFilter filter) {
         return resolveRawBoolFilter(filter);
       case Path:
         return resolveRawPathFilter(filter);
+      case In:
+        return resolveRawInFilter(filter);
       default:
         throw new UnsupportedOperationException("unsupported filter type: " + filter.getType());
     }
   }
 
+  private static Filter resolveRawInFilter(RawFilter filter) {
+    return new InFilter(
+        filter.getPath(),
+        resolveRawFilterInOp(filter.inOp),
+        filter.getArray().stream()
+            .map(ServerObjectMappingUtils::resolveRawValue)
+            .collect(Collectors.toList()));
+  }
+
   private static Filter resolveRawAndFilter(RawFilter andFilter) {
     List<Filter> filters = new ArrayList<>();
     for (RawFilter f : andFilter.getChildren()) {
@@ -148,6 +160,21 @@ private static Op resolveRawFilterOp(RawFilterOp op) {
     }
   }
 
+  private static InFilter.InOp resolveRawFilterInOp(RawFilterInOp rawFilterInOp) {
+    switch (rawFilterInOp) {
+      case IN_AND:
+        return InFilter.InOp.IN_AND;
+      case IN:
+        return InFilter.InOp.IN_OR;
+      case NOT_IN_AND:
+        return InFilter.InOp.NOT_IN_AND;
+      case NOT_IN:
+        return InFilter.InOp.NOT_IN_OR;
+      default:
+        throw new UnsupportedOperationException("unsupported filter in op: " + rawFilterInOp);
+    }
+  }
+
   private static Value resolveRawValue(RawValue RawValue) {
     Value value = null;
     switch (RawValue.getDataType()) {

dataSource/filesystem/src/main/thrift/core.thrift

@@ -36,6 +36,7 @@ enum RawFilterType {
     And,
     Or,
     Not,
+    In
 }
 
 enum RawFilterOp {
@@ -58,6 +59,13 @@ enum RawFilterOp {
     UNKNOWN,
 }
 
+enum RawFilterInOp {
+    IN,
+    NOT_IN,
+    IN_AND,
+    NOT_IN_AND
+}
+
 struct RawValue {
     1: required rpc.DataType dataType
     2: optional bool boolV
@@ -78,6 +86,8 @@ struct RawFilter {
     7: optional string pathB
     8: optional string path
     9: optional RawValue value
+    10: optional RawFilterInOp inOp
+    11: optional set<RawValue> array
 }
 
 struct RawHeader {