filter(ticdc): Remove useless code about MySQLReplicationRules (#10507)

ref #10457
pingcap · Jan 23, 2024 · 7a9af98 · 7a9af98
1 parent ac5fd9a
commit 7a9af98
Show file tree

Hide file tree

Showing 9 changed files with 10 additions and 182 deletions.
diff --git a/cdc/api/v2/model.go b/cdc/api/v2/model.go
@@ -19,7 +19,6 @@ import (
 
 	"github.com/pingcap/errors"
 	bf "github.com/pingcap/tidb-tools/pkg/binlog-filter"
-	filter "github.com/pingcap/tidb/pkg/util/table-filter"
 	"github.com/pingcap/tiflow/cdc/model"
 	"github.com/pingcap/tiflow/pkg/config"
 	cerror "github.com/pingcap/tiflow/pkg/errors"
@@ -234,30 +233,6 @@ func (c *ReplicaConfig) toInternalReplicaConfigWithOriginConfig(
 	res.BDRMode = c.BDRMode
 
 	if c.Filter != nil {
-		var mySQLReplicationRules *filter.MySQLReplicationRules
-		if c.Filter.MySQLReplicationRules != nil {
-			mySQLReplicationRules = &filter.MySQLReplicationRules{}
-			mySQLReplicationRules.DoDBs = c.Filter.DoDBs
-			mySQLReplicationRules.IgnoreDBs = c.Filter.IgnoreDBs
-			if c.Filter.MySQLReplicationRules.DoTables != nil {
-				for _, tbl := range c.Filter.MySQLReplicationRules.DoTables {
-					mySQLReplicationRules.DoTables = append(mySQLReplicationRules.DoTables,
-						&filter.Table{
-							Schema: tbl.Schema,
-							Name:   tbl.Name,
-						})
-				}
-			}
-			if c.Filter.MySQLReplicationRules.IgnoreTables != nil {
-				for _, tbl := range c.Filter.MySQLReplicationRules.IgnoreTables {
-					mySQLReplicationRules.IgnoreTables = append(mySQLReplicationRules.IgnoreTables,
-						&filter.Table{
-							Schema: tbl.Schema,
-							Name:   tbl.Name,
-						})
-				}
-			}
-		}
 		var efs []*config.EventFilterRule
 		if len(c.Filter.EventFilters) != 0 {
 			efs = make([]*config.EventFilterRule, len(c.Filter.EventFilters))
@@ -266,10 +241,9 @@ func (c *ReplicaConfig) toInternalReplicaConfigWithOriginConfig(
 			}
 		}
 		res.Filter = &config.FilterConfig{
-			Rules:                 c.Filter.Rules,
-			MySQLReplicationRules: mySQLReplicationRules,
-			IgnoreTxnStartTs:      c.Filter.IgnoreTxnStartTs,
-			EventFilters:          efs,
+			Rules:            c.Filter.Rules,
+			IgnoreTxnStartTs: c.Filter.IgnoreTxnStartTs,
+			EventFilters:     efs,
 		}
 	}
 	if c.Consistent != nil {
@@ -556,31 +530,6 @@ func ToAPIReplicaConfig(c *config.ReplicaConfig) *ReplicaConfig {
 	}
 
 	if cloned.Filter != nil {
-		var mySQLReplicationRules *MySQLReplicationRules
-		if c.Filter.MySQLReplicationRules != nil {
-			mySQLReplicationRules = &MySQLReplicationRules{}
-			mySQLReplicationRules.DoDBs = c.Filter.DoDBs
-			mySQLReplicationRules.IgnoreDBs = c.Filter.IgnoreDBs
-			if c.Filter.MySQLReplicationRules.DoTables != nil {
-				for _, tbl := range c.Filter.MySQLReplicationRules.DoTables {
-					mySQLReplicationRules.DoTables = append(mySQLReplicationRules.DoTables,
-						&Table{
-							Schema: tbl.Schema,
-							Name:   tbl.Name,
-						})
-				}
-			}
-			if c.Filter.MySQLReplicationRules.IgnoreTables != nil {
-				for _, tbl := range c.Filter.MySQLReplicationRules.IgnoreTables {
-					mySQLReplicationRules.IgnoreTables = append(mySQLReplicationRules.IgnoreTables,
-						&Table{
-							Schema: tbl.Schema,
-							Name:   tbl.Name,
-						})
-				}
-			}
-		}
-
 		var efs []EventFilterRule
 		if len(c.Filter.EventFilters) != 0 {
 			efs = make([]EventFilterRule, len(c.Filter.EventFilters))
@@ -590,10 +539,9 @@ func ToAPIReplicaConfig(c *config.ReplicaConfig) *ReplicaConfig {
 		}
 
 		res.Filter = &FilterConfig{
-			MySQLReplicationRules: mySQLReplicationRules,
-			Rules:                 cloned.Filter.Rules,
-			IgnoreTxnStartTs:      cloned.Filter.IgnoreTxnStartTs,
-			EventFilters:          efs,
+			Rules:            cloned.Filter.Rules,
+			IgnoreTxnStartTs: cloned.Filter.IgnoreTxnStartTs,
+			EventFilters:     efs,
 		}
 	}
 	if cloned.Sink != nil {
@@ -864,7 +812,6 @@ func GetDefaultReplicaConfig() *ReplicaConfig {
 // FilterConfig represents filter config for a changefeed
 // This is a duplicate of config.FilterConfig
 type FilterConfig struct {
-	*MySQLReplicationRules
 	Rules            []string          `json:"rules,omitempty"`
 	IgnoreTxnStartTs []uint64          `json:"ignore_txn_start_ts,omitempty"`
 	EventFilters     []EventFilterRule `json:"event_filters,omitempty"`
@@ -932,19 +879,6 @@ func ToAPIEventFilterRule(er *config.EventFilterRule) EventFilterRule {
 	return res
 }
 
-// MySQLReplicationRules is a set of rules based on MySQL's replication tableFilter.
-type MySQLReplicationRules struct {
-	// DoTables is an allowlist of tables.
-	DoTables []*Table `json:"do_tables,omitempty"`
-	// DoDBs is an allowlist of schemas.
-	DoDBs []string `json:"do_dbs,omitempty"`
-
-	// IgnoreTables is a blocklist of tables.
-	IgnoreTables []*Table `json:"ignore_tables,omitempty"`
-	// IgnoreDBs is a blocklist of schemas.
-	IgnoreDBs []string `json:"ignore_dbs,omitempty"`
-}
-
 // Table represents a qualified table name.
 type Table struct {
 	// Schema is the name of the schema (database) containing this table.

diff --git a/cdc/api/v2/model_test.go b/cdc/api/v2/model_test.go
@@ -20,7 +20,6 @@ import (
 	"time"
 
 	bf "github.com/pingcap/tidb-tools/pkg/binlog-filter"
-	filter "github.com/pingcap/tidb/pkg/util/table-filter"
 	"github.com/pingcap/tiflow/cdc/model"
 	"github.com/pingcap/tiflow/pkg/config"
 	"github.com/pingcap/tiflow/pkg/redo"
@@ -137,21 +136,7 @@ func TestToAPIReplicaConfig(t *testing.T) {
 		Storage:           "s3",
 	}
 	cfg.Filter = &config.FilterConfig{
-		Rules: []string{"a", "b", "c"},
-		MySQLReplicationRules: &filter.MySQLReplicationRules{
-			DoTables: []*filter.Table{{
-				Schema: "testdo",
-				Name:   "testgotable",
-			}},
-			DoDBs: []string{"ad", "bdo"},
-			IgnoreTables: []*filter.Table{
-				{
-					Schema: "testignore",
-					Name:   "testaaaingore",
-				},
-			},
-			IgnoreDBs: []string{"aa", "b2"},
-		},
+		Rules:            []string{"a", "b", "c"},
 		IgnoreTxnStartTs: []uint64{1, 2, 3},
 		EventFilters: []*config.EventFilterRule{{
 			Matcher:                  []string{"test.t1", "test.t2"},

diff --git a/cdc/model/changefeed_test.go b/cdc/model/changefeed_test.go
@@ -20,7 +20,6 @@ import (
 	"testing"
 	"time"
 
-	filter "github.com/pingcap/tidb/pkg/util/table-filter"
 	"github.com/pingcap/tiflow/pkg/config"
 	"github.com/pingcap/tiflow/pkg/errors"
 	"github.com/pingcap/tiflow/pkg/util"
@@ -142,34 +141,6 @@ func TestFillV1(t *testing.T) {
     "config":{
         "case-sensitive":true,
         "filter":{
-            "do-tables":[
-                {
-                    "db-name":"test",
-                    "tbl-name":"tbl1"
-                },
-                {
-                    "db-name":"test",
-                    "tbl-name":"tbl2"
-                }
-            ],
-            "do-dbs":[
-                "test1",
-                "sys1"
-            ],
-            "ignore-tables":[
-                {
-                    "db-name":"test",
-                    "tbl-name":"tbl3"
-                },
-                {
-                    "db-name":"test",
-                    "tbl-name":"tbl4"
-                }
-            ],
-            "ignore-dbs":[
-                "test",
-                "sys"
-            ],
             "ignore-txn-start-ts":[
                 1,
                 2
@@ -206,24 +177,6 @@ func TestFillV1(t *testing.T) {
 		Config: &config.ReplicaConfig{
 			CaseSensitive: true,
 			Filter: &config.FilterConfig{
-				MySQLReplicationRules: &filter.MySQLReplicationRules{
-					DoTables: []*filter.Table{{
-						Schema: "test",
-						Name:   "tbl1",
-					}, {
-						Schema: "test",
-						Name:   "tbl2",
-					}},
-					DoDBs: []string{"test1", "sys1"},
-					IgnoreTables: []*filter.Table{{
-						Schema: "test",
-						Name:   "tbl3",
-					}, {
-						Schema: "test",
-						Name:   "tbl4",
-					}},
-					IgnoreDBs: []string{"test", "sys"},
-				},
 				IgnoreTxnStartTs: []uint64{1, 2},
 			},
 			Mounter: &config.MounterConfig{

diff --git a/pkg/cmd/cli/cli_changefeed_create_test.go b/pkg/cmd/cli/cli_changefeed_create_test.go
@@ -74,11 +74,6 @@ func TestTomlFileToApiModel(t *testing.T) {
 	content := `
 	[filter]
 	rules = ['*.*', '!test.*']
-    ignore-dbs = ["a", "b"]
-    do-dbs = ["c", "d"]
-    [[filter.ignore-tables]]
-    db-name = "demo-db"
-    tbl-name = "tbl"
 `
 	err := os.WriteFile(path, []byte(content), 0o644)
 	require.Nil(t, err)

diff --git a/pkg/config/filter.go b/pkg/config/filter.go
@@ -15,13 +15,11 @@ package config
 
 import (
 	bf "github.com/pingcap/tidb-tools/pkg/binlog-filter"
-	filter "github.com/pingcap/tidb/pkg/util/table-filter"
 )
 
 // FilterConfig represents filter config for a changefeed
 type FilterConfig struct {
-	Rules []string `toml:"rules" json:"rules"`
-	*filter.MySQLReplicationRules
+	Rules            []string           `toml:"rules" json:"rules"`
 	IgnoreTxnStartTs []uint64           `toml:"ignore-txn-start-ts" json:"ignore-txn-start-ts"`
 	EventFilters     []*EventFilterRule `toml:"event-filters" json:"event-filters"`
 }

diff --git a/pkg/filter/utils.go b/pkg/filter/utils.go
@@ -36,9 +36,7 @@ func isSysSchema(db string) bool {
 func VerifyTableRules(cfg *config.FilterConfig) (tfilter.Filter, error) {
 	var f tfilter.Filter
 	var err error
-	if len(cfg.Rules) == 0 && cfg.MySQLReplicationRules != nil {
-		f, err = tfilter.ParseMySQLReplicationRules(cfg.MySQLReplicationRules)
-	} else {
+	if len(cfg.Rules) != 0 {
 		rules := cfg.Rules
 		if len(rules) == 0 {
 			rules = []string{"*.*"}

diff --git a/pkg/migrate/migrate_test.go b/pkg/migrate/migrate_test.go
@@ -25,7 +25,6 @@ import (
 	"time"
 
 	"github.com/pingcap/errors"
-	filter "github.com/pingcap/tidb/pkg/util/table-filter"
 	"github.com/pingcap/tiflow/cdc/model"
 	"github.com/pingcap/tiflow/pkg/config"
 	"github.com/pingcap/tiflow/pkg/etcd"
@@ -112,21 +111,7 @@ func TestMigration(t *testing.T) {
 		Storage:           "s3",
 	}
 	cfg.Filter = &config.FilterConfig{
-		Rules: []string{"a", "b", "c"},
-		MySQLReplicationRules: &filter.MySQLReplicationRules{
-			DoTables: []*filter.Table{{
-				Schema: "testdo",
-				Name:   "testgotable",
-			}},
-			DoDBs: []string{"ad", "bdo"},
-			IgnoreTables: []*filter.Table{
-				{
-					Schema: "testignore",
-					Name:   "testaaaingore",
-				},
-			},
-			IgnoreDBs: []string{"aa", "b2"},
-		},
+		Rules:            []string{"a", "b", "c"},
 		IgnoreTxnStartTs: []uint64{1, 2, 3},
 	}
 	info3 := model.ChangeFeedInfo{

diff --git a/tests/integration_tests/api_v2/cases.go b/tests/integration_tests/api_v2/cases.go
@@ -38,12 +38,6 @@ var customReplicaConfig = &ReplicaConfig{
 	SyncPointInterval:     util.AddressOf(JSONDuration{duration: 10 * time.Minute}),
 	SyncPointRetention:    util.AddressOf(JSONDuration{duration: 24 * time.Hour}),
 	Filter: &FilterConfig{
-		MySQLReplicationRules: &MySQLReplicationRules{
-			DoTables:     []*Table{{"a", "b"}, {"c", "d"}},
-			DoDBs:        []string{"a", "c"},
-			IgnoreTables: []*Table{{"d", "e"}, {"f", "g"}},
-			IgnoreDBs:    []string{"d", "x"},
-		},
 		IgnoreTxnStartTs: []uint64{1, 2, 3},
 		EventFilters: []EventFilterRule{{
 			Matcher:                  []string{"test.worker"},

diff --git a/tests/integration_tests/api_v2/model.go b/tests/integration_tests/api_v2/model.go
@@ -185,7 +185,6 @@ type ReplicaConfig struct {
 // FilterConfig represents filter config for a changefeed
 // This is a duplicate of config.FilterConfig
 type FilterConfig struct {
-	*MySQLReplicationRules
 	Rules            []string          `json:"rules,omitempty"`
 	IgnoreTxnStartTs []uint64          `json:"ignore_txn_start_ts,omitempty"`
 	EventFilters     []EventFilterRule `json:"event_filters,omitempty"`
@@ -209,19 +208,6 @@ type EventFilterRule struct {
 	IgnoreDeleteValueExpr    string `json:"ignore_delete_value_expr"`
 }
 
-// MySQLReplicationRules is a set of rules based on MySQL's replication tableFilter.
-type MySQLReplicationRules struct {
-	// DoTables is an allowlist of tables.
-	DoTables []*Table `json:"do_tables,omitempty"`
-	// DoDBs is an allowlist of schemas.
-	DoDBs []string `json:"do_dbs,omitempty"`
-
-	// IgnoreTables is a blocklist of tables.
-	IgnoreTables []*Table `json:"ignore_tables,omitempty"`
-	// IgnoreDBs is a blocklist of schemas.
-	IgnoreDBs []string `json:"ignore_dbs,omitempty"`
-}
-
 // Table represents a qualified table name.
 type Table struct {
 	// Schema is the name of the schema (database) containing this table.