Skip to content

Commit

Permalink
Add check constraints check
Browse files Browse the repository at this point in the history
  • Loading branch information
joechenrh committed Dec 3, 2024
1 parent ec82382 commit 9d2c8a6
Show file tree
Hide file tree
Showing 5 changed files with 35 additions and 6 deletions.
3 changes: 3 additions & 0 deletions sync_diff_inspector/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -378,6 +378,8 @@ type Config struct {
// set true if want to compare rows
// set false won't compare rows.
ExportFixSQL bool `toml:"export-fix-sql" json:"export-fix-sql"`
// whether check table constraints.
CheckConstraints bool `toml:"check-constraints" json:"check-constraints"`
// only check table struct without table data.
CheckStructOnly bool `toml:"check-struct-only" json:"check-struct-only"`
// experimental feature: only check table data without table struct
Expand Down Expand Up @@ -420,6 +422,7 @@ func NewConfig() *Config {
fs.StringVar(&cfg.DMTask, "dm-task", "", "identifier of dm task")
fs.IntVar(&cfg.CheckThreadCount, "check-thread-count", 4, "how many goroutines are created to check data")
fs.BoolVar(&cfg.ExportFixSQL, "export-fix-sql", true, "set true if want to compare rows or set to false will only compare checksum")
fs.BoolVar(&cfg.CheckConstraints, "check-constraints", false, "check table constraints.")
fs.BoolVar(&cfg.CheckStructOnly, "check-struct-only", false, "ignore check table's data")
fs.BoolVar(&cfg.SkipNonExistingTable, "skip-non-existing-table", false, "skip validation for tables that don't exist upstream or downstream")
fs.BoolVar(&cfg.CheckDataOnly, "check-data-only", false, "ignore check table's struct")
Expand Down
8 changes: 4 additions & 4 deletions sync_diff_inspector/diff/diff.go
Original file line number Diff line number Diff line change
Expand Up @@ -298,7 +298,7 @@ func (df *Diff) Equal(ctx context.Context) error {
}

// StructEqual compare tables from downstream
func (df *Diff) StructEqual(ctx context.Context) error {
func (df *Diff) StructEqual(ctx context.Context, checkConstraints bool) error {
tables := df.downstream.GetTables()
tableIndex := 0
if df.startRange != nil {
Expand All @@ -308,7 +308,7 @@ func (df *Diff) StructEqual(ctx context.Context) error {
isEqual, isSkip, isAllTableExist := false, true, tables[tableIndex].TableLack
if common.AllTableExist(isAllTableExist) {
var err error
isEqual, isSkip, err = df.compareStruct(ctx, tableIndex)
isEqual, isSkip, err = df.compareStruct(ctx, tableIndex, checkConstraints)
if err != nil {
return errors.Trace(err)
}
Expand All @@ -319,13 +319,13 @@ func (df *Diff) StructEqual(ctx context.Context) error {
return nil
}

func (df *Diff) compareStruct(ctx context.Context, tableIndex int) (isEqual bool, isSkip bool, err error) {
func (df *Diff) compareStruct(ctx context.Context, tableIndex int, checkConstraint bool) (isEqual bool, isSkip bool, err error) {
sourceTableInfos, err := df.upstream.GetSourceStructInfo(ctx, tableIndex)
if err != nil {
return false, true, errors.Trace(err)
}
table := df.downstream.GetTables()[tableIndex]
isEqual, isSkip = utils.CompareStruct(sourceTableInfos, table.Info)
isEqual, isSkip = utils.CompareStruct(sourceTableInfos, table.Info, checkConstraint)
table.IgnoreDataCheck = isSkip
return isEqual, isSkip, nil
}
Expand Down
2 changes: 1 addition & 1 deletion sync_diff_inspector/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ func checkSyncState(ctx context.Context, cfg *config.Config) bool {
defer d.Close()

if !cfg.CheckDataOnly {
err = d.StructEqual(ctx)
err = d.StructEqual(ctx, cfg.CheckConstraints)
if err != nil {
fmt.Printf("An error occurred while comparing table structure: %s, please check log info in %s for full details\n",
err, filepath.Join(cfg.Task.OutputDir, config.LogFileName))
Expand Down
4 changes: 4 additions & 0 deletions sync_diff_inspector/utils/table.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/pingcap/tidb/pkg/parser/ast"
pmodel "github.com/pingcap/tidb/pkg/parser/model"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/collate"
"github.com/pingcap/tidb/pkg/util/dbutil"
Expand All @@ -43,6 +44,9 @@ const (

func init() {
collate.SetNewCollationEnabledForTest(false)

// Enable constraint check
variable.EnableCheckConstraint.Store(true)
}

// addClusteredAnnotation add the `/*T![clustered_index] NONCLUSTERED */` for primary key of create table info
Expand Down
24 changes: 23 additions & 1 deletion sync_diff_inspector/utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -376,7 +376,7 @@ func sameProperties(c1, c2 *model.ColumnInfo) bool {
//
// isEqual : result of comparing tables' columns and indices
// isPanic : the differences of tables' struct can not be ignored. Need to skip data comparing.
func CompareStruct(upstreamTableInfos []*model.TableInfo, downstreamTableInfo *model.TableInfo) (isEqual bool, isPanic bool) {
func CompareStruct(upstreamTableInfos []*model.TableInfo, downstreamTableInfo *model.TableInfo, checkConstraint bool) (isEqual bool, isPanic bool) {
// compare columns
for _, upstreamTableInfo := range upstreamTableInfos {
if len(upstreamTableInfo.Columns) != len(downstreamTableInfo.Columns) {
Expand Down Expand Up @@ -430,6 +430,28 @@ func CompareStruct(upstreamTableInfos []*model.TableInfo, downstreamTableInfo *m
}
}

// compare constraints
if checkConstraint {
downstreamConstraints := downstreamTableInfo.Constraints
sort.Slice(downstreamConstraints, func(i, j int) bool {
return downstreamConstraints[i].Name.L < downstreamConstraints[j].Name.L
})
for _, upstreamTableInfo := range upstreamTableInfos {
upstreamConstraints := upstreamTableInfo.Constraints
if len(upstreamConstraints) != len(downstreamConstraints) {
return false, true
}
sort.Slice(upstreamConstraints, func(i, j int) bool {
return upstreamConstraints[i].Name.L < upstreamConstraints[j].Name.L
})
for i, upConstrain := range upstreamConstraints {
if upConstrain.ExprString != downstreamConstraints[i].ExprString {
return false, true
}
}
}
}

// compare indices
deleteIndicesSet := make(map[string]struct{})
unilateralIndicesSet := make(map[string]struct{})
Expand Down

0 comments on commit 9d2c8a6

Please sign in to comment.