Merge branch 'dev' into branch-2.0

CHANGELOG.md

@@ -1,5 +1,6 @@
 # 更新日志
 
+
 ## v 2.0.10.0
 
 对应 doris 2.0.10.
@@ -23,6 +24,7 @@
 - 添加选项以启动 pprof server
 - 允许配置 rpc 和 connection 超时
 
+
 ### Fix
 
 - restore 每次重试时使用不同的 label 名

cmd/ingest_binlog/ingest_binlog.go

@@ -151,7 +151,7 @@ func test_ingrest_binlog(src *base.Spec, dest *base.Spec) {
 	case "commit":
 		test_commit(dest)
 	case "abort":
-		panic("unkown abort action")
+		panic("unknown abort action")
 	case "ingest_be":
 		test_ingest_be()
 	default:

pkg/ccr/base/spec.go

@@ -520,7 +520,7 @@ func (s *Spec) CreateSnapshotAndWaitForDone(tables []string) (string, error) {
 		tableRefs = "`" + strings.Join(tables, "`,`") + "`"
 	}
 
-	// means source is a empty db, table numer is 0
+	// means source is a empty db, table number is 0
 	if tableRefs == "``" {
 		return "", xerror.Errorf(xerror.Normal, "source db is empty! you should have at least one table")
 	}
@@ -586,17 +586,21 @@ func (s *Spec) checkBackupFinished(snapshotName string) (BackupState, error) {
 }
 
 func (s *Spec) CheckBackupFinished(snapshotName string) (bool, error) {
-	log.Debugf("check backup state, datebase: %s, snapshot: %s", s.Database, snapshotName)
+	log.Debugf("check backup state, spec: %s, snapshot: %s", s.String(), snapshotName)
 
 	for i := 0; i < MAX_CHECK_RETRY_TIMES; i++ {
-		if backupState, err := s.checkBackupFinished(snapshotName); err != nil {
+		// Retry network related error to avoid full sync when the target network is interrupted, process is restarted.
+		if backupState, err := s.checkBackupFinished(snapshotName); err != nil && !isNetworkRelated(err) {
 			return false, err
-		} else if backupState == BackupStateFinished {
+		} else if err == nil && backupState == BackupStateFinished {
 			return true, nil
-		} else if backupState == BackupStateCancelled {
+		} else if err == nil && backupState == BackupStateCancelled {
 			return false, xerror.Errorf(xerror.Normal, "backup failed or canceled")
 		} else {
-			// BackupStatePending, BackupStateUnknown
+			// BackupStatePending, BackupStateUnknown or network related errors.
+			if err != nil {
+				log.Warnf("check backup state is failed, spec: %s, snapshot: %s, err: %v", s.String(), snapshotName, err)
+			}
 			time.Sleep(BACKUP_CHECK_DURATION)
 		}
 	}
@@ -647,19 +651,23 @@ func (s *Spec) checkRestoreFinished(snapshotName string) (RestoreState, string,
 }
 
 func (s *Spec) CheckRestoreFinished(snapshotName string) (bool, error) {
-	log.Debugf("check restore state is finished, spec: %s, datebase: %s, snapshot: %s", s.String(), s.Database, snapshotName)
+	log.Debugf("check restore state is finished, spec: %s, snapshot: %s", s.String(), snapshotName)
 
 	for i := 0; i < MAX_CHECK_RETRY_TIMES; i++ {
-		if restoreState, status, err := s.checkRestoreFinished(snapshotName); err != nil {
+		// Retry network related error to avoid full sync when the target network is interrupted, process is restarted.
+		if restoreState, status, err := s.checkRestoreFinished(snapshotName); err != nil && !isNetworkRelated(err) {
 			return false, err
-		} else if restoreState == RestoreStateFinished {
+		} else if err == nil && restoreState == RestoreStateFinished {
 			return true, nil
-		} else if restoreState == RestoreStateCancelled && strings.Contains(status, SIGNATURE_NOT_MATCHED) {
+		} else if err == nil && restoreState == RestoreStateCancelled && strings.Contains(status, SIGNATURE_NOT_MATCHED) {
 			return false, xerror.XWrapf(ErrRestoreSignatureNotMatched, "restore failed, spec: %s, snapshot: %s, status: %s", s.String(), snapshotName, status)
-		} else if restoreState == RestoreStateCancelled {
+		} else if err == nil && restoreState == RestoreStateCancelled {
 			return false, xerror.Errorf(xerror.Normal, "restore failed or canceled, spec: %s, snapshot: %s, status: %s", s.String(), snapshotName, status)
 		} else {
-			// RestoreStatePending, RestoreStateUnknown
+			// RestoreStatePending, RestoreStateUnknown or network error.
+			if err != nil {
+				log.Warnf("check restore state is failed, spec: %s, snapshot: %s, err: %v", s.String(), snapshotName, err)
+			}
 			time.Sleep(RESTORE_CHECK_DURATION)
 		}
 	}

pkg/ccr/job.go

@@ -1361,7 +1361,7 @@ func (j *Job) sync() error {
 func (j *Job) handleError(err error) error {
 	var xerr *xerror.XError
 	if !errors.As(err, &xerr) {
-		log.Warnf("convert error to xerror failed, err: %+v", err)
+		log.Errorf("convert error to xerror failed, err: %+v", err)
 		return nil
 	}
 

pkg/ccr/meta.go

@@ -98,7 +98,7 @@ func (m *Meta) GetDbId() (int64, error) {
 		}
 
 		// match parsedDbname == dbname, return dbId
-		// the defualt_cluster prefix of db name will be removed in Doris v2.1.
+		// the default_cluster prefix of db name will be removed in Doris v2.1.
 		// here we compare both db name and db full name to make it compatible.
 		if parsedDbName == dbName || parsedDbName == dbFullName {
 			m.DatabaseName2IdMap[dbFullName] = dbId

regression-test/data/table-sync/test_inverted_index.out

@@ -0,0 +1,275 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+1	andy	andy love apple	100
+1	bason	bason hate pear	100
+2	andy	andy love apple	100
+2	bason	bason hate pear	98
+3	andy	andy love apple	100
+3	bason	bason hate pear	99
+4	andy	andy love apple	100
+4	bason	bason hate pear	99
+
+-- !sql --
+1	andy	andy love apple	100
+2	andy	andy love apple	100
+3	andy	andy love apple	100
+4	andy	andy love apple	100
+
+-- !sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+4	bason	bason hate pear	99
+
+-- !sql --
+2	bason	bason hate pear	98
+
+-- !target_sql --
+1	andy	andy love apple	100
+1	bason	bason hate pear	100
+2	andy	andy love apple	100
+2	bason	bason hate pear	98
+3	andy	andy love apple	100
+3	bason	bason hate pear	99
+4	andy	andy love apple	100
+4	bason	bason hate pear	99
+
+-- !target_sql --
+1	andy	andy love apple	100
+2	andy	andy love apple	100
+3	andy	andy love apple	100
+4	andy	andy love apple	100
+
+-- !target_sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+4	bason	bason hate pear	99
+
+-- !target_sql --
+2	bason	bason hate pear	98
+
+-- !sql --
+1	andy	andy love apple	100
+1	bason	bason hate pear	100
+2	andy	andy love apple	100
+2	bason	bason hate pear	98
+3	andy	andy love apple	100
+3	bason	bason hate pear	99
+4	andy	andy love apple	100
+4	bason	bason hate pear	99
+5	andy	andy love apple	100
+5	bason	bason hate pear	99
+6	andy	andy love apple	98
+6	bason	bason hate pear	99
+
+-- !sql --
+1	andy	andy love apple	100
+2	andy	andy love apple	100
+3	andy	andy love apple	100
+4	andy	andy love apple	100
+5	andy	andy love apple	100
+6	andy	andy love apple	98
+
+-- !sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+4	bason	bason hate pear	99
+5	bason	bason hate pear	99
+6	bason	bason hate pear	99
+
+-- !sql --
+2	bason	bason hate pear	98
+6	andy	andy love apple	98
+
+-- !target_sql --
+1	andy	andy love apple	100
+1	bason	bason hate pear	100
+2	andy	andy love apple	100
+2	bason	bason hate pear	98
+3	andy	andy love apple	100
+3	bason	bason hate pear	99
+4	andy	andy love apple	100
+4	bason	bason hate pear	99
+5	andy	andy love apple	100
+5	bason	bason hate pear	99
+6	andy	andy love apple	98
+6	bason	bason hate pear	99
+
+-- !target_sql --
+1	andy	andy love apple	100
+2	andy	andy love apple	100
+3	andy	andy love apple	100
+4	andy	andy love apple	100
+5	andy	andy love apple	100
+6	andy	andy love apple	98
+
+-- !target_sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+4	bason	bason hate pear	99
+5	bason	bason hate pear	99
+6	bason	bason hate pear	99
+
+-- !target_sql --
+2	bason	bason hate pear	98
+6	andy	andy love apple	98
+
+-- !sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+4	andy	andy love apple	100
+
+-- !sql --
+4	andy	andy love apple	100
+
+-- !sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+
+-- !sql --
+2	bason	bason hate pear	98
+
+-- !target_sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+4	andy	andy love apple	100
+
+-- !target_sql --
+4	andy	andy love apple	100
+
+-- !target_sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+
+-- !target_sql --
+2	bason	bason hate pear	98
+
+-- !sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+4	andy	andy love apple	100
+5	andy	andy love apple	100
+6	andy	andy love apple	98
+
+-- !sql --
+4	andy	andy love apple	100
+5	andy	andy love apple	100
+6	andy	andy love apple	98
+
+-- !sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+
+-- !sql --
+2	bason	bason hate pear	98
+6	andy	andy love apple	98
+
+-- !target_sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+4	andy	andy love apple	100
+5	andy	andy love apple	100
+6	andy	andy love apple	98
+
+-- !target_sql --
+4	andy	andy love apple	100
+5	andy	andy love apple	100
+6	andy	andy love apple	98
+
+-- !target_sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+
+-- !target_sql --
+2	bason	bason hate pear	98
+6	andy	andy love apple	98
+
+-- !sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+4	andy	andy love apple	100
+
+-- !sql --
+4	andy	andy love apple	100
+
+-- !sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+
+-- !sql --
+2	bason	bason hate pear	98
+
+-- !target_sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+4	andy	andy love apple	100
+
+-- !target_sql --
+4	andy	andy love apple	100
+
+-- !target_sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+
+-- !target_sql --
+2	bason	bason hate pear	98
+
+-- !sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+4	andy	andy love apple	100
+5	andy	andy love apple	100
+6	andy	andy love apple	98
+
+-- !sql --
+4	andy	andy love apple	100
+5	andy	andy love apple	100
+6	andy	andy love apple	98
+
+-- !sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+
+-- !sql --
+2	bason	bason hate pear	98
+6	andy	andy love apple	98
+
+-- !target_sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+4	andy	andy love apple	100
+5	andy	andy love apple	100
+6	andy	andy love apple	98
+
+-- !target_sql --
+4	andy	andy love apple	100
+5	andy	andy love apple	100
+6	andy	andy love apple	98
+
+-- !target_sql --
+1	bason	bason hate pear	100
+2	bason	bason hate pear	98
+3	bason	bason hate pear	99
+
+-- !target_sql --
+2	bason	bason hate pear	98
+6	andy	andy love apple	98
+