From a4d7b01b6f9c4d305768109b6d44d30b0753f709 Mon Sep 17 00:00:00 2001 From: walter Date: Wed, 6 Nov 2024 16:43:03 +0800 Subject: [PATCH 1/2] Reuse the backup/restore task issued by the job itself (#218) Reusing it can effectively reduce overhead --- pkg/ccr/base/spec.go | 112 ++++++++++++-------------- pkg/ccr/base/specer.go | 8 +- pkg/ccr/job.go | 138 ++++++++++++++++++-------------- pkg/ccr/job_progress.go | 2 + pkg/ccr/label.go | 33 ++++++++ pkg/ccr/record/add_partition.go | 2 +- 6 files changed, 168 insertions(+), 127 deletions(-) create mode 100644 pkg/ccr/label.go diff --git a/pkg/ccr/base/spec.go b/pkg/ccr/base/spec.go index 93f715db..f525c37b 100644 --- a/pkg/ccr/base/spec.go +++ b/pkg/ccr/base/spec.go @@ -628,7 +628,7 @@ func (s *Spec) CheckTableExistsByName(tableName string) (bool, error) { } // mysql> BACKUP SNAPSHOT ccr.snapshot_20230605 TO `__keep_on_local__` ON ( src_1 ) PROPERTIES ("type" = "full"); -func (s *Spec) CreateSnapshot(tables []string) (string, error) { +func (s *Spec) CreateSnapshot(snapshotName string, tables []string) error { if tables == nil { tables = make([]string, 0) } @@ -636,56 +636,49 @@ func (s *Spec) CreateSnapshot(tables []string) (string, error) { tables = append(tables, s.Table) } - var snapshotName string var tableRefs string if len(tables) == 1 { - // snapshot name format "ccrs_${table}_${timestamp}" // table refs = table - snapshotName = fmt.Sprintf("ccrs_%s_%s_%d", s.Database, s.Table, time.Now().Unix()) tableRefs = utils.FormatKeywordName(tables[0]) } else { - // snapshot name format "ccrs_${db}_${timestamp}" // table refs = tables.join(", ") - snapshotName = fmt.Sprintf("ccrs_%s_%d", s.Database, time.Now().Unix()) tableRefs = "`" + strings.Join(tables, "`,`") + "`" } // means source is a empty db, table number is 0 if tableRefs == "``" { - return "", xerror.Errorf(xerror.Normal, "source db is empty! you should have at least one table") + return xerror.Errorf(xerror.Normal, "source db is empty! you should have at least one table") } db, err := s.Connect() if err != nil { - return "", err + return err } backupSnapshotSql := fmt.Sprintf("BACKUP SNAPSHOT %s.%s TO `__keep_on_local__` ON ( %s ) PROPERTIES (\"type\" = \"full\")", utils.FormatKeywordName(s.Database), utils.FormatKeywordName(snapshotName), tableRefs) log.Infof("create snapshot %s.%s, backup snapshot sql: %s", s.Database, snapshotName, backupSnapshotSql) _, err = db.Exec(backupSnapshotSql) if err != nil { - return "", xerror.Wrapf(err, xerror.Normal, "backup snapshot %s failed, sql: %s", snapshotName, backupSnapshotSql) + return xerror.Wrapf(err, xerror.Normal, "backup snapshot %s failed, sql: %s", snapshotName, backupSnapshotSql) } - return snapshotName, nil + return nil } // mysql> BACKUP SNAPSHOT ccr.snapshot_20230605 TO `__keep_on_local__` ON (src_1 PARTITION (`p1`)) PROPERTIES ("type" = "full"); -func (s *Spec) CreatePartialSnapshot(table string, partitions []string) (string, error) { +func (s *Spec) CreatePartialSnapshot(snapshotName, table string, partitions []string) error { if len(table) == 0 { - return "", xerror.Errorf(xerror.Normal, "source db is empty! you should have at least one table") + return xerror.Errorf(xerror.Normal, "source db is empty! you should have at least one table") } - // snapshot name format "ccrp_${table}_${timestamp}" // table refs = table - snapshotName := fmt.Sprintf("ccrp_%s_%s_%d", s.Database, s.Table, time.Now().Unix()) tableRef := utils.FormatKeywordName(table) log.Infof("create partial snapshot %s.%s", s.Database, snapshotName) db, err := s.Connect() if err != nil { - return "", err + return err } partitionRefs := "" @@ -698,10 +691,10 @@ func (s *Spec) CreatePartialSnapshot(table string, partitions []string) (string, log.Debugf("backup partial snapshot sql: %s", backupSnapshotSql) _, err = db.Exec(backupSnapshotSql) if err != nil { - return "", xerror.Wrapf(err, xerror.Normal, "backup partial snapshot %s failed, sql: %s", snapshotName, backupSnapshotSql) + return xerror.Wrapf(err, xerror.Normal, "backup partial snapshot %s failed, sql: %s", snapshotName, backupSnapshotSql) } - return snapshotName, nil + return nil } // TODO: Add TaskErrMsg @@ -757,103 +750,102 @@ func (s *Spec) CheckBackupFinished(snapshotName string) (bool, error) { } } -func (s *Spec) CancelBackupIfExists() error { - log.Debugf("cancel backup job if exists, database: %s", s.Database) +// Get the valid (running or finished) backup job with a unique prefix to indicate +// if a backup job needs to be issued again. +func (s *Spec) GetValidBackupJob(snapshotNamePrefix string) (string, error) { + log.Debugf("get valid backup job if exists, database: %s, label prefix: %s", s.Database, snapshotNamePrefix) db, err := s.Connect() if err != nil { - return err + return "", err } - query := fmt.Sprintf("SHOW BACKUP FROM %s", utils.FormatKeywordName(s.Database)) + query := fmt.Sprintf("SHOW BACKUP FROM %s WHERE SnapshotName LIKE \"%s%%\"", + utils.FormatKeywordName(s.Database), snapshotNamePrefix) log.Infof("show backup state sql: %s", query) rows, err := db.Query(query) if err != nil { - return xerror.Wrap(err, xerror.Normal, "query backup state failed") + return "", xerror.Wrap(err, xerror.Normal, "query backup state failed") } defer rows.Close() + labels := make([]string, 0) for rows.Next() { rowParser := utils.NewRowParser() if err := rowParser.Parse(rows); err != nil { - return xerror.Wrap(err, xerror.Normal, "scan backup state failed") + return "", xerror.Wrap(err, xerror.Normal, "scan backup state failed") } info, err := parseBackupInfo(rowParser) if err != nil { - return xerror.Wrap(err, xerror.Normal, "scan backup state failed") + return "", xerror.Wrap(err, xerror.Normal, "scan backup state failed") } log.Infof("check snapshot %s backup state [%v], create time: %s", info.SnapshotName, info.StateStr, info.CreateTime) - // Only cancel the running backup job issued by syncer - if !isSyncerIssuedJob(info.SnapshotName, s.Database) { + if info.State == BackupStateCancelled { continue } - if info.State == BackupStateFinished || info.State == BackupStateCancelled { - continue - } + labels = append(labels, info.SnapshotName) + } - cancelSql := fmt.Sprintf("CANCEL BACKUP FROM %s", s.Database) - log.Infof("cancel backup sql: %s, snapshot: %s", cancelSql, info.SnapshotName) - if _, err = db.Exec(cancelSql); err != nil { - return xerror.Wrapf(err, xerror.Normal, - "cancel backup job %s failed, database: %s", info.SnapshotName, s.Database) - } + // Return the last one. Assume that the result of `SHOW BACKUP` is ordered by CreateTime in ascending order. + if len(labels) != 0 { + return labels[len(labels)-1], nil } - return nil + + return "", nil } -func (s *Spec) CancelRestoreIfExists(srcDbName string) error { - log.Debugf("cancel restore job if exists, src db: %s", srcDbName) +// Get the valid (running or finished) restore job with a unique prefix to indicate +// if a restore job needs to be issued again. +func (s *Spec) GetValidRestoreJob(snapshotNamePrefix string) (string, error) { + log.Debugf("get valid restore job if exists, label prefix: %s", snapshotNamePrefix) db, err := s.Connect() if err != nil { - return err + return "", err } - query := fmt.Sprintf("SHOW RESTORE FROM %s", utils.FormatKeywordName(s.Database)) - log.Debugf("show restore state sql: %s", query) + query := fmt.Sprintf("SHOW RESTORE FROM %s WHERE Label LIKE \"%s%%\"", + utils.FormatKeywordName(s.Database), snapshotNamePrefix) + log.Infof("show restore state sql: %s", query) rows, err := db.Query(query) if err != nil { - return xerror.Wrap(err, xerror.Normal, "query restore state failed") + return "", xerror.Wrap(err, xerror.Normal, "query restore state failed") } defer rows.Close() + labels := make([]string, 0) for rows.Next() { rowParser := utils.NewRowParser() if err := rowParser.Parse(rows); err != nil { - return xerror.Wrap(err, xerror.Normal, "scan restore state failed") + return "", xerror.Wrap(err, xerror.Normal, "scan restore state failed") } info, err := parseRestoreInfo(rowParser) if err != nil { - return xerror.Wrap(err, xerror.Normal, "scan restore state failed") + return "", xerror.Wrap(err, xerror.Normal, "scan restore state failed") } log.Infof("check snapshot %s restore state: [%v], create time: %s", info.Label, info.StateStr, info.CreateTime) - // Only cancel the running restore job issued by syncer - if !isSyncerIssuedJob(info.Label, srcDbName) { + if info.State == RestoreStateFinished { continue } - if info.State == RestoreStateCancelled || info.State == RestoreStateFinished { - continue - } - - cancelSql := fmt.Sprintf("CANCEL RESTORE FROM %s", utils.FormatKeywordName(s.Database)) - log.Infof("cancel restore sql: %s, running snapshot %s", cancelSql, info.Label) + labels = append(labels, info.Label) + } - _, err = db.Exec(cancelSql) - if err != nil { - return xerror.Wrapf(err, xerror.Normal, "cancel running restore failed, snapshot %s", info.Label) - } + // Return the last one. Assume that the result of `SHOW BACKUP` is ordered by CreateTime in ascending order. + if len(labels) != 0 { + return labels[len(labels)-1], nil } - return nil + + return "", nil } // TODO: Add TaskErrMsg @@ -1240,9 +1232,3 @@ func correctAddPartitionSql(addPartitionSql string, addPartition *record.AddPart } return addPartitionSql } - -func isSyncerIssuedJob(label, dbName string) bool { - fullSyncPrefix := fmt.Sprintf("ccrs_%s", dbName) - partialSyncPrefix := fmt.Sprintf("ccrp_%s", dbName) - return strings.HasPrefix(label, fullSyncPrefix) || strings.HasPrefix(label, partialSyncPrefix) -} diff --git a/pkg/ccr/base/specer.go b/pkg/ccr/base/specer.go index ad00e290..9b5a4774 100644 --- a/pkg/ccr/base/specer.go +++ b/pkg/ccr/base/specer.go @@ -25,11 +25,11 @@ type Specer interface { CheckDatabaseExists() (bool, error) CheckTableExists() (bool, error) CheckTableExistsByName(tableName string) (bool, error) - CreatePartialSnapshot(table string, partitions []string) (string, error) - CreateSnapshot(tables []string) (string, error) + GetValidBackupJob(snapshotNamePrefix string) (string, error) + GetValidRestoreJob(snapshotNamePrefix string) (string, error) + CreatePartialSnapshot(snapshotName, table string, partitions []string) error + CreateSnapshot(snapshotName string, tables []string) error CheckBackupFinished(snapshotName string) (bool, error) - CancelBackupIfExists() error - CancelRestoreIfExists(srcDbName string) error CheckRestoreFinished(snapshotName string) (bool, error) GetRestoreSignatureNotMatchedTableOrView(snapshotName string) (string, bool, error) WaitTransactionDone(txnId int64) // busy wait diff --git a/pkg/ccr/job.go b/pkg/ccr/job.go index ddc0fd46..729b7b9c 100644 --- a/pkg/ccr/job.go +++ b/pkg/ccr/job.go @@ -36,13 +36,13 @@ const ( ) var ( - featureSchemaChangePartialSync bool - featureCleanTableAndPartitions bool - featureAtomicRestore bool - featureCreateViewDropExists bool - featureReplaceNotMatchedWithAlias bool - featureFilterShadowIndexesUpsert bool - featureCancelConflictBackupRestoreJob bool + featureSchemaChangePartialSync bool + featureCleanTableAndPartitions bool + featureAtomicRestore bool + featureCreateViewDropExists bool + featureReplaceNotMatchedWithAlias bool + featureFilterShadowIndexesUpsert bool + featureReuseRunningBackupRestoreJob bool ) func init() { @@ -60,8 +60,8 @@ func init() { "replace signature not matched tables with table alias during the full sync") flag.BoolVar(&featureFilterShadowIndexesUpsert, "feature_filter_shadow_indexes_upsert", true, "filter the upsert to the shadow indexes") - flag.BoolVar(&featureCancelConflictBackupRestoreJob, "feature_cancel_conflict_backup_restore_job", false, - "cancel the conflict backup/restore job before issue new backup/restore job") + flag.BoolVar(&featureReuseRunningBackupRestoreJob, "feature_reuse_running_backup_restore_job", false, + "reuse the running backup/restore issued by the job self") } type SyncType int @@ -363,19 +363,28 @@ func (j *Job) partialSync() error { case BeginCreateSnapshot: // Step 1: Create snapshot - log.Infof("partial sync status: create snapshot") - if featureCancelConflictBackupRestoreJob { - if err := j.ISrc.CancelBackupIfExists(); err != nil { + prefix := NewPartialSnapshotLabelPrefix(j.Name, j.progress.SyncId) + log.Infof("partial sync status: create snapshot with prefix %s", prefix) + + if featureReuseRunningBackupRestoreJob { + snapshotName, err := j.ISrc.GetValidBackupJob(prefix) + if err != nil { return err } + if snapshotName != "" { + log.Infof("partial sync status: find a valid backup job %s", snapshotName) + j.progress.NextSubVolatile(WaitBackupDone, snapshotName) + return nil + } } - snapshotName, err := j.ISrc.CreatePartialSnapshot(table, partitions) - if err != nil { + snapshotName := NewLabelWithTs(prefix) + if err := j.ISrc.CreatePartialSnapshot(snapshotName, table, partitions); err != nil { return err } j.progress.NextSubVolatile(WaitBackupDone, snapshotName) + return nil case WaitBackupDone: // Step 2: Wait backup job done @@ -387,7 +396,7 @@ func (j *Job) partialSync() error { } if !backupFinished { - log.Debugf("partial sync status: backup job %s is running, retry later", snapshotName) + log.Infof("partial sync status: backup job %s is running", snapshotName) return nil } @@ -479,20 +488,26 @@ func (j *Job) partialSync() error { j.progress.InMemoryData = inMemoryData } - // Step 5.1: cancel the running restore job which submitted by former progress, if exists - if featureCancelConflictBackupRestoreJob { - if err := j.IDest.CancelRestoreIfExists(j.Src.Database); err != nil { - return err + // Step 5.1: try reuse the exists restore job. + inMemoryData := j.progress.InMemoryData.(*inMemoryData) + snapshotName := inMemoryData.SnapshotName + if featureReuseRunningBackupRestoreJob { + name, err := j.IDest.GetValidRestoreJob(snapshotName) + if err != nil { + return nil + } + if name != "" { + log.Infof("partial sync status: find a valid restore job %s", name) + inMemoryData.RestoreLabel = name + j.progress.NextSubVolatile(WaitRestoreDone, inMemoryData) + break } } - // Step 5.2: start a new fullsync && persist - inMemoryData := j.progress.InMemoryData.(*inMemoryData) - snapshotName := inMemoryData.SnapshotName - restoreSnapshotName := restoreSnapshotName(snapshotName) + // Step 5.2: start a new fullsync & restore snapshot to dest + restoreSnapshotName := NewRestoreLabel(snapshotName) snapshotResp := inMemoryData.SnapshotResp - // Step 5.3: restore snapshot to dest dest := &j.Dest destRpc, err := j.factory.NewFeRpc(dest) if err != nil { @@ -542,6 +557,7 @@ func (j *Job) partialSync() error { inMemoryData.RestoreLabel = restoreSnapshotName j.progress.NextSubVolatile(WaitRestoreDone, inMemoryData) + return nil case WaitRestoreDone: // Step 6: Wait restore job done @@ -555,7 +571,7 @@ func (j *Job) partialSync() error { } if !restoreFinished { - log.Debugf("partial sync status: restore job %s is running", restoreSnapshotName) + log.Infof("partial sync status: restore job %s is running", restoreSnapshotName) return nil } @@ -651,7 +667,20 @@ func (j *Job) fullSync() error { case BeginCreateSnapshot: // Step 1: Create snapshot - log.Infof("fullsync status: create snapshot") + prefix := NewSnapshotLabelPrefix(j.Name, j.progress.SyncId) + log.Infof("fullsync status: create snapshot with prefix %s", prefix) + + if featureReuseRunningBackupRestoreJob { + snapshotName, err := j.ISrc.GetValidBackupJob(prefix) + if err != nil { + return err + } + if snapshotName != "" { + log.Infof("fullsync status: find a valid backup job %s", snapshotName) + j.progress.NextSubVolatile(WaitBackupDone, snapshotName) + return nil + } + } backupTableList := make([]string, 0) switch j.SyncType { @@ -673,17 +702,12 @@ func (j *Job) fullSync() error { return xerror.Errorf(xerror.Normal, "invalid sync type %s", j.SyncType) } - if featureCancelConflictBackupRestoreJob { - if err := j.ISrc.CancelBackupIfExists(); err != nil { - return err - } - } - - snapshotName, err := j.ISrc.CreateSnapshot(backupTableList) - if err != nil { + snapshotName := NewLabelWithTs(prefix) + if err := j.ISrc.CreateSnapshot(snapshotName, backupTableList); err != nil { return err } j.progress.NextSubVolatile(WaitBackupDone, snapshotName) + return nil case WaitBackupDone: // Step 2: Wait backup job done @@ -694,7 +718,7 @@ func (j *Job) fullSync() error { return err } if !backupFinished { - log.Debugf("fullsync status: backup job %s is running, retry later", snapshotName) + log.Infof("fullsync status: backup job %s is running", snapshotName) return nil } @@ -785,20 +809,26 @@ func (j *Job) fullSync() error { } // Step 5.1: cancel the running restore job which by the former process, if exists - if featureCancelConflictBackupRestoreJob { - if err := j.IDest.CancelRestoreIfExists(j.Src.Database); err != nil { - return err + inMemoryData := j.progress.InMemoryData.(*inMemoryData) + snapshotName := inMemoryData.SnapshotName + if featureReuseRunningBackupRestoreJob { + restoreSnapshotName, err := j.IDest.GetValidRestoreJob(snapshotName) + if err != nil { + return nil + } + if restoreSnapshotName != "" { + log.Infof("fullsync status: find a valid restore job %s", restoreSnapshotName) + inMemoryData.RestoreLabel = restoreSnapshotName + j.progress.NextSubVolatile(WaitRestoreDone, inMemoryData) + break } } - // Step 5.2: start a new fullsync && persist - inMemoryData := j.progress.InMemoryData.(*inMemoryData) - snapshotName := inMemoryData.SnapshotName - restoreSnapshotName := restoreSnapshotName(snapshotName) + // Step 5.2: start a new fullsync & restore snapshot to dest + restoreSnapshotName := NewRestoreLabel(snapshotName) snapshotResp := inMemoryData.SnapshotResp tableNameMapping := inMemoryData.TableNameMapping - // Step 5.3: restore snapshot to dest dest := &j.Dest destRpc, err := j.factory.NewFeRpc(dest) if err != nil { @@ -871,6 +901,7 @@ func (j *Job) fullSync() error { inMemoryData.RestoreLabel = restoreSnapshotName j.progress.NextSubVolatile(WaitRestoreDone, inMemoryData) + return nil case WaitRestoreDone: // Step 6: Wait restore job done @@ -902,7 +933,7 @@ func (j *Job) fullSync() error { if j.progress.TableAliases == nil { j.progress.TableAliases = make(map[string]string) } - j.progress.TableAliases[tableName] = tableAlias(tableName) + j.progress.TableAliases[tableName] = TableAlias(tableName) j.progress.NextSubVolatile(RestoreSnapshot, inMemoryData) break } @@ -925,7 +956,7 @@ func (j *Job) fullSync() error { } if !restoreFinished { - log.Debugf("fullsync status: restore job %s is running, retry later", restoreSnapshotName) + log.Info("fullsync status: restore job %s is running", restoreSnapshotName) return nil } @@ -2281,6 +2312,7 @@ func (j *Job) newSnapshot(commitSeq int64) error { j.progress.PartialSyncData = nil j.progress.TableAliases = nil + j.progress.SyncId += 1 switch j.SyncType { case TableSync: j.progress.NextWithPersist(commitSeq, TableFullSync, BeginCreateSnapshot, "") @@ -2320,8 +2352,9 @@ func (j *Job) newPartialSnapshot(table string, partitions []string, replace bool } j.progress.PartialSyncData = syncData j.progress.TableAliases = nil + j.progress.SyncId += 1 if replace { - alias := tableAlias(table) + alias := TableAlias(table) j.progress.TableAliases = make(map[string]string) j.progress.TableAliases[table] = alias log.Infof("new partial snapshot, commitSeq: %d, table: %s, alias: %s", commitSeq, table, alias) @@ -2703,16 +2736,3 @@ func isStatusContainsAny(status *tstatus.TStatus, patterns ...string) bool { } return false } - -func restoreSnapshotName(snapshotName string) string { - if snapshotName == "" { - return "" - } - - // use current seconds - return fmt.Sprintf("%s_r_%d", snapshotName, time.Now().Unix()) -} - -func tableAlias(tableName string) string { - return fmt.Sprintf("__ccr_%s_%d", tableName, time.Now().Unix()) -} diff --git a/pkg/ccr/job_progress.go b/pkg/ccr/job_progress.go index 72e36f20..6605f9f0 100644 --- a/pkg/ccr/job_progress.go +++ b/pkg/ccr/job_progress.go @@ -151,6 +151,8 @@ type JobProgress struct { // Sub sync state machine states SubSyncState SubSyncState `json:"sub_sync_state"` + // The sync id of full/partial snapshot + SyncId int64 `json:"job_sync_id"` // The commit seq where the target cluster has synced. PrevCommitSeq int64 `json:"prev_commit_seq"` CommitSeq int64 `json:"commit_seq"` diff --git a/pkg/ccr/label.go b/pkg/ccr/label.go new file mode 100644 index 00000000..43a1c4a7 --- /dev/null +++ b/pkg/ccr/label.go @@ -0,0 +1,33 @@ +package ccr + +import ( + "fmt" + "time" +) + +// snapshot name format "ccrs_${ccr_name}_${sync_id}" +func NewSnapshotLabelPrefix(ccrName string, syncId int64) string { + return fmt.Sprintf("ccrs_%s_%d", ccrName, syncId) +} + +// snapshot name format "ccrp_${ccr_name}_${sync_id}" +func NewPartialSnapshotLabelPrefix(ccrName string, syncId int64) string { + return fmt.Sprintf("ccrp_%s_%d", ccrName, syncId) +} + +func NewLabelWithTs(prefix string) string { + return fmt.Sprintf("%s_%d", prefix, time.Now().Unix()) +} + +func NewRestoreLabel(snapshotName string) string { + if snapshotName == "" { + return "" + } + + // use current seconds + return fmt.Sprintf("%s_r_%d", snapshotName, time.Now().Unix()) +} + +func TableAlias(tableName string) string { + return fmt.Sprintf("__ccr_%s_%d", tableName, time.Now().Unix()) +} diff --git a/pkg/ccr/record/add_partition.go b/pkg/ccr/record/add_partition.go index 065c3990..567913e4 100644 --- a/pkg/ccr/record/add_partition.go +++ b/pkg/ccr/record/add_partition.go @@ -95,7 +95,7 @@ func (addPartition *AddPartition) GetSql(destTableName string) string { if !strings.Contains(strings.ToUpper(addPartitionSql), "DISTRIBUTED BY") { if distributionInfo.Type == "HASH" { addPartitionSql = fmt.Sprintf("%s DISTRIBUTED BY HASH(%s)", addPartitionSql, - "`" + strings.Join(addPartition.getDistributionColumns(), "`,`") + "`") + "`"+strings.Join(addPartition.getDistributionColumns(), "`,`")+"`") } else { addPartitionSql = fmt.Sprintf("%s DISTRIBUTED BY RANDOM", addPartitionSql) } From a134ed7f434afbbab56bf8deb3e2e6f39fad7d40 Mon Sep 17 00:00:00 2001 From: walter Date: Wed, 6 Nov 2024 19:38:35 +0800 Subject: [PATCH 2/2] Reorg db/table sync suites (#219) Some requirements for adding suites 1. Each suite should be place in a unique dir, to avoid backup/restore job limit per db 2. Some idom 1. Kind: 1. cross_ds/ts: cross-feature related suites 2. db/table_sync: db or table sync related suites 3. table_sync_alias: table sync with alias related suites 4. db/table_ps_sync: db or table partial sync related suites 5. syncer: suites for ccr syncer itself 2. Resource: column,view,table ... 3. Operation: add,alter,drop,rename ... 3. The suite is organized in the directory hierarchy of Kind/Resource/Operation, and Resource is optional --- regression-test/common/helper.groovy | 51 +++++-- .../data/ccr_user_sync/test_common_sync.out | 1 - .../partition/drop_1/test_ds_part_drop_1.out} | 0 .../test_ts_dml_insert_overwrite.out} | 0 .../test_ts_tbl_res_inverted_idx.out} | 0 .../test_cds_fullsync_with_alias.groovy} | 2 +- .../test_cds_signature_not_matched.groovy} | 4 +- .../test_cds_sync_view_twice.groovy} | 3 +- .../keyword_name/test_cts_keyword.groovy} | 2 +- ...t_db_partial_sync_inc_add_partition.groovy | 4 +- .../test_db_partial_sync_inc_alter.groovy | 4 +- .../cache}/test_db_partial_sync_cache.groovy | 37 +---- ..._db_partial_sync_inc_drop_partition.groovy | 4 +- ...st_db_partial_sync_inc_lightning_sc.groovy | 4 +- .../merge/test_db_partial_sync_merge.groovy | 4 +- ..._partial_sync_inc_replace_partition.groovy | 4 +- ...est_db_partial_sync_inc_trunc_table.groovy | 4 +- .../test_db_partial_sync_inc_upsert.groovy | 4 +- .../column/basic/test_ds_col_basic.groovy} | 50 ++----- .../common/test_ds_common.groovy} | 2 +- .../test_ds_dml_insert_overwrite.groovy} | 5 +- .../mv/basic/test_ds_mv_basic.groovy} | 5 +- .../partition/drop/test_ds_part_drop.groovy} | 5 +- .../drop_1/test_ds_part_drop_1.groovy} | 4 +- .../replace/test_ds_part_replace.groovy} | 5 +- .../test_ds_clean_restore.groovy} | 4 +- .../test_ds_tbl_create_drop.groovy} | 7 +- .../test_ds_tbl_drop_create.groovy | 23 +++ .../table/rename/test_ds_tbl_rename.groovy} | 5 +- .../truncate/test_ds_tbl_truncate.groovy} | 4 +- .../view/basic/test_ds_view_basic.groovy | 111 ++++++++++++++ .../test_ds_view_drop_create.groovy} | 4 +- .../test_ds_view_drop_delete_create.groovy} | 4 +- .../test_syncer_ts_allow_table_exists.groovy} | 4 +- .../basic/test_tbl_ps_inc_basic.groovy} | 4 +- .../cache/test_tbl_ps_inc_cache.groovy} | 45 +----- .../column/add/test_ts_col_add.groovy} | 45 +----- .../add_agg/test_ts_col_add_agg.groovy} | 4 +- .../add_many/test_ts_col_add_many.groovy} | 4 +- .../alter_type/test_ts_col_alter_type.groovy} | 4 +- .../column/basic/test_ts_col_basic.groovy} | 6 +- .../column/drop/test_ts_col_drop.groovy} | 4 +- ...test_ts_col_filter_dropped_indexes.groovy} | 4 +- .../order_by/test_ts_col_order_by.groovy} | 4 +- .../common/test_ts_common.groovy} | 4 +- .../dml/delete/test_ts_dml_delete.groovy} | 4 +- .../test_ts_dml_insert_overwrite.groovy} | 4 +- .../test_tbl_index_add_bloom_filter.groovy} | 4 +- .../test_ts_index_add_bitmap.groovy} | 4 +- .../test_ts_index_add_inverted.groovy | 137 ++++++++++++++++++ .../test_ts_mv_create_drop.groovy} | 4 +- .../partition/add/test_ts_part_add.groovy} | 4 +- .../add_drop/test_tbl_part_add_drop.groovy} | 4 +- .../test_ts_part_clean_restore.groovy} | 4 +- .../replace/test_ts_part_replace.groovy} | 4 +- .../test_ts_part_replace_partial.groovy} | 4 +- .../rollup/add/test_ts_rollup_add.groovy} | 4 +- .../test_ts_table_modify_comment.groovy} | 2 +- .../table/rename/test_ts_tbl_rename.groovy} | 4 +- .../test_ts_tbl_res_auto_bucket.groovy} | 4 +- .../test_ts_tbl_res_inverted_idx.groovy} | 6 +- .../res_mow/test_ts_table_res_mow.groovy} | 4 +- .../test_ts_tbl_res_row_storage.groovy} | 4 +- .../test_ts_tbl_res_variant.groovy} | 4 +- .../truncate/test_ts_tbl_truncate.groovy} | 4 +- .../add_value/test_tsa_column_add.groovy} | 4 +- 66 files changed, 443 insertions(+), 276 deletions(-) delete mode 100644 regression-test/data/ccr_user_sync/test_common_sync.out rename regression-test/data/{usercases/cir_8537.out => db_sync/partition/drop_1/test_ds_part_drop_1.out} (100%) rename regression-test/data/{table-sync/test_insert_overwrite.out => table_sync/dml/insert_overwrite/test_ts_dml_insert_overwrite.out} (100%) rename regression-test/data/{table-sync/test_inverted_index.out => table_sync/table/res_inverted_idx/test_ts_tbl_res_inverted_idx.out} (100%) rename regression-test/suites/{db-sync-fullsync-with-alias/test_db_sync_fullsync_with_alias.groovy => cross_ds/fullsync_with_alias/test_cds_fullsync_with_alias.groovy} (99%) rename regression-test/suites/{db-sync-signature-not-matched/test_db_sync_signature_not_matched.groovy => cross_ds/signature_not_matched/test_cds_signature_not_matched.groovy} (97%) rename regression-test/suites/{db-sv/test_sync_view_twice.groovy => cross_ds/sync_view_twice/test_cds_sync_view_twice.groovy} (98%) rename regression-test/suites/{table-sync/test_keyword_name.groovy => cross_ts/keyword_name/test_cts_keyword.groovy} (99%) rename regression-test/suites/{db-ps-inc => db_ps_inc}/add_partition/test_db_partial_sync_inc_add_partition.groovy (97%) rename regression-test/suites/{db-ps-inc => db_ps_inc}/alter/test_db_partial_sync_inc_alter.groovy (97%) rename regression-test/suites/{db-partial-sync-cache => db_ps_inc/cache}/test_db_partial_sync_cache.groovy (74%) rename regression-test/suites/{db-ps-inc => db_ps_inc}/drop_partition/test_db_partial_sync_inc_drop_partition.groovy (97%) rename regression-test/suites/{db-ps-inc => db_ps_inc}/lightning_sc/test_db_partial_sync_inc_lightning_sc.groovy (97%) rename regression-test/suites/{db-ps-inc => db_ps_inc}/merge/test_db_partial_sync_merge.groovy (97%) rename regression-test/suites/{db-ps-inc => db_ps_inc}/replace_partition/test_db_partial_sync_inc_replace_partition.groovy (97%) rename regression-test/suites/{db-ps-inc => db_ps_inc}/truncate_table/test_db_partial_sync_inc_trunc_table.groovy (97%) rename regression-test/suites/{db-ps-inc => db_ps_inc}/upsert/test_db_partial_sync_inc_upsert.groovy (96%) rename regression-test/suites/{db-sync-schema-change/test_db_sync_schema_change.groovy => db_sync/column/basic/test_ds_col_basic.groovy} (82%) rename regression-test/suites/{db-sync-common/test_db_sync.groovy => db_sync/common/test_ds_common.groovy} (99%) rename regression-test/suites/{db-sync-insert-overwrite/test_db_insert_overwrite.groovy => db_sync/dml/insert_overwrite/test_ds_dml_insert_overwrite.groovy} (97%) rename regression-test/suites/{db-sv-and-mv/test_view_and_mv.groovy => db_sync/mv/basic/test_ds_mv_basic.groovy} (97%) rename regression-test/suites/{db-sync-drop-partition/test_drop_partition.groovy => db_sync/partition/drop/test_ds_part_drop.groovy} (97%) rename regression-test/suites/{usercases/cir_8537.groovy => db_sync/partition/drop_1/test_ds_part_drop_1.groovy} (97%) rename regression-test/suites/{db-sync-replace-partition/test_db_sync_replace_partition.groovy => db_sync/partition/replace/test_ds_part_replace.groovy} (97%) rename regression-test/suites/{db-sync-clean-restore/test_db_sync_clean_restore.groovy => db_sync/table/clean_restore/test_ds_clean_restore.groovy} (98%) rename regression-test/suites/{db-sync-add-drop-table/test_db_sync_add_drop_table.groovy => db_sync/table/create_drop/test_ds_tbl_create_drop.groovy} (97%) create mode 100644 regression-test/suites/db_sync/table/drop_create/test_ds_tbl_drop_create.groovy rename regression-test/suites/{db-sync-rename-table/test_db_sync_rename_table.groovy => db_sync/table/rename/test_ds_tbl_rename.groovy} (97%) rename regression-test/suites/{db-sync-truncate-table/test_db_sync_truncate_table.groovy => db_sync/table/truncate/test_ds_tbl_truncate.groovy} (96%) create mode 100644 regression-test/suites/db_sync/view/basic/test_ds_view_basic.groovy rename regression-test/suites/{db-sv-drop-create/test_sync_view_drop_create.groovy => db_sync/view/drop_create/test_ds_view_drop_create.groovy} (97%) rename regression-test/suites/{db-sv-drop-delete-create/test_sync_view_drop_delete_create.groovy => db_sync/view/drop_delete_create/test_ds_view_drop_delete_create.groovy} (97%) rename regression-test/suites/{table-sync/test_allow_table_exists.groovy => syncer/ts_allow_table_exists/test_syncer_ts_allow_table_exists.groovy} (97%) rename regression-test/suites/{table-partial-sync/test_table_partial_sync_incremental.groovy => table_ps_inc/basic/test_tbl_ps_inc_basic.groovy} (96%) rename regression-test/suites/{table-partial-sync/test_table_partial_sync_cache.groovy => table_ps_inc/cache/test_tbl_ps_inc_cache.groovy} (70%) rename regression-test/suites/{table-schema-change/test_add_column.groovy => table_sync/column/add/test_ts_col_add.groovy} (82%) rename regression-test/suites/{table-schema-change/test_add_agg_column.groovy => table_sync/column/add_agg/test_ts_col_add_agg.groovy} (98%) rename regression-test/suites/{table-schema-change/test_add_many_column.groovy => table_sync/column/add_many/test_ts_col_add_many.groovy} (97%) rename regression-test/suites/{table-schema-change/test_alter_type.groovy => table_sync/column/alter_type/test_ts_col_alter_type.groovy} (98%) rename regression-test/suites/{table-sync/test_column_ops.groovy => table_sync/column/basic/test_ts_col_basic.groovy} (97%) rename regression-test/suites/{table-schema-change/test_drop_column.groovy => table_sync/column/drop/test_ts_col_drop.groovy} (98%) rename regression-test/suites/{table-schema-change/test_filter_dropped_indexes.groovy => table_sync/column/filter_dropped_indexes/test_ts_col_filter_dropped_indexes.groovy} (96%) rename regression-test/suites/{table-schema-change/test_order_by.groovy => table_sync/column/order_by/test_ts_col_order_by.groovy} (97%) rename regression-test/suites/{table-sync/test_common.groovy => table_sync/common/test_ts_common.groovy} (98%) rename regression-test/suites/{table-sync/test_delete.groovy => table_sync/dml/delete/test_ts_dml_delete.groovy} (98%) rename regression-test/suites/{table-sync/test_insert_overwrite.groovy => table_sync/dml/insert_overwrite/test_ts_dml_insert_overwrite.groovy} (97%) rename regression-test/suites/{table-sync/test_bloomfilter_index.groovy => table_sync/index/add_bf/test_tbl_index_add_bloom_filter.groovy} (97%) rename regression-test/suites/{table-sync/test_bitmap_index.groovy => table_sync/index/add_bitmap/test_ts_index_add_bitmap.groovy} (97%) create mode 100644 regression-test/suites/table_sync/index/add_inverted/test_ts_index_add_inverted.groovy rename regression-test/suites/{table-sync/test_materialized_view.groovy => table_sync/mv/create_drop/test_ts_mv_create_drop.groovy} (97%) rename regression-test/suites/{table-sync/test_add_partition.groovy => table_sync/partition/add/test_ts_part_add.groovy} (98%) rename regression-test/suites/{table-sync/test_partition_ops.groovy => table_sync/partition/add_drop/test_tbl_part_add_drop.groovy} (97%) rename regression-test/suites/{table-sync/test_restore_clean_partitions.groovy => table_sync/partition/clean_restore/test_ts_part_clean_restore.groovy} (97%) rename regression-test/suites/{table-sync/test_replace_partition.groovy => table_sync/partition/replace/test_ts_part_replace.groovy} (98%) rename regression-test/suites/{table-sync/test_replace_partial_partition.groovy => table_sync/partition/replace_partial/test_ts_part_replace_partial.groovy} (97%) rename regression-test/suites/{table-sync/test_rollup.groovy => table_sync/rollup/add/test_ts_rollup_add.groovy} (96%) rename regression-test/suites/{table-sync/test_table_comment.groovy => table_sync/table/modify_comment/test_ts_table_modify_comment.groovy} (98%) rename regression-test/suites/{table-sync/test_rename.groovy => table_sync/table/rename/test_ts_tbl_rename.groovy} (97%) rename regression-test/suites/{table-sync/test_auto_bucket.groovy => table_sync/table/res_auto_bucket/test_ts_tbl_res_auto_bucket.groovy} (96%) rename regression-test/suites/{table-sync/test_inverted_index.groovy => table_sync/table/res_inverted_idx/test_ts_tbl_res_inverted_idx.groovy} (97%) rename regression-test/suites/{table-sync/test_mow.groovy => table_sync/table/res_mow/test_ts_table_res_mow.groovy} (97%) rename regression-test/suites/{table-sync/test_row_storage.groovy => table_sync/table/res_row_storage/test_ts_tbl_res_row_storage.groovy} (96%) rename regression-test/suites/{table-sync/test_variant.groovy => table_sync/table/res_variant/test_ts_tbl_res_variant.groovy} (96%) rename regression-test/suites/{table-sync/test_truncate_table.groovy => table_sync/table/truncate/test_ts_tbl_truncate.groovy} (97%) rename regression-test/suites/{table-sync-with-alias/test_lightning_schema_change.groovy => table_sync_alias/column/add_value/test_tsa_column_add.groovy} (96%) diff --git a/regression-test/common/helper.groovy b/regression-test/common/helper.groovy index 32692d6a..ba41f4fc 100644 --- a/regression-test/common/helper.groovy +++ b/regression-test/common/helper.groovy @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +import com.google.common.collect.Maps + +import java.util.Map + class Helper { def suite def context @@ -31,11 +35,40 @@ class Helper { } String randomSuffix() { - return UUID.randomUUID().toString().replace("-", "") + def hashCode = UUID.randomUUID().toString().replace("-", "").hashCode() + if (hashCode < 0) { + hashCode *= -1; + } + return Integer.toString(hashCode) + } + + def get_ccr_body(String table, String db = null) { + if (db == null) { + db = context.dbName + } + + def gson = new com.google.gson.Gson() + + Map srcSpec = context.getSrcSpec(db) + srcSpec.put("table", table) + + Map destSpec = context.getDestSpec(db) + destSpec.put("table", table) + + Map body = Maps.newHashMap() + String name = context.suiteName + if (!table.equals("")) { + name = name + "_" + table + } + body.put("name", name) + body.put("src", srcSpec) + body.put("dest", destSpec) + + return gson.toJson(body) } void ccrJobDelete(table = "") { - def bodyJson = suite.get_ccr_body "${table}" + def bodyJson = get_ccr_body "${table}" suite.httpTest { uri "/delete" endpoint syncerAddress @@ -45,7 +78,7 @@ class Helper { } void ccrJobCreate(table = "") { - def bodyJson = suite.get_ccr_body "${table}" + def bodyJson = get_ccr_body "${table}" suite.httpTest { uri "/create_ccr" endpoint syncerAddress @@ -55,7 +88,7 @@ class Helper { } void ccrJobCreateAllowTableExists(table = "") { - def bodyJson = suite.get_ccr_body "${table}" + def bodyJson = get_ccr_body "${table}" def jsonSlurper = new groovy.json.JsonSlurper() def object = jsonSlurper.parseText "${bodyJson}" object['allow_table_exists'] = true @@ -71,7 +104,7 @@ class Helper { } void ccrJobPause(table = "") { - def bodyJson = suite.get_ccr_body "${table}" + def bodyJson = get_ccr_body "${table}" suite.httpTest { uri "/pause" endpoint syncerAddress @@ -81,7 +114,7 @@ class Helper { } void ccrJobResume(table = "") { - def bodyJson = suite.get_ccr_body "${table}" + def bodyJson = get_ccr_body "${table}" suite.httpTest { uri "/resume" endpoint syncerAddress @@ -91,7 +124,7 @@ class Helper { } void ccrJobDesync(table = "") { - def bodyJson = suite.get_ccr_body "${table}" + def bodyJson = get_ccr_body "${table}" suite.httpTest { uri "/desync" endpoint syncerAddress @@ -221,7 +254,7 @@ class Helper { } void force_fullsync(tableName = "") { - def bodyJson = suite.get_ccr_body "${tableName}" + def bodyJson = get_ccr_body "${tableName}" suite.httpTest { uri "/force_fullsync" endpoint syncerAddress @@ -231,7 +264,7 @@ class Helper { } Object get_job_progress(tableName = "") { - def request_body = suite.get_ccr_body(tableName) + def request_body = get_ccr_body(tableName) def get_job_progress_uri = { check_func -> suite.httpTest { uri "/job_progress" diff --git a/regression-test/data/ccr_user_sync/test_common_sync.out b/regression-test/data/ccr_user_sync/test_common_sync.out deleted file mode 100644 index c33d3b5c..00000000 --- a/regression-test/data/ccr_user_sync/test_common_sync.out +++ /dev/null @@ -1 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this diff --git a/regression-test/data/usercases/cir_8537.out b/regression-test/data/db_sync/partition/drop_1/test_ds_part_drop_1.out similarity index 100% rename from regression-test/data/usercases/cir_8537.out rename to regression-test/data/db_sync/partition/drop_1/test_ds_part_drop_1.out diff --git a/regression-test/data/table-sync/test_insert_overwrite.out b/regression-test/data/table_sync/dml/insert_overwrite/test_ts_dml_insert_overwrite.out similarity index 100% rename from regression-test/data/table-sync/test_insert_overwrite.out rename to regression-test/data/table_sync/dml/insert_overwrite/test_ts_dml_insert_overwrite.out diff --git a/regression-test/data/table-sync/test_inverted_index.out b/regression-test/data/table_sync/table/res_inverted_idx/test_ts_tbl_res_inverted_idx.out similarity index 100% rename from regression-test/data/table-sync/test_inverted_index.out rename to regression-test/data/table_sync/table/res_inverted_idx/test_ts_tbl_res_inverted_idx.out diff --git a/regression-test/suites/db-sync-fullsync-with-alias/test_db_sync_fullsync_with_alias.groovy b/regression-test/suites/cross_ds/fullsync_with_alias/test_cds_fullsync_with_alias.groovy similarity index 99% rename from regression-test/suites/db-sync-fullsync-with-alias/test_db_sync_fullsync_with_alias.groovy rename to regression-test/suites/cross_ds/fullsync_with_alias/test_cds_fullsync_with_alias.groovy index 97ef2eca..212c3448 100644 --- a/regression-test/suites/db-sync-fullsync-with-alias/test_db_sync_fullsync_with_alias.groovy +++ b/regression-test/suites/cross_ds/fullsync_with_alias/test_cds_fullsync_with_alias.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_db_sync_fullsync_with_alias") { +suite("test_cds_fullsync_with_alias") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) diff --git a/regression-test/suites/db-sync-signature-not-matched/test_db_sync_signature_not_matched.groovy b/regression-test/suites/cross_ds/signature_not_matched/test_cds_signature_not_matched.groovy similarity index 97% rename from regression-test/suites/db-sync-signature-not-matched/test_db_sync_signature_not_matched.groovy rename to regression-test/suites/cross_ds/signature_not_matched/test_cds_signature_not_matched.groovy index aea2cc8b..6ee38190 100644 --- a/regression-test/suites/db-sync-signature-not-matched/test_db_sync_signature_not_matched.groovy +++ b/regression-test/suites/cross_ds/signature_not_matched/test_cds_signature_not_matched.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_db_sync_signature_not_matched") { +suite("test_cds_signature_not_matched") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_db_sync_sig_not_matched_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 20 def opPartitonName = "less" diff --git a/regression-test/suites/db-sv/test_sync_view_twice.groovy b/regression-test/suites/cross_ds/sync_view_twice/test_cds_sync_view_twice.groovy similarity index 98% rename from regression-test/suites/db-sv/test_sync_view_twice.groovy rename to regression-test/suites/cross_ds/sync_view_twice/test_cds_sync_view_twice.groovy index fdef937d..d9fb13f7 100644 --- a/regression-test/suites/db-sv/test_sync_view_twice.groovy +++ b/regression-test/suites/cross_ds/sync_view_twice/test_cds_sync_view_twice.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_sync_view_twice") { +suite("test_cds_sync_view_twice") { def versions = sql_return_maparray "show variables like 'version_comment'" if (versions[0].Value.contains('doris-2.0.') || versions[0].Value.contains('doris-2.1.')) { logger.info("2.0/2.1 not support this case, current version is: ${versions[0].Value}") @@ -96,3 +96,4 @@ suite("test_sync_view_twice") { def view_size = target_sql "SHOW VIEW FROM ${tableDuplicate0}" assertTrue(view_size.size() == 1); } + diff --git a/regression-test/suites/table-sync/test_keyword_name.groovy b/regression-test/suites/cross_ts/keyword_name/test_cts_keyword.groovy similarity index 99% rename from regression-test/suites/table-sync/test_keyword_name.groovy rename to regression-test/suites/cross_ts/keyword_name/test_cts_keyword.groovy index 8bb5f6fd..ea9a0779 100644 --- a/regression-test/suites/table-sync/test_keyword_name.groovy +++ b/regression-test/suites/cross_ts/keyword_name/test_cts_keyword.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_keyword_name") { +suite("test_cts_keyword_name") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) diff --git a/regression-test/suites/db-ps-inc/add_partition/test_db_partial_sync_inc_add_partition.groovy b/regression-test/suites/db_ps_inc/add_partition/test_db_partial_sync_inc_add_partition.groovy similarity index 97% rename from regression-test/suites/db-ps-inc/add_partition/test_db_partial_sync_inc_add_partition.groovy rename to regression-test/suites/db_ps_inc/add_partition/test_db_partial_sync_inc_add_partition.groovy index 11c693e6..1705fb6a 100644 --- a/regression-test/suites/db-ps-inc/add_partition/test_db_partial_sync_inc_add_partition.groovy +++ b/regression-test/suites/db_ps_inc/add_partition/test_db_partial_sync_inc_add_partition.groovy @@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_add_partition") { return } - def tableName = "tbl_" + UUID.randomUUID().toString().replace("-", "") - def tableName1 = "tbl_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() + def tableName1 = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/db-ps-inc/alter/test_db_partial_sync_inc_alter.groovy b/regression-test/suites/db_ps_inc/alter/test_db_partial_sync_inc_alter.groovy similarity index 97% rename from regression-test/suites/db-ps-inc/alter/test_db_partial_sync_inc_alter.groovy rename to regression-test/suites/db_ps_inc/alter/test_db_partial_sync_inc_alter.groovy index b1c010fa..7c5da37b 100644 --- a/regression-test/suites/db-ps-inc/alter/test_db_partial_sync_inc_alter.groovy +++ b/regression-test/suites/db_ps_inc/alter/test_db_partial_sync_inc_alter.groovy @@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_alter") { return } - def tableName = "tbl_" + UUID.randomUUID().toString().replace("-", "") - def tableName1 = "tbl_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() + def tableName1 = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/db-partial-sync-cache/test_db_partial_sync_cache.groovy b/regression-test/suites/db_ps_inc/cache/test_db_partial_sync_cache.groovy similarity index 74% rename from regression-test/suites/db-partial-sync-cache/test_db_partial_sync_cache.groovy rename to regression-test/suites/db_ps_inc/cache/test_db_partial_sync_cache.groovy index 54d45f20..a64afe09 100644 --- a/regression-test/suites/db-partial-sync-cache/test_db_partial_sync_cache.groovy +++ b/regression-test/suites/db_ps_inc/cache/test_db_partial_sync_cache.groovy @@ -18,7 +18,7 @@ suite("test_db_partial_sync_cache") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_partial_sync_cache_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 @@ -38,34 +38,6 @@ suite("test_db_partial_sync_cache") { return object.name } - def get_job_progress = { ccr_name -> - def request_body = """ {"name":"${ccr_name}"} """ - def get_job_progress_uri = { check_func -> - httpTest { - uri "/job_progress" - endpoint helper.syncerAddress - body request_body - op "post" - check check_func - } - } - - def result = null - get_job_progress_uri.call() { code, body -> - if (!"${code}".toString().equals("200")) { - throw "request failed, code: ${code}, body: ${body}" - } - def jsonSlurper = new groovy.json.JsonSlurper() - def object = jsonSlurper.parseText "${body}" - if (!object.success) { - throw "request failed, error msg: ${object.error_msg}" - } - logger.info("job progress: ${object.job_progress}") - result = jsonSlurper.parseText object.job_progress - } - return result - } - helper.enableDbBinlog() sql "DROP TABLE IF EXISTS ${tableName}" sql """ @@ -93,15 +65,12 @@ suite("test_db_partial_sync_cache") { """ sql "sync" - def bodyJson = get_ccr_body "" - ccr_name = get_ccr_name(bodyJson) helper.ccrJobCreate() - logger.info("ccr job name: ${ccr_name}") assertTrue(helper.checkRestoreFinishTimesOf("${tableName}", 30)) assertTrue(helper.checkSelectTimesOf("SELECT * FROM ${tableName}", insert_num, 60)) - first_job_progress = get_job_progress(ccr_name) + first_job_progress = helper.get_job_progress() logger.info("=== Test 1: add first column case ===") // binlog type: ALTER_JOB, binlog data: @@ -140,7 +109,7 @@ suite("test_db_partial_sync_cache") { assertTrue(helper.checkSelectTimesOf("SELECT * FROM ${tableName}", insert_num + 1, 60)) // no full sync triggered. - last_job_progress = get_job_progress(ccr_name) + last_job_progress = helper.get_job_progress() assertTrue(last_job_progress.full_sync_start_at == first_job_progress.full_sync_start_at) } diff --git a/regression-test/suites/db-ps-inc/drop_partition/test_db_partial_sync_inc_drop_partition.groovy b/regression-test/suites/db_ps_inc/drop_partition/test_db_partial_sync_inc_drop_partition.groovy similarity index 97% rename from regression-test/suites/db-ps-inc/drop_partition/test_db_partial_sync_inc_drop_partition.groovy rename to regression-test/suites/db_ps_inc/drop_partition/test_db_partial_sync_inc_drop_partition.groovy index 7149b046..0727dcd5 100644 --- a/regression-test/suites/db-ps-inc/drop_partition/test_db_partial_sync_inc_drop_partition.groovy +++ b/regression-test/suites/db_ps_inc/drop_partition/test_db_partial_sync_inc_drop_partition.groovy @@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_drop_partition") { return } - def tableName = "tbl_" + UUID.randomUUID().toString().replace("-", "") - def tableName1 = "tbl_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() + def tableName1 = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/db-ps-inc/lightning_sc/test_db_partial_sync_inc_lightning_sc.groovy b/regression-test/suites/db_ps_inc/lightning_sc/test_db_partial_sync_inc_lightning_sc.groovy similarity index 97% rename from regression-test/suites/db-ps-inc/lightning_sc/test_db_partial_sync_inc_lightning_sc.groovy rename to regression-test/suites/db_ps_inc/lightning_sc/test_db_partial_sync_inc_lightning_sc.groovy index 168bdcdf..a9e7151a 100644 --- a/regression-test/suites/db-ps-inc/lightning_sc/test_db_partial_sync_inc_lightning_sc.groovy +++ b/regression-test/suites/db_ps_inc/lightning_sc/test_db_partial_sync_inc_lightning_sc.groovy @@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_lightning_sc") { return } - def tableName = "tbl_" + UUID.randomUUID().toString().replace("-", "") - def tableName1 = "tbl_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() + def tableName1 = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/db-ps-inc/merge/test_db_partial_sync_merge.groovy b/regression-test/suites/db_ps_inc/merge/test_db_partial_sync_merge.groovy similarity index 97% rename from regression-test/suites/db-ps-inc/merge/test_db_partial_sync_merge.groovy rename to regression-test/suites/db_ps_inc/merge/test_db_partial_sync_merge.groovy index 3dd38524..25eb6077 100644 --- a/regression-test/suites/db-ps-inc/merge/test_db_partial_sync_merge.groovy +++ b/regression-test/suites/db_ps_inc/merge/test_db_partial_sync_merge.groovy @@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_merge") { return } - def tableName = "tbl_" + UUID.randomUUID().toString().replace("-", "") - def tableName1 = "tbl_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() + def tableName1 = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/db-ps-inc/replace_partition/test_db_partial_sync_inc_replace_partition.groovy b/regression-test/suites/db_ps_inc/replace_partition/test_db_partial_sync_inc_replace_partition.groovy similarity index 97% rename from regression-test/suites/db-ps-inc/replace_partition/test_db_partial_sync_inc_replace_partition.groovy rename to regression-test/suites/db_ps_inc/replace_partition/test_db_partial_sync_inc_replace_partition.groovy index 1cf5b83f..6ae48fec 100644 --- a/regression-test/suites/db-ps-inc/replace_partition/test_db_partial_sync_inc_replace_partition.groovy +++ b/regression-test/suites/db_ps_inc/replace_partition/test_db_partial_sync_inc_replace_partition.groovy @@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_replace_partition") { return } - def tableName = "tbl_" + UUID.randomUUID().toString().replace("-", "") - def tableName1 = "tbl_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() + def tableName1 = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/db-ps-inc/truncate_table/test_db_partial_sync_inc_trunc_table.groovy b/regression-test/suites/db_ps_inc/truncate_table/test_db_partial_sync_inc_trunc_table.groovy similarity index 97% rename from regression-test/suites/db-ps-inc/truncate_table/test_db_partial_sync_inc_trunc_table.groovy rename to regression-test/suites/db_ps_inc/truncate_table/test_db_partial_sync_inc_trunc_table.groovy index 7a3d3c5f..de5819aa 100644 --- a/regression-test/suites/db-ps-inc/truncate_table/test_db_partial_sync_inc_trunc_table.groovy +++ b/regression-test/suites/db_ps_inc/truncate_table/test_db_partial_sync_inc_trunc_table.groovy @@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_trunc_table") { return } - def tableName = "tbl_" + UUID.randomUUID().toString().replace("-", "") - def tableName1 = "tbl_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() + def tableName1 = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/db-ps-inc/upsert/test_db_partial_sync_inc_upsert.groovy b/regression-test/suites/db_ps_inc/upsert/test_db_partial_sync_inc_upsert.groovy similarity index 96% rename from regression-test/suites/db-ps-inc/upsert/test_db_partial_sync_inc_upsert.groovy rename to regression-test/suites/db_ps_inc/upsert/test_db_partial_sync_inc_upsert.groovy index 6bb000ba..975bc909 100644 --- a/regression-test/suites/db-ps-inc/upsert/test_db_partial_sync_inc_upsert.groovy +++ b/regression-test/suites/db_ps_inc/upsert/test_db_partial_sync_inc_upsert.groovy @@ -23,8 +23,8 @@ suite("test_db_partial_sync_inc_upsert") { return } - def tableName = "tbl_sync_incremental_" + UUID.randomUUID().toString().replace("-", "") - def tableName1 = "tbl_sync_incremental_1_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() + def tableName1 = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/db-sync-schema-change/test_db_sync_schema_change.groovy b/regression-test/suites/db_sync/column/basic/test_ds_col_basic.groovy similarity index 82% rename from regression-test/suites/db-sync-schema-change/test_db_sync_schema_change.groovy rename to regression-test/suites/db_sync/column/basic/test_ds_col_basic.groovy index d6e8a96c..7c7039b5 100644 --- a/regression-test/suites/db-sync-schema-change/test_db_sync_schema_change.groovy +++ b/regression-test/suites/db_sync/column/basic/test_ds_col_basic.groovy @@ -14,11 +14,16 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_db_sync_schema_change") { +suite("test_ds_col_basic") { + // 1. add first key column + // 2. add last key column + // 3. add value column + // 4. add last value column + def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_add_column_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 @@ -32,40 +37,6 @@ suite("test_db_sync_schema_change") { } } - def get_ccr_name = { ccr_body_json -> - def jsonSlurper = new groovy.json.JsonSlurper() - def object = jsonSlurper.parseText "${ccr_body_json}" - return object.name - } - - def get_job_progress = { ccr_name -> - def request_body = """ {"name":"${ccr_name}"} """ - def get_job_progress_uri = { check_func -> - httpTest { - uri "/job_progress" - endpoint helper.syncerAddress - body request_body - op "post" - check check_func - } - } - - def result = null - get_job_progress_uri.call() { code, body -> - if (!"${code}".toString().equals("200")) { - throw "request failed, code: ${code}, body: ${body}" - } - def jsonSlurper = new groovy.json.JsonSlurper() - def object = jsonSlurper.parseText "${body}" - if (!object.success) { - throw "request failed, error msg: ${object.error_msg}" - } - logger.info("job progress: ${object.job_progress}") - result = jsonSlurper.parseText object.job_progress - } - return result - } - helper.enableDbBinlog() sql "DROP TABLE IF EXISTS ${tableName}" sql """ @@ -94,14 +65,11 @@ suite("test_db_sync_schema_change") { sql "sync" helper.ccrJobDelete() - def bodyJson = get_ccr_body "" - ccr_name = get_ccr_name(bodyJson) helper.ccrJobCreate() - logger.info("ccr job name: ${ccr_name}") assertTrue(helper.checkRestoreFinishTimesOf("${tableName}", 30)) - first_job_progress = get_job_progress(ccr_name) + first_job_progress = helper.get_job_progress() logger.info("=== Test 1: add first column case ===") // binlog type: ALTER_JOB, binlog data: @@ -230,7 +198,7 @@ suite("test_db_sync_schema_change") { assertTrue(helper.checkShowTimesOf("SHOW COLUMNS FROM `${tableName}`", has_column_last_value, 60, "target_sql")) // no full sync triggered. - last_job_progress = get_job_progress(ccr_name) + last_job_progress = helper.get_job_progress() assertTrue(last_job_progress.full_sync_start_at == first_job_progress.full_sync_start_at) } diff --git a/regression-test/suites/db-sync-common/test_db_sync.groovy b/regression-test/suites/db_sync/common/test_ds_common.groovy similarity index 99% rename from regression-test/suites/db-sync-common/test_db_sync.groovy rename to regression-test/suites/db_sync/common/test_ds_common.groovy index 36e642a1..bcddb664 100644 --- a/regression-test/suites/db-sync-common/test_db_sync.groovy +++ b/regression-test/suites/db_sync/common/test_ds_common.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_db_sync") { +suite("test_ds_common") { def versions = sql_return_maparray "show variables like 'version_comment'" if (versions[0].Value.contains('doris-2.0.')) { logger.info("2.0 not support AUTO PARTITION, current version is: ${versions[0].Value}") diff --git a/regression-test/suites/db-sync-insert-overwrite/test_db_insert_overwrite.groovy b/regression-test/suites/db_sync/dml/insert_overwrite/test_ds_dml_insert_overwrite.groovy similarity index 97% rename from regression-test/suites/db-sync-insert-overwrite/test_db_insert_overwrite.groovy rename to regression-test/suites/db_sync/dml/insert_overwrite/test_ds_dml_insert_overwrite.groovy index e784dcd3..15ff805e 100644 --- a/regression-test/suites/db-sync-insert-overwrite/test_db_insert_overwrite.groovy +++ b/regression-test/suites/db_sync/dml/insert_overwrite/test_ds_dml_insert_overwrite.groovy @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_db_insert_overwrite") { +suite("test_ds_dml_insert_overwrite") { def versions = sql_return_maparray "show variables like 'version_comment'" if (versions[0].Value.contains('doris-2.0.')) { logger.info("2.0 not support INSERT OVERWRITE yet, current version is: ${versions[0].Value}") @@ -33,7 +33,7 @@ suite("test_db_insert_overwrite") { // 1. create temp partitions // 2. insert into temp partitions // 3. replace overlap partitions - def tableName = "tbl_insert_overwrite_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def uniqueTable = "${tableName}_unique" def test_num = 0 def insert_num = 5 @@ -126,3 +126,4 @@ suite("test_db_insert_overwrite") { assertTrue(helper.checkSelectTimesOf("SELECT * FROM ${uniqueTable}", 5, 60)) } + diff --git a/regression-test/suites/db-sv-and-mv/test_view_and_mv.groovy b/regression-test/suites/db_sync/mv/basic/test_ds_mv_basic.groovy similarity index 97% rename from regression-test/suites/db-sv-and-mv/test_view_and_mv.groovy rename to regression-test/suites/db_sync/mv/basic/test_ds_mv_basic.groovy index 09bcb6c5..23200255 100644 --- a/regression-test/suites/db-sv-and-mv/test_view_and_mv.groovy +++ b/regression-test/suites/db_sync/mv/basic/test_ds_mv_basic.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_view_and_mv") { +suite("test_ds_mv_basic") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) @@ -59,7 +59,7 @@ suite("test_view_and_mv") { return res.size() == 0 } - def suffix = UUID.randomUUID().toString().replace("-", "") + def suffix = helper.randomSuffix() def tableDuplicate0 = "tbl_duplicate_0_${suffix}" createDuplicateTable(tableDuplicate0) sql """ @@ -109,3 +109,4 @@ suite("test_view_and_mv") { assertTrue(helper.checkSelectTimesOf("SELECT * FROM ${tableDuplicate0}", 5, 5)) } + diff --git a/regression-test/suites/db-sync-drop-partition/test_drop_partition.groovy b/regression-test/suites/db_sync/partition/drop/test_ds_part_drop.groovy similarity index 97% rename from regression-test/suites/db-sync-drop-partition/test_drop_partition.groovy rename to regression-test/suites/db_sync/partition/drop/test_ds_part_drop.groovy index 5d65cd2f..573d3841 100644 --- a/regression-test/suites/db-sync-drop-partition/test_drop_partition.groovy +++ b/regression-test/suites/db_sync/partition/drop/test_ds_part_drop.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_drop_partition_without_fullsync") { +suite("test_ds_part_drop") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_partition_ops_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 90 // insert into last partition def opPartitonName = "less" @@ -142,3 +142,4 @@ suite("test_drop_partition_without_fullsync") { assertTrue(show_backup_result.size() == backup_num) } + diff --git a/regression-test/suites/usercases/cir_8537.groovy b/regression-test/suites/db_sync/partition/drop_1/test_ds_part_drop_1.groovy similarity index 97% rename from regression-test/suites/usercases/cir_8537.groovy rename to regression-test/suites/db_sync/partition/drop_1/test_ds_part_drop_1.groovy index 33e2aad0..76761d18 100644 --- a/regression-test/suites/usercases/cir_8537.groovy +++ b/regression-test/suites/db_sync/partition/drop_1/test_ds_part_drop_1.groovy @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("usercases_cir_8537") { +suite("test_ds_part_drop_1") { // Case description // Insert data and drop a partition, then the ccr syncer wouldn't get the partition ids from the source cluster. @@ -22,7 +22,7 @@ suite("usercases_cir_8537") { .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) def caseName = "usercases_cir_8537" - def tableName = "${caseName}_sales_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def syncerAddress = "127.0.0.1:9190" def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/db-sync-replace-partition/test_db_sync_replace_partition.groovy b/regression-test/suites/db_sync/partition/replace/test_ds_part_replace.groovy similarity index 97% rename from regression-test/suites/db-sync-replace-partition/test_db_sync_replace_partition.groovy rename to regression-test/suites/db_sync/partition/replace/test_ds_part_replace.groovy index fe4be10e..d7333a33 100644 --- a/regression-test/suites/db-sync-replace-partition/test_db_sync_replace_partition.groovy +++ b/regression-test/suites/db_sync/partition/replace/test_ds_part_replace.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_db_sync_replace_partition") { +suite("test_ds_part_replace") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def baseTableName = "test_replace_partition_" + UUID.randomUUID().toString().replace("-", "") + def baseTableName = "tbl_replace_partition_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 def opPartitonName = "less0" @@ -153,3 +153,4 @@ suite("test_db_sync_replace_partition") { assertTrue(object.olap_table_list[0].partition_names.size() == 1) assertTrue(object.olap_table_list[0].partition_names[0] == "p2"); } + diff --git a/regression-test/suites/db-sync-clean-restore/test_db_sync_clean_restore.groovy b/regression-test/suites/db_sync/table/clean_restore/test_ds_clean_restore.groovy similarity index 98% rename from regression-test/suites/db-sync-clean-restore/test_db_sync_clean_restore.groovy rename to regression-test/suites/db_sync/table/clean_restore/test_ds_clean_restore.groovy index 018f2479..3804944e 100644 --- a/regression-test/suites/db-sync-clean-restore/test_db_sync_clean_restore.groovy +++ b/regression-test/suites/db_sync/table/clean_restore/test_ds_clean_restore.groovy @@ -15,14 +15,14 @@ // specific language governing permissions and limitations // under the License. -suite("test_db_sync_clean_restore") { +suite("test_ds_clean_restore") { // FIXME(walter) fix clean tables. return def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_db_sync_clean_restore_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 20 def opPartitonName = "less" diff --git a/regression-test/suites/db-sync-add-drop-table/test_db_sync_add_drop_table.groovy b/regression-test/suites/db_sync/table/create_drop/test_ds_tbl_create_drop.groovy similarity index 97% rename from regression-test/suites/db-sync-add-drop-table/test_db_sync_add_drop_table.groovy rename to regression-test/suites/db_sync/table/create_drop/test_ds_tbl_create_drop.groovy index 37d3fa86..3349de28 100644 --- a/regression-test/suites/db-sync-add-drop-table/test_db_sync_add_drop_table.groovy +++ b/regression-test/suites/db_sync/table/create_drop/test_ds_tbl_create_drop.groovy @@ -14,12 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. - -suite("test_db_sync_add_drop_table") { +suite("test_ds_tbl_create_drop") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_db_sync_add_drop_table_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 10 def opPartitonName = "less" @@ -58,7 +57,6 @@ suite("test_db_sync_add_drop_table") { assertTrue(helper.checkRestoreFinishTimesOf("${tableName}_1", 60)) - logger.info("=== Test 1: Check table and backup size ===") sql "sync" assertTrue(helper.checkShowTimesOf(""" SHOW TABLES LIKE "${tableName}_1" """, exist, 60, "target")) @@ -135,4 +133,3 @@ suite("test_db_sync_add_drop_table") { logger.info("backups after drop old table: ${show_backup_result}") assertTrue(show_backup_result.size() == backup_num) } - diff --git a/regression-test/suites/db_sync/table/drop_create/test_ds_tbl_drop_create.groovy b/regression-test/suites/db_sync/table/drop_create/test_ds_tbl_drop_create.groovy new file mode 100644 index 00000000..76dd570c --- /dev/null +++ b/regression-test/suites/db_sync/table/drop_create/test_ds_tbl_drop_create.groovy @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_ds_tbl_drop_create") { + def helper = new GroovyShell(new Binding(['suite': delegate])) + .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) + + // TBD +} diff --git a/regression-test/suites/db-sync-rename-table/test_db_sync_rename_table.groovy b/regression-test/suites/db_sync/table/rename/test_ds_tbl_rename.groovy similarity index 97% rename from regression-test/suites/db-sync-rename-table/test_db_sync_rename_table.groovy rename to regression-test/suites/db_sync/table/rename/test_ds_tbl_rename.groovy index fbf433a9..ec3fa72c 100644 --- a/regression-test/suites/db-sync-rename-table/test_db_sync_rename_table.groovy +++ b/regression-test/suites/db_sync/table/rename/test_ds_tbl_rename.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_db_sync_rename_table") { +suite("test_ds_tbl_rename") { def versions = sql_return_maparray "show variables like 'version_comment'" if (versions[0].Value.contains('doris-2.0.') || versions[0].Value.contains('doris-2.1.')) { logger.info("2.0/2.1 not support this case, current version is: ${versions[0].Value}") @@ -25,7 +25,7 @@ suite("test_db_sync_rename_table") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_rename_table_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 10 def opPartitonName = "less" @@ -119,4 +119,3 @@ suite("test_db_sync_rename_table") { assertTrue(helper.checkSelectTimesOf("SELECT * FROM ${newTableName} WHERE id = 2", 1, 30)) } - diff --git a/regression-test/suites/db-sync-truncate-table/test_db_sync_truncate_table.groovy b/regression-test/suites/db_sync/table/truncate/test_ds_tbl_truncate.groovy similarity index 96% rename from regression-test/suites/db-sync-truncate-table/test_db_sync_truncate_table.groovy rename to regression-test/suites/db_sync/table/truncate/test_ds_tbl_truncate.groovy index 4b557039..d29fad4a 100644 --- a/regression-test/suites/db-sync-truncate-table/test_db_sync_truncate_table.groovy +++ b/regression-test/suites/db_sync/table/truncate/test_ds_tbl_truncate.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_db_sync_truncate_table") { +suite("test_ds_tbl_truncate") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def baseTableName = "tbl_truncate_table_" + helper.randomSuffix() + def baseTableName = "tbl_truncate_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 def opPartitonName = "less0" diff --git a/regression-test/suites/db_sync/view/basic/test_ds_view_basic.groovy b/regression-test/suites/db_sync/view/basic/test_ds_view_basic.groovy new file mode 100644 index 00000000..2f39c2bd --- /dev/null +++ b/regression-test/suites/db_sync/view/basic/test_ds_view_basic.groovy @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_ds_view_basic") { + def helper = new GroovyShell(new Binding(['suite': delegate])) + .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) + + def createDuplicateTable = { tableName -> + sql """ + CREATE TABLE if NOT EXISTS ${tableName} + ( + user_id BIGINT NOT NULL COMMENT "用户 ID", + name VARCHAR(20) COMMENT "用户姓名", + age INT COMMENT "用户年龄" + ) + ENGINE=OLAP + DUPLICATE KEY(user_id) + DISTRIBUTED BY HASH(user_id) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "binlog.enable" = "true" + ) + """ + } + + def checkRestoreRowsTimesOf = {rowSize, times -> Boolean + Boolean ret = false + while (times > 0) { + def sqlInfo = target_sql "SHOW RESTORE FROM TEST_${context.dbName}" + if (sqlInfo.size() == rowSize) { + ret = true + break + } else if (--times > 0 && sqlInfo.size < rowSize) { + sleep(sync_gap_time) + } + } + + return ret + } + + def exist = { res -> Boolean + return res.size() != 0 + } + def notExist = { res -> Boolean + return res.size() == 0 + } + + def suffix = helper.randomSuffix() + def tableDuplicate0 = "tbl_duplicate_0_${suffix}" + createDuplicateTable(tableDuplicate0) + sql """ + INSERT INTO ${tableDuplicate0} VALUES + (1, "Emily", 25), + (2, "Benjamin", 35), + (3, "Olivia", 28), + (4, "Alexander", 60), + (5, "Ava", 17); + """ + + sql "ALTER DATABASE ${context.dbName} SET properties (\"binlog.enable\" = \"true\")" + + helper.ccrJobDelete() + helper.ccrJobCreate() + + assertTrue(helper.checkRestoreFinishTimesOf("${tableDuplicate0}", 30)) + assertTrue(helper.checkSelectTimesOf("SELECT * FROM ${tableDuplicate0}", 5, 30)) + + logger.info("=== Test1: create view and materialized view ===") + sql """ + CREATE VIEW view_test_${suffix} (k1, name, v1) + AS + SELECT user_id as k1, name, SUM(age) FROM ${tableDuplicate0} + GROUP BY k1,name; + """ + + sql """ + create materialized view user_id_name_${suffix} as + select user_id, name from ${tableDuplicate0}; + """ + + assertTrue(helper.checkRestoreFinishTimesOf("view_test_${suffix}", 30)) + + explain { + sql("select user_id, name from ${tableDuplicate0}") + contains "user_id_name" + } + + logger.info("=== Test 2: delete job ===") + test_num = 5 + helper.ccrJobDelete() + + sql """ + INSERT INTO ${tableDuplicate0} VALUES (6, "Zhangsan", 31) + """ + + assertTrue(helper.checkSelectTimesOf("SELECT * FROM ${tableDuplicate0}", 5, 5)) +} diff --git a/regression-test/suites/db-sv-drop-create/test_sync_view_drop_create.groovy b/regression-test/suites/db_sync/view/drop_create/test_ds_view_drop_create.groovy similarity index 97% rename from regression-test/suites/db-sv-drop-create/test_sync_view_drop_create.groovy rename to regression-test/suites/db_sync/view/drop_create/test_ds_view_drop_create.groovy index 965efbc2..b8f37696 100644 --- a/regression-test/suites/db-sv-drop-create/test_sync_view_drop_create.groovy +++ b/regression-test/suites/db_sync/view/drop_create/test_ds_view_drop_create.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_sync_view_drop_create") { +suite("test_ds_view_drop_create") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) @@ -44,7 +44,7 @@ suite("test_sync_view_drop_create") { return res.size() == 0 } - def suffix = UUID.randomUUID().toString().replace("-", "") + def suffix = helper.randomSuffix() def tableDuplicate0 = "tbl_duplicate_0_${suffix}" sql """ DROP VIEW IF EXISTS view_test_${suffix} """ sql """ DROP VIEW IF EXISTS view_test_1_${suffix} """ diff --git a/regression-test/suites/db-sv-drop-delete-create/test_sync_view_drop_delete_create.groovy b/regression-test/suites/db_sync/view/drop_delete_create/test_ds_view_drop_delete_create.groovy similarity index 97% rename from regression-test/suites/db-sv-drop-delete-create/test_sync_view_drop_delete_create.groovy rename to regression-test/suites/db_sync/view/drop_delete_create/test_ds_view_drop_delete_create.groovy index 73981c04..a4bb0c4a 100644 --- a/regression-test/suites/db-sv-drop-delete-create/test_sync_view_drop_delete_create.groovy +++ b/regression-test/suites/db_sync/view/drop_delete_create/test_ds_view_drop_delete_create.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_sync_view_drop_delete_create") { +suite("test_ds_view_drop_delete_create") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) @@ -44,7 +44,7 @@ suite("test_sync_view_drop_delete_create") { return res.size() == 0 } - def suffix = UUID.randomUUID().toString().replace("-", "") + def suffix = helper.randomSuffix() def tableDuplicate0 = "tbl_duplicate_0_${suffix}" createDuplicateTable(tableDuplicate0) sql """ diff --git a/regression-test/suites/table-sync/test_allow_table_exists.groovy b/regression-test/suites/syncer/ts_allow_table_exists/test_syncer_ts_allow_table_exists.groovy similarity index 97% rename from regression-test/suites/table-sync/test_allow_table_exists.groovy rename to regression-test/suites/syncer/ts_allow_table_exists/test_syncer_ts_allow_table_exists.groovy index a4f18c86..5b411e6b 100644 --- a/regression-test/suites/table-sync/test_allow_table_exists.groovy +++ b/regression-test/suites/syncer/ts_allow_table_exists/test_syncer_ts_allow_table_exists.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_allow_table_exists") { +suite("test_syncer_ts_allow_tablet_exists") { def versions = sql_return_maparray "show variables like 'version_comment'" if (versions[0].Value.contains('doris-2.0.') || versions[0].Value.contains('doris-2.1')) { logger.info("2.0/2.1 not support this case, current version is: ${versions[0].Value}") @@ -25,7 +25,7 @@ suite("test_allow_table_exists") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_allow_exists_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 20 def opPartitonName = "less" diff --git a/regression-test/suites/table-partial-sync/test_table_partial_sync_incremental.groovy b/regression-test/suites/table_ps_inc/basic/test_tbl_ps_inc_basic.groovy similarity index 96% rename from regression-test/suites/table-partial-sync/test_table_partial_sync_incremental.groovy rename to regression-test/suites/table_ps_inc/basic/test_tbl_ps_inc_basic.groovy index 06eb5905..f771c061 100644 --- a/regression-test/suites/table-partial-sync/test_table_partial_sync_incremental.groovy +++ b/regression-test/suites/table_ps_inc/basic/test_tbl_ps_inc_basic.groovy @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_table_partial_sync_incremental") { +suite("test_tbl_ps_inc_basic") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) @@ -23,7 +23,7 @@ suite("test_table_partial_sync_incremental") { return } - def tableName = "tbl_sync_incremental_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table-partial-sync/test_table_partial_sync_cache.groovy b/regression-test/suites/table_ps_inc/cache/test_tbl_ps_inc_cache.groovy similarity index 70% rename from regression-test/suites/table-partial-sync/test_table_partial_sync_cache.groovy rename to regression-test/suites/table_ps_inc/cache/test_tbl_ps_inc_cache.groovy index 6b59f449..dd939aab 100644 --- a/regression-test/suites/table-partial-sync/test_table_partial_sync_cache.groovy +++ b/regression-test/suites/table_ps_inc/cache/test_tbl_ps_inc_cache.groovy @@ -14,11 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_table_partial_sync_cache") { +suite("test_tbl_ps_inc_cache") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_sync_cache_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 @@ -32,40 +32,6 @@ suite("test_table_partial_sync_cache") { } } - def get_ccr_name = { ccr_body_json -> - def jsonSlurper = new groovy.json.JsonSlurper() - def object = jsonSlurper.parseText "${ccr_body_json}" - return object.name - } - - def get_job_progress = { ccr_name -> - def request_body = """ {"name":"${ccr_name}"} """ - def get_job_progress_uri = { check_func -> - httpTest { - uri "/job_progress" - endpoint helper.syncerAddress - body request_body - op "post" - check check_func - } - } - - def result = null - get_job_progress_uri.call() { code, body -> - if (!"${code}".toString().equals("200")) { - throw "request failed, code: ${code}, body: ${body}" - } - def jsonSlurper = new groovy.json.JsonSlurper() - def object = jsonSlurper.parseText "${body}" - if (!object.success) { - throw "request failed, error msg: ${object.error_msg}" - } - logger.info("job progress: ${object.job_progress}") - result = jsonSlurper.parseText object.job_progress - } - return result - } - sql "DROP TABLE IF EXISTS ${tableName}" sql """ CREATE TABLE if NOT EXISTS ${tableName} @@ -92,15 +58,12 @@ suite("test_table_partial_sync_cache") { """ sql "sync" - def bodyJson = get_ccr_body "${tableName}" - ccr_name = get_ccr_name(bodyJson) helper.ccrJobCreate(tableName) - logger.info("ccr job name: ${ccr_name}") assertTrue(helper.checkRestoreFinishTimesOf("${tableName}", 30)) assertTrue(helper.checkSelectTimesOf("SELECT * FROM ${tableName}", insert_num, 60)) - first_job_progress = get_job_progress(ccr_name) + first_job_progress = helper.get_job_progress(tableName) logger.info("=== Test 1: add first column case ===") // binlog type: ALTER_JOB, binlog data: @@ -139,7 +102,7 @@ suite("test_table_partial_sync_cache") { assertTrue(helper.checkSelectTimesOf("SELECT * FROM ${tableName}", insert_num + 1, 60)) // no full sync triggered. - last_job_progress = get_job_progress(ccr_name) + last_job_progress = helper.get_job_progress(tableName) assertTrue(last_job_progress.full_sync_start_at == first_job_progress.full_sync_start_at) } diff --git a/regression-test/suites/table-schema-change/test_add_column.groovy b/regression-test/suites/table_sync/column/add/test_ts_col_add.groovy similarity index 82% rename from regression-test/suites/table-schema-change/test_add_column.groovy rename to regression-test/suites/table_sync/column/add/test_ts_col_add.groovy index c9af4550..aa9d1089 100644 --- a/regression-test/suites/table-schema-change/test_add_column.groovy +++ b/regression-test/suites/table_sync/column/add/test_ts_col_add.groovy @@ -14,11 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_add_column") { +suite("test_ts_col_add") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_add_column" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 @@ -32,40 +32,6 @@ suite("test_add_column") { } } - def get_ccr_name = { ccr_body_json -> - def jsonSlurper = new groovy.json.JsonSlurper() - def object = jsonSlurper.parseText "${ccr_body_json}" - return object.name - } - - def get_job_progress = { ccr_name -> - def request_body = """ {"name":"${ccr_name}"} """ - def get_job_progress_uri = { check_func -> - httpTest { - uri "/job_progress" - endpoint helper.syncerAddress - body request_body - op "post" - check check_func - } - } - - def result = null - get_job_progress_uri.call() { code, body -> - if (!"${code}".toString().equals("200")) { - throw "request failed, code: ${code}, body: ${body}" - } - def jsonSlurper = new groovy.json.JsonSlurper() - def object = jsonSlurper.parseText "${body}" - if (!object.success) { - throw "request failed, error msg: ${object.error_msg}" - } - logger.info("job progress: ${object.job_progress}") - result = jsonSlurper.parseText object.job_progress - } - return result - } - sql "DROP TABLE IF EXISTS ${tableName}" sql """ CREATE TABLE if NOT EXISTS ${tableName} @@ -92,14 +58,11 @@ suite("test_add_column") { """ sql "sync" - def bodyJson = get_ccr_body "${tableName}" - ccr_name = get_ccr_name(bodyJson) helper.ccrJobCreate(tableName) - logger.info("ccr job name: ${ccr_name}") assertTrue(helper.checkRestoreFinishTimesOf("${tableName}", 30)) - first_job_progress = get_job_progress(ccr_name) + first_job_progress = helper.get_job_progress(tableName) logger.info("=== Test 1: add first column case ===") // binlog type: ALTER_JOB, binlog data: @@ -228,6 +191,6 @@ suite("test_add_column") { assertTrue(helper.checkShowTimesOf("SHOW COLUMNS FROM `${tableName}`", has_column_last_value, 60, "target_sql")) // no full sync triggered. - last_job_progress = get_job_progress(ccr_name) + last_job_progress = helper.get_job_progress(tableName) assertTrue(last_job_progress.full_sync_start_at == first_job_progress.full_sync_start_at) } diff --git a/regression-test/suites/table-schema-change/test_add_agg_column.groovy b/regression-test/suites/table_sync/column/add_agg/test_ts_col_add_agg.groovy similarity index 98% rename from regression-test/suites/table-schema-change/test_add_agg_column.groovy rename to regression-test/suites/table_sync/column/add_agg/test_ts_col_add_agg.groovy index 96b5e58d..304dc6d3 100644 --- a/regression-test/suites/table-schema-change/test_add_agg_column.groovy +++ b/regression-test/suites/table_sync/column/add_agg/test_ts_col_add_agg.groovy @@ -14,11 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_add_agg_column") { +suite("test_ts_col_add_agg") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_add_agg_column" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table-schema-change/test_add_many_column.groovy b/regression-test/suites/table_sync/column/add_many/test_ts_col_add_many.groovy similarity index 97% rename from regression-test/suites/table-schema-change/test_add_many_column.groovy rename to regression-test/suites/table_sync/column/add_many/test_ts_col_add_many.groovy index 60b9b1c6..2c185a1a 100644 --- a/regression-test/suites/table-schema-change/test_add_many_column.groovy +++ b/regression-test/suites/table_sync/column/add_many/test_ts_col_add_many.groovy @@ -14,11 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_add_many_column") { +suite("test_ts_col_add_many") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_add_many_column" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table-schema-change/test_alter_type.groovy b/regression-test/suites/table_sync/column/alter_type/test_ts_col_alter_type.groovy similarity index 98% rename from regression-test/suites/table-schema-change/test_alter_type.groovy rename to regression-test/suites/table_sync/column/alter_type/test_ts_col_alter_type.groovy index 1931a786..f96e1908 100644 --- a/regression-test/suites/table-schema-change/test_alter_type.groovy +++ b/regression-test/suites/table_sync/column/alter_type/test_ts_col_alter_type.groovy @@ -14,11 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_alter_type") { +suite("test_ts_col_alter_type") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_alter_type" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table-sync/test_column_ops.groovy b/regression-test/suites/table_sync/column/basic/test_ts_col_basic.groovy similarity index 97% rename from regression-test/suites/table-sync/test_column_ops.groovy rename to regression-test/suites/table_sync/column/basic/test_ts_col_basic.groovy index 01266e56..5997a7ed 100644 --- a/regression-test/suites/table-sync/test_column_ops.groovy +++ b/regression-test/suites/table_sync/column/basic/test_ts_col_basic.groovy @@ -14,11 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_column_ops") { +suite("test_ts_col_basic") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_column_ops_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 @@ -154,7 +154,7 @@ suite("test_column_ops") { MODIFY COLUMN `id` COMMENT 'index of one test number' """ assertTrue(checkColumnCommentTimesOf(tableName, - [test: "test number", id: "index of one test number", cost: ""], 30)) + [test: "test number", id: "index of one test number", _cost: ""], 30)) logger.info("=== Test 5: drop column case ===") diff --git a/regression-test/suites/table-schema-change/test_drop_column.groovy b/regression-test/suites/table_sync/column/drop/test_ts_col_drop.groovy similarity index 98% rename from regression-test/suites/table-schema-change/test_drop_column.groovy rename to regression-test/suites/table_sync/column/drop/test_ts_col_drop.groovy index c2d49980..f424ea18 100644 --- a/regression-test/suites/table-schema-change/test_drop_column.groovy +++ b/regression-test/suites/table_sync/column/drop/test_ts_col_drop.groovy @@ -14,11 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_drop_column") { +suite("test_ts_col_drop") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_drop_column_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table-schema-change/test_filter_dropped_indexes.groovy b/regression-test/suites/table_sync/column/filter_dropped_indexes/test_ts_col_filter_dropped_indexes.groovy similarity index 96% rename from regression-test/suites/table-schema-change/test_filter_dropped_indexes.groovy rename to regression-test/suites/table_sync/column/filter_dropped_indexes/test_ts_col_filter_dropped_indexes.groovy index e125956a..32287396 100644 --- a/regression-test/suites/table-schema-change/test_filter_dropped_indexes.groovy +++ b/regression-test/suites/table_sync/column/filter_dropped_indexes/test_ts_col_filter_dropped_indexes.groovy @@ -14,11 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_filter_dropped_indexes") { +suite("test_ts_col_filter_dropped_indexes") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_filter_dropped_indexes_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table-schema-change/test_order_by.groovy b/regression-test/suites/table_sync/column/order_by/test_ts_col_order_by.groovy similarity index 97% rename from regression-test/suites/table-schema-change/test_order_by.groovy rename to regression-test/suites/table_sync/column/order_by/test_ts_col_order_by.groovy index 88f6b150..4cb57069 100644 --- a/regression-test/suites/table-schema-change/test_order_by.groovy +++ b/regression-test/suites/table_sync/column/order_by/test_ts_col_order_by.groovy @@ -14,11 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_order_by") { +suite("test_ts_col_order_by") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_order_by_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table-sync/test_common.groovy b/regression-test/suites/table_sync/common/test_ts_common.groovy similarity index 98% rename from regression-test/suites/table-sync/test_common.groovy rename to regression-test/suites/table_sync/common/test_ts_common.groovy index 39074ad0..6ed9a5b4 100644 --- a/regression-test/suites/table-sync/test_common.groovy +++ b/regression-test/suites/table_sync/common/test_ts_common.groovy @@ -14,11 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_common") { +suite("test_ts_common") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_common_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def uniqueTable = "${tableName}_unique" def aggregateTable = "${tableName}_aggregate" def duplicateTable = "${tableName}_duplicate" diff --git a/regression-test/suites/table-sync/test_delete.groovy b/regression-test/suites/table_sync/dml/delete/test_ts_dml_delete.groovy similarity index 98% rename from regression-test/suites/table-sync/test_delete.groovy rename to regression-test/suites/table_sync/dml/delete/test_ts_dml_delete.groovy index 120efb16..e26bd0f7 100644 --- a/regression-test/suites/table-sync/test_delete.groovy +++ b/regression-test/suites/table_sync/dml/delete/test_ts_dml_delete.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_delete") { +suite("test_ts_dml_delete") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_delete_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 29 diff --git a/regression-test/suites/table-sync/test_insert_overwrite.groovy b/regression-test/suites/table_sync/dml/insert_overwrite/test_ts_dml_insert_overwrite.groovy similarity index 97% rename from regression-test/suites/table-sync/test_insert_overwrite.groovy rename to regression-test/suites/table_sync/dml/insert_overwrite/test_ts_dml_insert_overwrite.groovy index cb49d251..522c74fd 100644 --- a/regression-test/suites/table-sync/test_insert_overwrite.groovy +++ b/regression-test/suites/table_sync/dml/insert_overwrite/test_ts_dml_insert_overwrite.groovy @@ -14,7 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_insert_overwrite") { +suite("test_ts_dml_insert_overwrite") { def versions = sql_return_maparray "show variables like 'version_comment'" if (versions[0].Value.contains('doris-2.0.')) { logger.info("2.0 not support this case, current version is: ${versions[0].Value}") @@ -33,7 +33,7 @@ suite("test_insert_overwrite") { // 1. create temp partitions // 2. insert into temp partitions // 3. replace overlap partitions - def tableName = "tbl_insert_overwrite_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def uniqueTable = "${tableName}_unique" def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table-sync/test_bloomfilter_index.groovy b/regression-test/suites/table_sync/index/add_bf/test_tbl_index_add_bloom_filter.groovy similarity index 97% rename from regression-test/suites/table-sync/test_bloomfilter_index.groovy rename to regression-test/suites/table_sync/index/add_bf/test_tbl_index_add_bloom_filter.groovy index 63df704e..9c3761fd 100644 --- a/regression-test/suites/table-sync/test_bloomfilter_index.groovy +++ b/regression-test/suites/table_sync/index/add_bf/test_tbl_index_add_bloom_filter.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_bloomfilter_index") { +suite("test_tbl_index_add_bloom_filter") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_bloomfilter_index_" + helper.randomSuffix() + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table-sync/test_bitmap_index.groovy b/regression-test/suites/table_sync/index/add_bitmap/test_ts_index_add_bitmap.groovy similarity index 97% rename from regression-test/suites/table-sync/test_bitmap_index.groovy rename to regression-test/suites/table_sync/index/add_bitmap/test_ts_index_add_bitmap.groovy index 2f937f24..44da05a7 100644 --- a/regression-test/suites/table-sync/test_bitmap_index.groovy +++ b/regression-test/suites/table_sync/index/add_bitmap/test_ts_index_add_bitmap.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_bitmap_index") { +suite("test_ts_index_add_bitmap") { logger.info("test bitmap index will be replaced by inverted index") return - def tableName = "tbl_bitmap_index_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def syncerAddress = "127.0.0.1:9190" def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table_sync/index/add_inverted/test_ts_index_add_inverted.groovy b/regression-test/suites/table_sync/index/add_inverted/test_ts_index_add_inverted.groovy new file mode 100644 index 00000000..ed1bd1eb --- /dev/null +++ b/regression-test/suites/table_sync/index/add_inverted/test_ts_index_add_inverted.groovy @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("test_index_add_inverted") { + def helper = new GroovyShell(new Binding(['suite': delegate])) + .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) + + def tableName = "tbl_" + helper.randomSuffix() + def test_num = 0 + def insert_num = 5 + + def exist = { res -> Boolean + return res.size() != 0 + } + + def has_count = { count -> + return { res -> Boolean + res.size() == count + } + } + + sql "DROP TABLE IF EXISTS ${tableName}" + sql """ + CREATE TABLE if NOT EXISTS ${tableName} + ( + `test` INT, + `id` INT, + `value` String, + `value1` String + ) + ENGINE=OLAP + UNIQUE KEY(`test`, `id`) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "binlog.enable" = "true" + ) + """ + + def values = []; + for (int index = 0; index < insert_num; index++) { + values.add("(${test_num}, ${index}, '${index}', '${index}')") + } + sql """ + INSERT INTO ${tableName} VALUES ${values.join(",")} + """ + sql "sync" + + helper.ccrJobCreate(tableName) + assertTrue(helper.checkRestoreFinishTimesOf("${tableName}", 30)) + + logger.info("=== Test 1: add inverted index ===") + sql """ + ALTER TABLE ${tableName} + ADD INDEX idx_inverted(value) USING INVERTED + """ + sql "sync" + + sql """ INSERT INTO ${tableName} VALUES (1, 1, "1", "1") """ + + assertTrue(helper.checkShowTimesOf(""" + SHOW ALTER TABLE COLUMN + FROM ${context.dbName} + WHERE TableName = "${tableName}" AND State = "FINISHED" + """, + has_count(1), 30)) + + sql """ + BUILD INDEX idx_inverted ON ${tableName} + """ + sql "sync" + + sql """ INSERT INTO ${tableName} VALUES (2, 2, "2", "2") """ + + assertTrue(helper.checkShowTimesOf(""" + SHOW BUILD INDEX FROM ${context.dbName} + WHERE TableName = "${tableName}" AND State = "FINISHED" + """, + has_count(1), 30)) + + sql """ + ALTER TABLE ${tableName} + DROP INDEX idx_inverted + """ + sql "sync" + + assertTrue(helper.checkShowTimesOf(""" + SHOW ALTER TABLE COLUMN + FROM ${context.dbName} + WHERE TableName = "${tableName}" AND State = "FINISHED" + """, + has_count(2), 30)) + + sql """ INSERT INTO ${tableName} VALUES (3, 3, "3", "3")""" + + // FIXME(walter) no such binlogs + + // logger.info("=== Test 2: build bloom filter ===") + // sql """ + // ALTER TABLE ${tableName} + // SET ("bloom_filter_columns" = "value,value1") + // """ + // sql "sync" + + // assertTrue(helper.checkShowTimesOf(""" + // SHOW ALTER TABLE COLUMN + // FROM ${context.dbName} + // WHERE TableName = "${tableName}" AND State = "FINISHED" + // """, + // has_count(3), 30)) + + // // drop bloom filter + // sql """ + // ALTER TABLE ${tableName} + // SET ("bloom_filter_columns" = "") + // """ + // assertTrue(helper.checkShowTimesOf(""" + // SHOW ALTER TABLE COLUMN + // FROM ${context.dbName} + // WHERE TableName = "${tableName}" AND State = "FINISHED" + // """, + // has_count(4), 30)) +} + diff --git a/regression-test/suites/table-sync/test_materialized_view.groovy b/regression-test/suites/table_sync/mv/create_drop/test_ts_mv_create_drop.groovy similarity index 97% rename from regression-test/suites/table-sync/test_materialized_view.groovy rename to regression-test/suites/table_sync/mv/create_drop/test_ts_mv_create_drop.groovy index 70fb3232..55c68ec1 100644 --- a/regression-test/suites/table-sync/test_materialized_view.groovy +++ b/regression-test/suites/table_sync/mv/create_drop/test_ts_mv_create_drop.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_materialized_index") { +suite("test_ts_mv_create_drop") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_materialized_sync_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table-sync/test_add_partition.groovy b/regression-test/suites/table_sync/partition/add/test_ts_part_add.groovy similarity index 98% rename from regression-test/suites/table-sync/test_add_partition.groovy rename to regression-test/suites/table_sync/partition/add/test_ts_part_add.groovy index 69224a7b..eebb3308 100644 --- a/regression-test/suites/table-sync/test_add_partition.groovy +++ b/regression-test/suites/table_sync/partition/add/test_ts_part_add.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_add_partition") { +suite("test_ts_part_add") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def baseTableName = "test_add_partition_" + helper.randomSuffix() + def baseTableName = "test_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 def opPartitonName = "less0" diff --git a/regression-test/suites/table-sync/test_partition_ops.groovy b/regression-test/suites/table_sync/partition/add_drop/test_tbl_part_add_drop.groovy similarity index 97% rename from regression-test/suites/table-sync/test_partition_ops.groovy rename to regression-test/suites/table_sync/partition/add_drop/test_tbl_part_add_drop.groovy index 9634e737..3d9936f8 100644 --- a/regression-test/suites/table-sync/test_partition_ops.groovy +++ b/regression-test/suites/table_sync/partition/add_drop/test_tbl_part_add_drop.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_partition_ops") { +suite("test_tbl_part_add_drop") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_partition_ops_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 def opPartitonName = "less0" diff --git a/regression-test/suites/table-sync/test_restore_clean_partitions.groovy b/regression-test/suites/table_sync/partition/clean_restore/test_ts_part_clean_restore.groovy similarity index 97% rename from regression-test/suites/table-sync/test_restore_clean_partitions.groovy rename to regression-test/suites/table_sync/partition/clean_restore/test_ts_part_clean_restore.groovy index c00c6827..19bb1355 100644 --- a/regression-test/suites/table-sync/test_restore_clean_partitions.groovy +++ b/regression-test/suites/table_sync/partition/clean_restore/test_ts_part_clean_restore.groovy @@ -15,13 +15,13 @@ // specific language governing permissions and limitations // under the License. -suite("test_restore_clean_partitions") { +suite("test_ts_part_clean_restore") { // FIXME(walter) fix clean partitions. return def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_clean_partitions_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 20 def sync_gap_time = 5000 diff --git a/regression-test/suites/table-sync/test_replace_partition.groovy b/regression-test/suites/table_sync/partition/replace/test_ts_part_replace.groovy similarity index 98% rename from regression-test/suites/table-sync/test_replace_partition.groovy rename to regression-test/suites/table_sync/partition/replace/test_ts_part_replace.groovy index b7daeaef..54af3ba8 100644 --- a/regression-test/suites/table-sync/test_replace_partition.groovy +++ b/regression-test/suites/table_sync/partition/replace/test_ts_part_replace.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_replace_partition") { +suite("test_ts_part_replace") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def baseTableName = "test_replace_partition_" + UUID.randomUUID().toString().replace("-", "") + def baseTableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 def opPartitonName = "less0" diff --git a/regression-test/suites/table-sync/test_replace_partial_partition.groovy b/regression-test/suites/table_sync/partition/replace_partial/test_ts_part_replace_partial.groovy similarity index 97% rename from regression-test/suites/table-sync/test_replace_partial_partition.groovy rename to regression-test/suites/table_sync/partition/replace_partial/test_ts_part_replace_partial.groovy index b2c53937..e3b14f32 100644 --- a/regression-test/suites/table-sync/test_replace_partial_partition.groovy +++ b/regression-test/suites/table_sync/partition/replace_partial/test_ts_part_replace_partial.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_replace_partial_partition") { +suite("test_ts_part_replace_partial") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def baseTableName = "test_replace_partial_p_" + UUID.randomUUID().toString().replace("-", "") + def baseTableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 def opPartitonName = "less0" diff --git a/regression-test/suites/table-sync/test_rollup.groovy b/regression-test/suites/table_sync/rollup/add/test_ts_rollup_add.groovy similarity index 96% rename from regression-test/suites/table-sync/test_rollup.groovy rename to regression-test/suites/table_sync/rollup/add/test_ts_rollup_add.groovy index 815b6b97..8166d064 100644 --- a/regression-test/suites/table-sync/test_rollup.groovy +++ b/regression-test/suites/table_sync/rollup/add/test_ts_rollup_add.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_rollup_sync") { +suite("test_ts_rollup_add") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_rollup_sync_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table-sync/test_table_comment.groovy b/regression-test/suites/table_sync/table/modify_comment/test_ts_table_modify_comment.groovy similarity index 98% rename from regression-test/suites/table-sync/test_table_comment.groovy rename to regression-test/suites/table_sync/table/modify_comment/test_ts_table_modify_comment.groovy index 75017e6e..2ee9e058 100644 --- a/regression-test/suites/table-sync/test_table_comment.groovy +++ b/regression-test/suites/table_sync/table/modify_comment/test_ts_table_modify_comment.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_table_comment") { +suite("test_ts_table_modify_comment") { def tableName = "tbl_comment" + UUID.randomUUID().toString().replace("-", "") def syncerAddress = "127.0.0.1:9190" diff --git a/regression-test/suites/table-sync/test_rename.groovy b/regression-test/suites/table_sync/table/rename/test_ts_tbl_rename.groovy similarity index 97% rename from regression-test/suites/table-sync/test_rename.groovy rename to regression-test/suites/table_sync/table/rename/test_ts_tbl_rename.groovy index f13b28b3..644196e3 100644 --- a/regression-test/suites/table-sync/test_rename.groovy +++ b/regression-test/suites/table_sync/table/rename/test_ts_tbl_rename.groovy @@ -15,14 +15,14 @@ // specific language governing permissions and limitations // under the License. -suite("test_rename") { +suite("test_ts_tbl_rename") { logger.info("exit because test_rename is not supported yet") return def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_rename_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table-sync/test_auto_bucket.groovy b/regression-test/suites/table_sync/table/res_auto_bucket/test_ts_tbl_res_auto_bucket.groovy similarity index 96% rename from regression-test/suites/table-sync/test_auto_bucket.groovy rename to regression-test/suites/table_sync/table/res_auto_bucket/test_ts_tbl_res_auto_bucket.groovy index b2c2bca5..85d57359 100644 --- a/regression-test/suites/table-sync/test_auto_bucket.groovy +++ b/regression-test/suites/table_sync/table/res_auto_bucket/test_ts_tbl_res_auto_bucket.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_auto_bucket") { +suite("test_ts_tbl_res_auto_bucket") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "test_auto_bucket_" + helper.randomSuffix() + def tableName = "test_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 def opPartitonName = "less0" diff --git a/regression-test/suites/table-sync/test_inverted_index.groovy b/regression-test/suites/table_sync/table/res_inverted_idx/test_ts_tbl_res_inverted_idx.groovy similarity index 97% rename from regression-test/suites/table-sync/test_inverted_index.groovy rename to regression-test/suites/table_sync/table/res_inverted_idx/test_ts_tbl_res_inverted_idx.groovy index d0dc0ba4..7b8893ab 100644 --- a/regression-test/suites/table-sync/test_inverted_index.groovy +++ b/regression-test/suites/table_sync/table/res_inverted_idx/test_ts_tbl_res_inverted_idx.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_inverted_index") { +suite("test_ts_tbl_res_inverted_idx") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) @@ -128,7 +128,7 @@ suite("test_inverted_index") { /** * test for unique key table with mow */ - tableName = "tbl_inverted_index_unique_mow_" + helper.randomSuffix() + tableName = "tbl_inverted_index_uniq_mow_" + helper.randomSuffix() sql """ DROP TABLE IF EXISTS ${tableName}; """ sql """ @@ -161,7 +161,7 @@ suite("test_inverted_index") { /** * test for unique key table with mor */ - tableName = "tbl_inverted_index_unique_mor_" + helper.randomSuffix() + tableName = "tbl_inverted_index_uniq_mor_" + helper.randomSuffix() sql """ DROP TABLE IF EXISTS ${tableName}; """ sql """ diff --git a/regression-test/suites/table-sync/test_mow.groovy b/regression-test/suites/table_sync/table/res_mow/test_ts_table_res_mow.groovy similarity index 97% rename from regression-test/suites/table-sync/test_mow.groovy rename to regression-test/suites/table_sync/table/res_mow/test_ts_table_res_mow.groovy index df03331f..c569e1e3 100644 --- a/regression-test/suites/table-sync/test_mow.groovy +++ b/regression-test/suites/table_sync/table/res_mow/test_ts_table_res_mow.groovy @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -suite("test_mow") { +suite("test_ts_table_res_mow") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_mow_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 String response diff --git a/regression-test/suites/table-sync/test_row_storage.groovy b/regression-test/suites/table_sync/table/res_row_storage/test_ts_tbl_res_row_storage.groovy similarity index 96% rename from regression-test/suites/table-sync/test_row_storage.groovy rename to regression-test/suites/table_sync/table/res_row_storage/test_ts_tbl_res_row_storage.groovy index c29afe00..321dd583 100644 --- a/regression-test/suites/table-sync/test_row_storage.groovy +++ b/regression-test/suites/table_sync/table/res_row_storage/test_ts_tbl_res_row_storage.groovy @@ -14,11 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_row_storage") { +suite("test_ts_tbl_res_row_storage") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_row_storage_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table-sync/test_variant.groovy b/regression-test/suites/table_sync/table/res_variant/test_ts_tbl_res_variant.groovy similarity index 96% rename from regression-test/suites/table-sync/test_variant.groovy rename to regression-test/suites/table_sync/table/res_variant/test_ts_tbl_res_variant.groovy index be989dc2..d4e293ba 100644 --- a/regression-test/suites/table-sync/test_variant.groovy +++ b/regression-test/suites/table_sync/table/res_variant/test_ts_tbl_res_variant.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_variant_ccr") { +suite("test_ts_tbl_res_variant") { def versions = sql_return_maparray "show variables like 'version_comment'" if (versions[0].Value.contains('doris-2.0.')) { logger.info("2.0 not support variant case, current version is: ${versions[0].Value}") @@ -25,7 +25,7 @@ suite("test_variant_ccr") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "test_variant_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "test_" + helper.randomSuffix() def insert_num = 5 sql """ diff --git a/regression-test/suites/table-sync/test_truncate_table.groovy b/regression-test/suites/table_sync/table/truncate/test_ts_tbl_truncate.groovy similarity index 97% rename from regression-test/suites/table-sync/test_truncate_table.groovy rename to regression-test/suites/table_sync/table/truncate/test_ts_tbl_truncate.groovy index 4b0c4d45..6164f4d8 100644 --- a/regression-test/suites/table-sync/test_truncate_table.groovy +++ b/regression-test/suites/table_sync/table/truncate/test_ts_tbl_truncate.groovy @@ -14,11 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_truncate") { +suite("test_ts_tbl_truncate") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "tbl_truncate_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "tbl_" + helper.randomSuffix() def test_num = 0 def insert_num = 5 diff --git a/regression-test/suites/table-sync-with-alias/test_lightning_schema_change.groovy b/regression-test/suites/table_sync_alias/column/add_value/test_tsa_column_add.groovy similarity index 96% rename from regression-test/suites/table-sync-with-alias/test_lightning_schema_change.groovy rename to regression-test/suites/table_sync_alias/column/add_value/test_tsa_column_add.groovy index 601c21c8..965085bf 100644 --- a/regression-test/suites/table-sync-with-alias/test_lightning_schema_change.groovy +++ b/regression-test/suites/table_sync_alias/column/add_value/test_tsa_column_add.groovy @@ -14,11 +14,11 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. -suite("test_lightning_schema_change") { +suite("test_tsa_column_add") { def helper = new GroovyShell(new Binding(['suite': delegate])) .evaluate(new File("${context.config.suitePath}/../common", "helper.groovy")) - def tableName = "test_add_column_" + UUID.randomUUID().toString().replace("-", "") + def tableName = "test_" + helper.randomSuffix() def test_num = 0 def insert_num = 5