From adf8574eb41a4626181c976ec3b4c6fd8374d60a Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 8 Jan 2025 14:25:39 +0530 Subject: [PATCH 1/3] fix some commands to retrieve uuid as early as possible --- yb-voyager/cmd/analyzeSchema.go | 10 +++++----- yb-voyager/cmd/assessMigrationBulkCommand.go | 11 +++++------ yb-voyager/cmd/assessMigrationCommand.go | 17 ++++++++--------- yb-voyager/cmd/endMigrationCommand.go | 6 +++++- yb-voyager/cmd/importData.go | 11 +++++------ yb-voyager/cmd/importDataFileCommand.go | 11 +++++------ yb-voyager/cmd/importSchema.go | 12 ++++++------ yb-voyager/src/callhome/diagnostics.go | 2 +- 8 files changed, 40 insertions(+), 40 deletions(-) diff --git a/yb-voyager/cmd/analyzeSchema.go b/yb-voyager/cmd/analyzeSchema.go index f23c13d0bf..41b3a2aaf8 100644 --- a/yb-voyager/cmd/analyzeSchema.go +++ b/yb-voyager/cmd/analyzeSchema.go @@ -1123,10 +1123,6 @@ func checkConversions(sqlInfoArr []sqlInfo, filePath string) { } func analyzeSchema() { - err := retrieveMigrationUUID() - if err != nil { - utils.ErrExit("failed to get migration UUID: %w", err) - } utils.PrintAndLog("Analyzing schema for target YugabyteDB version %s\n", targetDbVersion) schemaAnalysisStartedEvent := createSchemaAnalysisStartedEvent() @@ -1287,9 +1283,13 @@ var analyzeSchemaCmd = &cobra.Command{ "For more details and examples, visit https://docs.yugabyte.com/preview/yugabyte-voyager/reference/schema-migration/analyze-schema/", Long: ``, PreRun: func(cmd *cobra.Command, args []string) { + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } validOutputFormats := []string{"html", "json", "txt", "xml"} validateReportOutputFormat(validOutputFormats, analyzeSchemaReportFormat) - err := validateAndSetTargetDbVersionFlag() + err = validateAndSetTargetDbVersionFlag() if err != nil { utils.ErrExit("%v", err) } diff --git a/yb-voyager/cmd/assessMigrationBulkCommand.go b/yb-voyager/cmd/assessMigrationBulkCommand.go index fb44192f8f..7cc9d9e4b7 100644 --- a/yb-voyager/cmd/assessMigrationBulkCommand.go +++ b/yb-voyager/cmd/assessMigrationBulkCommand.go @@ -48,7 +48,11 @@ var assessMigrationBulkCmd = &cobra.Command{ Long: "Bulk Assessment of multiple schemas across one or more Oracle database instances", PreRun: func(cmd *cobra.Command, args []string) { - err := validateFleetConfigFile(fleetConfigPath) + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } + err = validateFleetConfigFile(fleetConfigPath) if err != nil { utils.ErrExit("%s", err.Error()) } @@ -156,11 +160,6 @@ func assessMigrationBulk() error { return fmt.Errorf("failed to parse fleet config file: %w", err) } - err = retrieveMigrationUUID() - if err != nil { - return fmt.Errorf("failed to get migration UUID: %w", err) - } - for _, dbConfig := range bulkAssessmentDBConfigs { utils.PrintAndLog("\nAssessing '%s' schema", dbConfig.GetSchemaIdentifier()) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index f6c17cbb4b..5dd794d8fc 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -79,13 +79,17 @@ var assessMigrationCmd = &cobra.Command{ Long: fmt.Sprintf("Assess the migration from source (%s) database to YugabyteDB.", strings.Join(assessMigrationSupportedDBTypes, ", ")), PreRun: func(cmd *cobra.Command, args []string) { + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } validateSourceDBTypeForAssessMigration() setExportFlagsDefaults() validateSourceSchema() validatePortRange() validateSSLMode() validateOracleParams() - err := validateAndSetTargetDbVersionFlag() + err = validateAndSetTargetDbVersionFlag() if err != nil { utils.ErrExit("%v", err) } @@ -320,11 +324,6 @@ func assessMigration() (err error) { checkStartCleanForAssessMigration(assessmentMetadataDirFlag != "") CreateMigrationProjectIfNotExists(source.DBType, exportDir) - err = retrieveMigrationUUID() - if err != nil { - return fmt.Errorf("failed to get migration UUID: %w", err) - } - utils.PrintAndLog("Assessing for migration to target YugabyteDB version %s\n", targetDbVersion) assessmentDir := filepath.Join(exportDir, "assessment") @@ -915,10 +914,10 @@ func generateAssessmentReport() (err error) { func getAssessmentReportContentFromAnalyzeSchema() error { /* - Here we are generating analyze schema report which converts issue instance to analyze schema issue - Then in assessment codepath we extract the required information from analyze schema issue which could have been done directly from issue instance(TODO) + Here we are generating analyze schema report which converts issue instance to analyze schema issue + Then in assessment codepath we extract the required information from analyze schema issue which could have been done directly from issue instance(TODO) - But current Limitation is analyze schema currently uses regexp etc to detect some issues(not using parser). + But current Limitation is analyze schema currently uses regexp etc to detect some issues(not using parser). */ schemaAnalysisReport := analyzeSchemaInternal(&source, true) assessmentReport.MigrationComplexity = schemaAnalysisReport.MigrationComplexity diff --git a/yb-voyager/cmd/endMigrationCommand.go b/yb-voyager/cmd/endMigrationCommand.go index f016c8cbd2..6f60870f3f 100644 --- a/yb-voyager/cmd/endMigrationCommand.go +++ b/yb-voyager/cmd/endMigrationCommand.go @@ -51,6 +51,11 @@ var endMigrationCmd = &cobra.Command{ if utils.IsDirectoryEmpty(exportDir) { utils.ErrExit("export directory is empty, nothing to end") } + + err = retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } }, Run: endMigrationCommandFn, @@ -76,7 +81,6 @@ func endMigrationCommandFn(cmd *cobra.Command, args []string) { utils.ErrExit("error while checking streaming mode: %w\n", err) } - retrieveMigrationUUID() checkIfEndCommandCanBePerformed(msr) // backing up the state from the export directory diff --git a/yb-voyager/cmd/importData.go b/yb-voyager/cmd/importData.go index 99e71c969b..027170fc59 100644 --- a/yb-voyager/cmd/importData.go +++ b/yb-voyager/cmd/importData.go @@ -81,7 +81,11 @@ var importDataCmd = &cobra.Command{ importerRole = TARGET_DB_IMPORTER_ROLE } sourceDBType = GetSourceDBTypeFromMSR() - err := validateImportFlags(cmd, importerRole) + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } + err = validateImportFlags(cmd, importerRole) if err != nil { utils.ErrExit("Error: %s", err.Error()) } @@ -132,11 +136,6 @@ func importDataCommandFn(cmd *cobra.Command, args []string) { utils.ErrExit("Failed to get migration status record: %s", err) } - err = retrieveMigrationUUID() - if err != nil { - utils.ErrExit("failed to get migration UUID: %w", err) - } - // Check if target DB has the required permissions if tconf.RunGuardrailsChecks { checkImportDataPermissions() diff --git a/yb-voyager/cmd/importDataFileCommand.go b/yb-voyager/cmd/importDataFileCommand.go index 0deaee34b0..228cf7e6f1 100644 --- a/yb-voyager/cmd/importDataFileCommand.go +++ b/yb-voyager/cmd/importDataFileCommand.go @@ -72,10 +72,13 @@ var importDataFileCmd = &cobra.Command{ sourceDBType = POSTGRESQL // dummy value - this command is not affected by it sqlname.SourceDBType = sourceDBType CreateMigrationProjectIfNotExists(sourceDBType, exportDir) - + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } tconf.Schema = strings.ToLower(tconf.Schema) tdb = tgtdb.NewTargetDB(&tconf) - err := tdb.Init() + err = tdb.Init() if err != nil { utils.ErrExit("Failed to initialize the target DB: %s", err) } @@ -91,10 +94,6 @@ var importDataFileCmd = &cobra.Command{ dataStore = datastore.NewDataStore(dataDir) importFileTasks := prepareImportFileTasks() prepareForImportDataCmd(importFileTasks) - err := retrieveMigrationUUID() - if err != nil { - utils.ErrExit("failed to get migration UUID: %w", err) - } importData(importFileTasks) packAndSendImportDataFilePayload(COMPLETE) diff --git a/yb-voyager/cmd/importSchema.go b/yb-voyager/cmd/importSchema.go index 388535a2e3..ad63f08528 100644 --- a/yb-voyager/cmd/importSchema.go +++ b/yb-voyager/cmd/importSchema.go @@ -47,8 +47,12 @@ var importSchemaCmd = &cobra.Command{ if tconf.TargetDBType == "" { tconf.TargetDBType = YUGABYTEDB } + err := retrieveMigrationUUID() + if err != nil { + utils.ErrExit("failed to get migration UUID: %w", err) + } sourceDBType = GetSourceDBTypeFromMSR() - err := validateImportFlags(cmd, TARGET_DB_IMPORTER_ROLE) + err = validateImportFlags(cmd, TARGET_DB_IMPORTER_ROLE) if err != nil { utils.ErrExit("Error: %s", err.Error()) } @@ -79,10 +83,6 @@ var flagRefreshMViews utils.BoolStr var invalidTargetIndexesCache map[string]bool func importSchema() error { - err := retrieveMigrationUUID() - if err != nil { - return fmt.Errorf("failed to get migration UUID: %w", err) - } tconf.Schema = strings.ToLower(tconf.Schema) @@ -93,7 +93,7 @@ func importSchema() error { // available always and this is just for initialisation of tdb and marking it nil again back. tconf.Schema = "public" tdb = tgtdb.NewTargetDB(&tconf) - err = tdb.Init() + err := tdb.Init() if err != nil { utils.ErrExit("Failed to initialize the target DB: %s", err) } diff --git a/yb-voyager/src/callhome/diagnostics.go b/yb-voyager/src/callhome/diagnostics.go index 8e53134162..4ea7b10c51 100644 --- a/yb-voyager/src/callhome/diagnostics.go +++ b/yb-voyager/src/callhome/diagnostics.go @@ -58,7 +58,7 @@ CREATE TABLE diagnostics ( migration_type TEXT, time_taken_sec int, status TEXT, - host_ip character varying 255 -- set by the callhome service + host_ip character varying (255), -- set in callhome service PRIMARY KEY (migration_uuid, migration_phase, collected_at) ); From e12e60508f6c5d2716ab22d229f6b5a418fb820e Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 8 Jan 2025 14:53:29 +0530 Subject: [PATCH 2/3] fix retrieve uuid should called once we basic structure of export-dir exists --- yb-voyager/cmd/assessMigrationCommand.go | 3 +-- yb-voyager/cmd/endMigrationCommand.go | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 601272ff32..96cc14f507 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -80,6 +80,7 @@ var assessMigrationCmd = &cobra.Command{ Long: fmt.Sprintf("Assess the migration from source (%s) database to YugabyteDB.", strings.Join(assessMigrationSupportedDBTypes, ", ")), PreRun: func(cmd *cobra.Command, args []string) { + CreateMigrationProjectIfNotExists(source.DBType, exportDir) err := retrieveMigrationUUID() if err != nil { utils.ErrExit("failed to get migration UUID: %w", err) @@ -323,8 +324,6 @@ func assessMigration() (err error) { schemaDir = filepath.Join(assessmentMetadataDir, "schema") checkStartCleanForAssessMigration(assessmentMetadataDirFlag != "") - CreateMigrationProjectIfNotExists(source.DBType, exportDir) - utils.PrintAndLog("Assessing for migration to target YugabyteDB version %s\n", targetDbVersion) assessmentDir := filepath.Join(exportDir, "assessment") diff --git a/yb-voyager/cmd/endMigrationCommand.go b/yb-voyager/cmd/endMigrationCommand.go index 6f60870f3f..75243668e6 100644 --- a/yb-voyager/cmd/endMigrationCommand.go +++ b/yb-voyager/cmd/endMigrationCommand.go @@ -43,19 +43,19 @@ var endMigrationCmd = &cobra.Command{ Long: "End the current migration and cleanup all metadata stored in databases(Target, Source-Replica and Source) and export-dir", PreRun: func(cmd *cobra.Command, args []string) { - err := validateEndMigrationFlags(cmd) - if err != nil { - utils.ErrExit(err.Error()) - } - if utils.IsDirectoryEmpty(exportDir) { utils.ErrExit("export directory is empty, nothing to end") } - err = retrieveMigrationUUID() + err := retrieveMigrationUUID() if err != nil { utils.ErrExit("failed to get migration UUID: %w", err) } + err = validateEndMigrationFlags(cmd) + if err != nil { + utils.ErrExit(err.Error()) + } + }, Run: endMigrationCommandFn, From 39f5117f082cca34d10630f52844dfeba25f886c Mon Sep 17 00:00:00 2001 From: priyanshi-yb Date: Wed, 8 Jan 2025 21:54:40 +0530 Subject: [PATCH 3/3] review commetn --- yb-voyager/cmd/importData.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yb-voyager/cmd/importData.go b/yb-voyager/cmd/importData.go index 027170fc59..9afba49830 100644 --- a/yb-voyager/cmd/importData.go +++ b/yb-voyager/cmd/importData.go @@ -80,11 +80,11 @@ var importDataCmd = &cobra.Command{ if importerRole == "" { importerRole = TARGET_DB_IMPORTER_ROLE } - sourceDBType = GetSourceDBTypeFromMSR() err := retrieveMigrationUUID() if err != nil { utils.ErrExit("failed to get migration UUID: %w", err) } + sourceDBType = GetSourceDBTypeFromMSR() err = validateImportFlags(cmd, importerRole) if err != nil { utils.ErrExit("Error: %s", err.Error())