Added automation suite for Voyager upgrades

migtests/lib/yb.py

@@ -52,7 +52,8 @@ def new_source_db():
 
 def verify_colocation(tgt, source_db_type):
 	print("Verifying the colocation of the tables")
-	json_file = "export-dir/assessment/reports/migration_assessment_report.json"
+	export_dir = os.getenv("EXPORT_DIR", "export-dir")
+	json_file = f"{export_dir}/assessment/reports/migration_assessment_report.json"
 
 	sharded_tables, colocated_tables = fetch_sharded_and_colocated_tables(json_file)
 

migtests/scripts/functions.sh

@@ -917,9 +917,16 @@ compare_sql_files() {
 
     # Compare the normalized files
     compare_files "$normalized_file1" "$normalized_file2"
-
-    # Clean up temporary files
+
+    compare_status=$?
+
+	# Clean up temporary files
     rm "$normalized_file1" "$normalized_file2"
+
+    # Exit with the status from compare_files if there are differences
+    if [ $compare_status -ne 0 ]; then
+        exit $compare_status
+    fi
 }
 
 
@@ -1044,3 +1051,62 @@ cutover_to_target() {
 
     yb-voyager initiate cutover to target ${args} $*
 }
+
+resolve_and_install_dependencies() {
+    local error_output="$1"
+
+    # Check if there are unmet dependencies
+    if echo "$error_output" | grep -q "The following packages have unmet dependencies:"; then
+        echo "Unmet dependencies found, resolving..."
+
+        # Extract dependencies, capturing package name and exact version
+        dependencies=$(echo "$error_output" | grep -oP 'Depends: \K[^\s]+ \(=[^\)]+')
+
+        # Remove " (= " part and format the dependencies to package-name=version format
+        dependencies=$(echo "$dependencies" | sed 's/ (= /=/g')
+
+        # Loop through dependencies and install them
+        for dep in $dependencies; do
+            echo "Installing dependency: $dep"
+            sudo apt-get install -y --allow-downgrades "$dep"
+        done
+    else
+        echo "No unmet dependencies found."
+    fi
+}
+
+verify_voyager_version() {
+    local expected_version="$1"
+    local actual_version
+
+    actual_version=$(yb-voyager version 2>/dev/null | grep -m1 '^VERSION=' | cut -d'=' -f2)
+
+    # Handle the special case where expected_version is "local"
+    if [[ "$expected_version" == "local" && "$actual_version" == "main" ]]; then
+        echo "Success: yb-voyager version matches the expected condition (installation: local, version: main)."
+        return 0
+    fi
+
+    # General case for matching versions
+    if [[ "$actual_version" == "$expected_version" ]]; then
+        echo "Success: yb-voyager version matches the expected version ($expected_version)."
+        return 0
+    else
+        echo "Error: yb-voyager version mismatch. Expected: $expected_version, Got: ${actual_version:-'Unknown'}."
+        return 1
+    fi
+}
+
+run_script() {
+    local script_file=$1
+    if [ -f "${script_file}" ]; then
+        step "Running script: ${script_file}"
+        source "${script_file}"
+        if [ $? -ne 0 ]; then
+            echo "Error: Script ${script_file} returned an error. Exiting."
+            return 1
+        fi
+    else
+        echo "Script ${script_file} not found, skipping."
+    fi
+}

migtests/scripts/upgrade-and-run-after.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+if [ $# -gt 3 ]; then
+    echo "Usage: $0 TEST_TYPE TEST_NAME [env.sh]"
+    exit 1
+fi
+
+export YB_VOYAGER_SEND_DIAGNOSTICS=false
+export TEST_TYPE=$1
+export TEST_NAME=$2
+
+export REPO_ROOT="${PWD}"
+export SCRIPTS="${REPO_ROOT}/migtests/scripts"
+export TESTS_DIR="${REPO_ROOT}/migtests/tests"
+export TEST_TYPE_DIR="${TESTS_DIR}/upgrade-tests/${TEST_TYPE}"
+export TEST_DIR="${TESTS_DIR}/${TEST_NAME}"
+export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"}
+export PYTHONPATH="${REPO_ROOT}/migtests/lib"
+export LAST_BREAKING_RELEASE=${LAST_BREAKING_RELEASE:-"1.8.5"}
+export RELEASE_TO_UPGRADE_TO=${RELEASE_TO_UPGRADE_TO:-"local"}
+
+# Load environment configurations
+if [ $3 != "" ]; then
+    if [ ! -f "${TEST_DIR}/$3" ]; then
+        echo "$3 file not found in the test directory"
+        exit 1
+    fi
+    source ${TEST_DIR}/$3
+else
+    source ${TEST_DIR}/env.sh
+fi
+source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh
+source ${SCRIPTS}/yugabytedb/env.sh
+source ${SCRIPTS}/functions.sh
+
+if [ -n "${SOURCE_DB_SSL_MODE}" ]; then
+   export EXPORT_DIR="${EXPORT_DIR}_ssl"
+   export SOURCE_DB_NAME="${SOURCE_DB_NAME}_ssl"
+   export TARGET_DB_NAME="${TARGET_DB_NAME}_ssl"
+fi
+
+main() {
+    echo ${REPO_ROOT}
+
+    step "START: ${TEST_NAME}"
+    print_env
+
+    pushd ${TEST_DIR}
+
+    # yes | ${REPO_ROOT}/installer_scripts/install-yb-voyager --version ${RELEASE_TO_UPGRADE_TO}
+    # source ~/.bashrc
+
+    step "Check Voyager version after upgrade."
+    yb-voyager version
+    verify_voyager_version ${RELEASE_TO_UPGRADE_TO}
+
+    # Run after steps
+    run_script "${TEST_TYPE_DIR}/after"
+
+    # Clean up
+    step "Cleaning up."
+    ./cleanup-db
+    rm -rf "${EXPORT_DIR}"
+    run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};"
+}
+
+main

migtests/scripts/upgrade-and-run-before.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+
+set -e
+set -x
+
+if [ $# -gt 3 ]; then
+    echo "Usage: $0 TEST_TYPE TEST_NAME [env.sh]"
+    exit 1
+fi
+
+export YB_VOYAGER_SEND_DIAGNOSTICS=false
+export TEST_TYPE=$1
+export TEST_NAME=$2
+
+export REPO_ROOT="${PWD}"
+export SCRIPTS="${REPO_ROOT}/migtests/scripts"
+export TESTS_DIR="${REPO_ROOT}/migtests/tests"
+export TEST_TYPE_DIR="${TESTS_DIR}/upgrade-tests/${TEST_TYPE}"
+export TEST_DIR="${TESTS_DIR}/${TEST_NAME}"
+export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"}
+export PYTHONPATH="${REPO_ROOT}/migtests/lib"
+export LAST_BREAKING_RELEASE=${LAST_BREAKING_RELEASE:-"1.8.6"}
+export RELEASE_TO_UPGRADE_TO=${RELEASE_TO_UPGRADE_TO:-"local"}
+
+# Load environment configurations
+if [ $3 != "" ]; then
+    if [ ! -f "${TEST_DIR}/$3" ]; then
+        echo "$3 file not found in the test directory"
+        exit 1
+    fi
+    source ${TEST_DIR}/$3
+else
+    source ${TEST_DIR}/env.sh
+fi
+source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh
+source ${SCRIPTS}/yugabytedb/env.sh
+source ${SCRIPTS}/functions.sh
+
+if [ -n "${SOURCE_DB_SSL_MODE}" ]; then
+   export EXPORT_DIR="${EXPORT_DIR}_ssl"
+   export SOURCE_DB_NAME="${SOURCE_DB_NAME}_ssl"
+   export TARGET_DB_NAME="${TARGET_DB_NAME}_ssl"
+fi
+
+main() {
+    echo ${REPO_ROOT}
+    echo "Deleting and recreating export-dir."
+    rm -rf ${EXPORT_DIR}
+    mkdir -p ${EXPORT_DIR}
+    chmod +x ${TEST_DIR}/init-db ${TEST_DIR}/cleanup-db
+
+    step "START: ${TEST_NAME}"
+    print_env
+
+    pushd ${TEST_DIR}
+
+    step "Initialize source database."
+    ./init-db
+
+    step "Create target database."
+	run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};"
+	if [ "${SOURCE_DB_TYPE}" = "postgresql" ] || [ "${SOURCE_DB_TYPE}" = "oracle" ]; then
+		run_ysql yugabyte "CREATE DATABASE ${TARGET_DB_NAME} with COLOCATION=TRUE"
+	else
+		run_ysql yugabyte "CREATE DATABASE ${TARGET_DB_NAME}"
+	fi
+
+
+    # LOG_FILE="/tmp/install-yb-voyager.log"
+    # # Delete the log file if it exists
+    # if [ -e "$LOG_FILE" ]; then
+    #     echo "Log file exists. Deleting it."
+    #     sudo rm -f "$LOG_FILE" || { echo "Failed to delete $LOG_FILE"; exit 1; }
+    # fi
+    # # Install older version of yb-voyager
+    # step "Installing Voyager version ${LAST_BREAKING_RELEASE}."
+
+    # yes | ${REPO_ROOT}/installer_scripts/install-yb-voyager --version ${LAST_BREAKING_RELEASE}
+
+    # source ~/.bashrc
+    step "Check Voyager version."
+
+    yb-voyager version
+    verify_voyager_version ${LAST_BREAKING_RELEASE}
+
+    step "Grant permissions."
+    grant_permissions ${SOURCE_DB_NAME} ${SOURCE_DB_TYPE} ${SOURCE_DB_SCHEMA}
+
+    # Run before steps
+    run_script "${TEST_TYPE_DIR}/before"
+
+}
+
+main

migtests/tests/pg/adventureworks/validate

@@ -136,6 +136,8 @@ def migration_completed_checks(tgt):
 	print(f"count returned - {cnt_type_domain}")
 	assert cnt_type_domain == 6
 
+	yb.verify_colocation(tgt, "postgresql")
+
 
 if __name__ == "__main__":
 	main()

migtests/tests/pg/partitions-with-indexes/env.sh

@@ -1,3 +1,3 @@
 export SOURCE_DB_TYPE="postgresql"
-export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"partitions"}
+export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"partitions_with_index"}
 export SOURCE_DB_SCHEMA="public,p1,p2"

migtests/tests/upgrade-tests/assess-export/after

@@ -0,0 +1,39 @@
+	step "Export schema."
+	export_schema
+	find ${EXPORT_DIR}/schema -name '*.sql' -printf "'%p'\n"| xargs grep -wh CREATE
+
+	step "Analyze schema."
+	analyze_schema
+	tail -20 ${EXPORT_DIR}/reports/schema_analysis_report.json
+
+	step "Fix schema."
+	if [ -x "${TEST_DIR}/fix-schema" ]
+	then
+		 "${TEST_DIR}/fix-schema"
+	fi
+
+	if [ "${MOVE_PK_FROM_ALTER_TO_CREATE}" = true ] ; then
+		"${SCRIPTS}/add-pk-from-alter-to-create"
+	fi
+
+	step "Import schema."
+	import_schema --continue-on-error true
+	run_ysql ${TARGET_DB_NAME} "\dt"
+
+	if [ -f "${EXPORT_DIR}/schema/failed.sql" ]; then
+	    if [ -f "${TEST_DIR}/expected_files/expected_failed.sql" ]; then
+	        # Both files exist, proceed with comparison
+	        compare_sql_files "${EXPORT_DIR}/schema/failed.sql" "${TEST_DIR}/expected_files/expected_failed.sql"
+	    else
+	        # expected_failed.sql does not exist, error out
+	        echo "Error: unexpected failed.sql found."
+	        cat "${EXPORT_DIR}/schema/failed.sql"
+	        exit 1
+	    fi
+	fi
+
+	step "Run Schema validations."
+	if [ -x "${TEST_DIR}/validate-schema" ]
+	then
+		 "${TEST_DIR}/validate-schema"
+	fi

migtests/tests/upgrade-tests/assess-export/before

@@ -0,0 +1,19 @@
+	step "Assess Migration"
+	if [ "${SOURCE_DB_TYPE}" = "postgresql" ] || [ "${SOURCE_DB_TYPE}" == "oracle" ]; then
+		assess_migration || {
+			cat_log_file "yb-voyager-assess-migration.log"
+			cat_file ${EXPORT_DIR}/assessment/metadata/yb-voyager-assessment.log
+		}
+
+		step "Validate Assessment Reports"
+		# Checking if the assessment reports were created
+		if [ -f "${EXPORT_DIR}/assessment/reports/migration_assessment_report.html" ] && [ -f "${EXPORT_DIR}/assessment/reports/migration_assessment_report.json" ]; then
+			echo "Assessment reports created successfully."
+			validate_failure_reasoning "${EXPORT_DIR}/assessment/reports/migration_assessment_report.json"
+			#TODO: Further validation to be added
+		else
+			echo "Error: Assessment reports were not created successfully."
+			cat_log_file "yb-voyager-assess-migration.log"
+			exit 1
+		fi
+	fi

migtests/tests/upgrade-tests/export-import/after

@@ -0,0 +1,49 @@
+    step "Run export-data-status"
+	export_data_status
+
+	expected_file="${TEST_DIR}/export_data_status-report.json"
+	actual_file="${EXPORT_DIR}/reports/export-data-status-report.json"
+
+	if [ "${EXPORT_TABLE_LIST}" != "" ]
+	then
+		expected_file="${TEST_DIR}/export-data-status-with-table-list-report.json"
+	fi
+
+	step "Verify export-data-status report"
+	verify_report ${expected_file} ${actual_file}
+
+	step "Analyze schema."
+	analyze_schema
+
+	step "Import schema."
+	import_schema
+	run_ysql ${TARGET_DB_NAME} "\dt"
+
+	step "Run Schema validations."
+	if [ -x "${TEST_DIR}/validate-schema" ]
+	then
+		 "${TEST_DIR}/validate-schema"
+	fi
+
+	step "Import data."
+	import_data
+
+	step "Import remaining schema (FK, index, and trigger) and Refreshing MViews if present."
+	import_schema --post-snapshot-import true --refresh-mviews=true
+	run_ysql ${TARGET_DB_NAME} "\di"
+	run_ysql ${TARGET_DB_NAME} "\dft" 
+
+	step "Run import-data-status"
+	import_data_status
+
+	expected_file="${TEST_DIR}/import_data_status-report.json"
+	actual_file="${EXPORT_DIR}/reports/import-data-status-report.json"
+
+	step "Verify import-data-status report"
+	verify_report ${expected_file} ${actual_file}
+
+	step "Run validations."
+	if [ -x "${TEST_DIR}/validate" ]
+	then
+		 "${TEST_DIR}/validate"
+	fi