diff --git a/migtests/lib/yb.py b/migtests/lib/yb.py index 854de33fe4..c9351928d8 100644 --- a/migtests/lib/yb.py +++ b/migtests/lib/yb.py @@ -52,7 +52,8 @@ def new_source_db(): def verify_colocation(tgt, source_db_type): print("Verifying the colocation of the tables") - json_file = "export-dir/assessment/reports/migration_assessment_report.json" + export_dir = os.getenv("EXPORT_DIR", "export-dir") + json_file = f"{export_dir}/assessment/reports/migration_assessment_report.json" sharded_tables, colocated_tables = fetch_sharded_and_colocated_tables(json_file) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 18f3ff9a28..8dd015b03f 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -917,9 +917,16 @@ compare_sql_files() { # Compare the normalized files compare_files "$normalized_file1" "$normalized_file2" - - # Clean up temporary files + + compare_status=$? + + # Clean up temporary files rm "$normalized_file1" "$normalized_file2" + + # Exit with the status from compare_files if there are differences + if [ $compare_status -ne 0 ]; then + exit $compare_status + fi } @@ -1044,3 +1051,62 @@ cutover_to_target() { yb-voyager initiate cutover to target ${args} $* } + +resolve_and_install_dependencies() { + local error_output="$1" + + # Check if there are unmet dependencies + if echo "$error_output" | grep -q "The following packages have unmet dependencies:"; then + echo "Unmet dependencies found, resolving..." + + # Extract dependencies, capturing package name and exact version + dependencies=$(echo "$error_output" | grep -oP 'Depends: \K[^\s]+ \(=[^\)]+') + + # Remove " (= " part and format the dependencies to package-name=version format + dependencies=$(echo "$dependencies" | sed 's/ (= /=/g') + + # Loop through dependencies and install them + for dep in $dependencies; do + echo "Installing dependency: $dep" + sudo apt-get install -y --allow-downgrades "$dep" + done + else + echo "No unmet dependencies found." + fi +} + +verify_voyager_version() { + local expected_version="$1" + local actual_version + + actual_version=$(yb-voyager version 2>/dev/null | grep -m1 '^VERSION=' | cut -d'=' -f2) + + # Handle the special case where expected_version is "local" + if [[ "$expected_version" == "local" && "$actual_version" == "main" ]]; then + echo "Success: yb-voyager version matches the expected condition (installation: local, version: main)." + return 0 + fi + + # General case for matching versions + if [[ "$actual_version" == "$expected_version" ]]; then + echo "Success: yb-voyager version matches the expected version ($expected_version)." + return 0 + else + echo "Error: yb-voyager version mismatch. Expected: $expected_version, Got: ${actual_version:-'Unknown'}." + return 1 + fi +} + +run_script() { + local script_file=$1 + if [ -f "${script_file}" ]; then + step "Running script: ${script_file}" + source "${script_file}" + if [ $? -ne 0 ]; then + echo "Error: Script ${script_file} returned an error. Exiting." + return 1 + fi + else + echo "Script ${script_file} not found, skipping." + fi +} \ No newline at end of file diff --git a/migtests/scripts/upgrade-and-run-after.sh b/migtests/scripts/upgrade-and-run-after.sh new file mode 100755 index 0000000000..8b60a64c1e --- /dev/null +++ b/migtests/scripts/upgrade-and-run-after.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +set -e +set -x + +if [ $# -gt 3 ]; then + echo "Usage: $0 TEST_TYPE TEST_NAME [env.sh]" + exit 1 +fi + +export YB_VOYAGER_SEND_DIAGNOSTICS=false +export TEST_TYPE=$1 +export TEST_NAME=$2 + +export REPO_ROOT="${PWD}" +export SCRIPTS="${REPO_ROOT}/migtests/scripts" +export TESTS_DIR="${REPO_ROOT}/migtests/tests" +export TEST_TYPE_DIR="${TESTS_DIR}/upgrade-tests/${TEST_TYPE}" +export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" +export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"} +export PYTHONPATH="${REPO_ROOT}/migtests/lib" +export LAST_BREAKING_RELEASE=${LAST_BREAKING_RELEASE:-"1.8.5"} +export RELEASE_TO_UPGRADE_TO=${RELEASE_TO_UPGRADE_TO:-"local"} + +# Load environment configurations +if [ $3 != "" ]; then + if [ ! -f "${TEST_DIR}/$3" ]; then + echo "$3 file not found in the test directory" + exit 1 + fi + source ${TEST_DIR}/$3 +else + source ${TEST_DIR}/env.sh +fi +source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh +source ${SCRIPTS}/yugabytedb/env.sh +source ${SCRIPTS}/functions.sh + +if [ -n "${SOURCE_DB_SSL_MODE}" ]; then + export EXPORT_DIR="${EXPORT_DIR}_ssl" + export SOURCE_DB_NAME="${SOURCE_DB_NAME}_ssl" + export TARGET_DB_NAME="${TARGET_DB_NAME}_ssl" +fi + +main() { + echo ${REPO_ROOT} + + step "START: ${TEST_NAME}" + print_env + + pushd ${TEST_DIR} + + # yes | ${REPO_ROOT}/installer_scripts/install-yb-voyager --version ${RELEASE_TO_UPGRADE_TO} + # source ~/.bashrc + + step "Check Voyager version after upgrade." + yb-voyager version + verify_voyager_version ${RELEASE_TO_UPGRADE_TO} + + # Run after steps + run_script "${TEST_TYPE_DIR}/after" + + # Clean up + step "Cleaning up." + ./cleanup-db + rm -rf "${EXPORT_DIR}" + run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" +} + +main \ No newline at end of file diff --git a/migtests/scripts/upgrade-and-run-before.sh b/migtests/scripts/upgrade-and-run-before.sh new file mode 100755 index 0000000000..16ba4ca75e --- /dev/null +++ b/migtests/scripts/upgrade-and-run-before.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +set -e +set -x + +if [ $# -gt 3 ]; then + echo "Usage: $0 TEST_TYPE TEST_NAME [env.sh]" + exit 1 +fi + +export YB_VOYAGER_SEND_DIAGNOSTICS=false +export TEST_TYPE=$1 +export TEST_NAME=$2 + +export REPO_ROOT="${PWD}" +export SCRIPTS="${REPO_ROOT}/migtests/scripts" +export TESTS_DIR="${REPO_ROOT}/migtests/tests" +export TEST_TYPE_DIR="${TESTS_DIR}/upgrade-tests/${TEST_TYPE}" +export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" +export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"} +export PYTHONPATH="${REPO_ROOT}/migtests/lib" +export LAST_BREAKING_RELEASE=${LAST_BREAKING_RELEASE:-"1.8.6"} +export RELEASE_TO_UPGRADE_TO=${RELEASE_TO_UPGRADE_TO:-"local"} + +# Load environment configurations +if [ $3 != "" ]; then + if [ ! -f "${TEST_DIR}/$3" ]; then + echo "$3 file not found in the test directory" + exit 1 + fi + source ${TEST_DIR}/$3 +else + source ${TEST_DIR}/env.sh +fi +source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh +source ${SCRIPTS}/yugabytedb/env.sh +source ${SCRIPTS}/functions.sh + +if [ -n "${SOURCE_DB_SSL_MODE}" ]; then + export EXPORT_DIR="${EXPORT_DIR}_ssl" + export SOURCE_DB_NAME="${SOURCE_DB_NAME}_ssl" + export TARGET_DB_NAME="${TARGET_DB_NAME}_ssl" +fi + +main() { + echo ${REPO_ROOT} + echo "Deleting and recreating export-dir." + rm -rf ${EXPORT_DIR} + mkdir -p ${EXPORT_DIR} + chmod +x ${TEST_DIR}/init-db ${TEST_DIR}/cleanup-db + + step "START: ${TEST_NAME}" + print_env + + pushd ${TEST_DIR} + + step "Initialize source database." + ./init-db + + step "Create target database." + run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" + if [ "${SOURCE_DB_TYPE}" = "postgresql" ] || [ "${SOURCE_DB_TYPE}" = "oracle" ]; then + run_ysql yugabyte "CREATE DATABASE ${TARGET_DB_NAME} with COLOCATION=TRUE" + else + run_ysql yugabyte "CREATE DATABASE ${TARGET_DB_NAME}" + fi + + + # LOG_FILE="/tmp/install-yb-voyager.log" + # # Delete the log file if it exists + # if [ -e "$LOG_FILE" ]; then + # echo "Log file exists. Deleting it." + # sudo rm -f "$LOG_FILE" || { echo "Failed to delete $LOG_FILE"; exit 1; } + # fi + # # Install older version of yb-voyager + # step "Installing Voyager version ${LAST_BREAKING_RELEASE}." + + # yes | ${REPO_ROOT}/installer_scripts/install-yb-voyager --version ${LAST_BREAKING_RELEASE} + + # source ~/.bashrc + step "Check Voyager version." + + yb-voyager version + verify_voyager_version ${LAST_BREAKING_RELEASE} + + step "Grant permissions." + grant_permissions ${SOURCE_DB_NAME} ${SOURCE_DB_TYPE} ${SOURCE_DB_SCHEMA} + + # Run before steps + run_script "${TEST_TYPE_DIR}/before" + +} + +main \ No newline at end of file diff --git a/migtests/tests/pg/adventureworks/validate b/migtests/tests/pg/adventureworks/validate index a10348bb64..20f2bc3f0e 100755 --- a/migtests/tests/pg/adventureworks/validate +++ b/migtests/tests/pg/adventureworks/validate @@ -136,6 +136,8 @@ def migration_completed_checks(tgt): print(f"count returned - {cnt_type_domain}") assert cnt_type_domain == 6 + yb.verify_colocation(tgt, "postgresql") + if __name__ == "__main__": main() diff --git a/migtests/tests/pg/partitions-with-indexes/env.sh b/migtests/tests/pg/partitions-with-indexes/env.sh index 6b22305323..7f216dc8da 100644 --- a/migtests/tests/pg/partitions-with-indexes/env.sh +++ b/migtests/tests/pg/partitions-with-indexes/env.sh @@ -1,3 +1,3 @@ export SOURCE_DB_TYPE="postgresql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"partitions"} +export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"partitions_with_index"} export SOURCE_DB_SCHEMA="public,p1,p2" diff --git a/migtests/tests/upgrade-tests/assess-export/after b/migtests/tests/upgrade-tests/assess-export/after new file mode 100644 index 0000000000..3a14139f4f --- /dev/null +++ b/migtests/tests/upgrade-tests/assess-export/after @@ -0,0 +1,39 @@ + step "Export schema." + export_schema + find ${EXPORT_DIR}/schema -name '*.sql' -printf "'%p'\n"| xargs grep -wh CREATE + + step "Analyze schema." + analyze_schema + tail -20 ${EXPORT_DIR}/reports/schema_analysis_report.json + + step "Fix schema." + if [ -x "${TEST_DIR}/fix-schema" ] + then + "${TEST_DIR}/fix-schema" + fi + + if [ "${MOVE_PK_FROM_ALTER_TO_CREATE}" = true ] ; then + "${SCRIPTS}/add-pk-from-alter-to-create" + fi + + step "Import schema." + import_schema --continue-on-error true + run_ysql ${TARGET_DB_NAME} "\dt" + + if [ -f "${EXPORT_DIR}/schema/failed.sql" ]; then + if [ -f "${TEST_DIR}/expected_files/expected_failed.sql" ]; then + # Both files exist, proceed with comparison + compare_sql_files "${EXPORT_DIR}/schema/failed.sql" "${TEST_DIR}/expected_files/expected_failed.sql" + else + # expected_failed.sql does not exist, error out + echo "Error: unexpected failed.sql found." + cat "${EXPORT_DIR}/schema/failed.sql" + exit 1 + fi + fi + + step "Run Schema validations." + if [ -x "${TEST_DIR}/validate-schema" ] + then + "${TEST_DIR}/validate-schema" + fi \ No newline at end of file diff --git a/migtests/tests/upgrade-tests/assess-export/before b/migtests/tests/upgrade-tests/assess-export/before new file mode 100644 index 0000000000..cd1c5517f7 --- /dev/null +++ b/migtests/tests/upgrade-tests/assess-export/before @@ -0,0 +1,19 @@ + step "Assess Migration" + if [ "${SOURCE_DB_TYPE}" = "postgresql" ] || [ "${SOURCE_DB_TYPE}" == "oracle" ]; then + assess_migration || { + cat_log_file "yb-voyager-assess-migration.log" + cat_file ${EXPORT_DIR}/assessment/metadata/yb-voyager-assessment.log + } + + step "Validate Assessment Reports" + # Checking if the assessment reports were created + if [ -f "${EXPORT_DIR}/assessment/reports/migration_assessment_report.html" ] && [ -f "${EXPORT_DIR}/assessment/reports/migration_assessment_report.json" ]; then + echo "Assessment reports created successfully." + validate_failure_reasoning "${EXPORT_DIR}/assessment/reports/migration_assessment_report.json" + #TODO: Further validation to be added + else + echo "Error: Assessment reports were not created successfully." + cat_log_file "yb-voyager-assess-migration.log" + exit 1 + fi + fi diff --git a/migtests/tests/upgrade-tests/export-import/after b/migtests/tests/upgrade-tests/export-import/after new file mode 100644 index 0000000000..bac5f8163c --- /dev/null +++ b/migtests/tests/upgrade-tests/export-import/after @@ -0,0 +1,49 @@ + step "Run export-data-status" + export_data_status + + expected_file="${TEST_DIR}/export_data_status-report.json" + actual_file="${EXPORT_DIR}/reports/export-data-status-report.json" + + if [ "${EXPORT_TABLE_LIST}" != "" ] + then + expected_file="${TEST_DIR}/export-data-status-with-table-list-report.json" + fi + + step "Verify export-data-status report" + verify_report ${expected_file} ${actual_file} + + step "Analyze schema." + analyze_schema + + step "Import schema." + import_schema + run_ysql ${TARGET_DB_NAME} "\dt" + + step "Run Schema validations." + if [ -x "${TEST_DIR}/validate-schema" ] + then + "${TEST_DIR}/validate-schema" + fi + + step "Import data." + import_data + + step "Import remaining schema (FK, index, and trigger) and Refreshing MViews if present." + import_schema --post-snapshot-import true --refresh-mviews=true + run_ysql ${TARGET_DB_NAME} "\di" + run_ysql ${TARGET_DB_NAME} "\dft" + + step "Run import-data-status" + import_data_status + + expected_file="${TEST_DIR}/import_data_status-report.json" + actual_file="${EXPORT_DIR}/reports/import-data-status-report.json" + + step "Verify import-data-status report" + verify_report ${expected_file} ${actual_file} + + step "Run validations." + if [ -x "${TEST_DIR}/validate" ] + then + "${TEST_DIR}/validate" + fi \ No newline at end of file diff --git a/migtests/tests/upgrade-tests/export-import/before b/migtests/tests/upgrade-tests/export-import/before new file mode 100644 index 0000000000..945c54e9c2 --- /dev/null +++ b/migtests/tests/upgrade-tests/export-import/before @@ -0,0 +1,49 @@ + step "Assess Migration" + if [ "${SOURCE_DB_TYPE}" = "postgresql" ] || [ "${SOURCE_DB_TYPE}" == "oracle" ]; then + assess_migration || { + cat_log_file "yb-voyager-assess-migration.log" + cat_file ${EXPORT_DIR}/assessment/metadata/yb-voyager-assessment.log + } + post_assess_migration + fi + + step "Export schema." + export_schema + + step "Analyze schema." + analyze_schema + + step "Fix schema." + if [ -x "${TEST_DIR}/fix-schema" ] + then + "${TEST_DIR}/fix-schema" + fi + + if [ "${MOVE_PK_FROM_ALTER_TO_CREATE}" = true ] ; then + "${SCRIPTS}/add-pk-from-alter-to-create" + fi + + step "Export data." + # false if exit code of export_data is non-zero + export_data || { + cat_log_file "yb-voyager-export-data.log" + cat_log_file "debezium-source_db_exporter.log" + exit 1 + } + + cat ${EXPORT_DIR}/data/export_status.json || echo "No export_status.json found." + cat ${EXPORT_DIR}/metainfo/dataFileDescriptor.json + + step "Run export-data-status" + export_data_status + + expected_file="${TEST_DIR}/export_data_status-report.json" + actual_file="${EXPORT_DIR}/reports/export-data-status-report.json" + + if [ "${EXPORT_TABLE_LIST}" != "" ] + then + expected_file="${TEST_DIR}/export-data-status-with-table-list-report.json" + fi + + step "Verify export-data-status report" + verify_report ${expected_file} ${actual_file} diff --git a/migtests/tests/upgrade-tests/import-status/after b/migtests/tests/upgrade-tests/import-status/after new file mode 100644 index 0000000000..5fece0e673 --- /dev/null +++ b/migtests/tests/upgrade-tests/import-status/after @@ -0,0 +1,22 @@ + step "Run export-data-status" + export_data_status + + expected_file="${TEST_DIR}/export_data_status-report.json" + actual_file="${EXPORT_DIR}/reports/export-data-status-report.json" + + if [ "${EXPORT_TABLE_LIST}" != "" ] + then + expected_file="${TEST_DIR}/export-data-status-with-table-list-report.json" + fi + + step "Verify export-data-status report" + verify_report ${expected_file} ${actual_file} + + step "Run import-data-status" + import_data_status + + expected_file="${TEST_DIR}/import_data_status-report.json" + actual_file="${EXPORT_DIR}/reports/import-data-status-report.json" + + step "Verify import-data-status report" + verify_report ${expected_file} ${actual_file} diff --git a/migtests/tests/upgrade-tests/import-status/before b/migtests/tests/upgrade-tests/import-status/before new file mode 100644 index 0000000000..73b7a45156 --- /dev/null +++ b/migtests/tests/upgrade-tests/import-status/before @@ -0,0 +1,34 @@ + step "Export schema." + export_schema + + step "Export data." + # false if exit code of export_data is non-zero + export_data || { + cat_log_file "yb-voyager-export-data.log" + cat_log_file "debezium-source_db_exporter.log" + exit 1 + } + + cat ${EXPORT_DIR}/data/export_status.json || echo "No export_status.json found." + cat ${EXPORT_DIR}/metainfo/dataFileDescriptor.json + + step "Import schema." + import_schema + run_ysql ${TARGET_DB_NAME} "\dt" + + step "Import data." + import_data + + step "Import remaining schema (FK, index, and trigger) and Refreshing MViews if present." + import_schema --post-snapshot-import true --refresh-mviews=true + run_ysql ${TARGET_DB_NAME} "\di" + run_ysql ${TARGET_DB_NAME} "\dft" + + step "Run import-data-status" + import_data_status + + expected_file="${TEST_DIR}/import_data_status-report.json" + actual_file="${EXPORT_DIR}/reports/import-data-status-report.json" + + step "Verify import-data-status report" + verify_report ${expected_file} ${actual_file} diff --git a/migtests/tests/upgrade-tests/schema-data/after b/migtests/tests/upgrade-tests/schema-data/after new file mode 100644 index 0000000000..24fd75d325 --- /dev/null +++ b/migtests/tests/upgrade-tests/schema-data/after @@ -0,0 +1,47 @@ + step "Export data." + # false if exit code of export_data is non-zero + export_data || { + cat_log_file "yb-voyager-export-data.log" + cat_log_file "debezium-source_db_exporter.log" + exit 1 + } + + cat ${EXPORT_DIR}/data/export_status.json || echo "No export_status.json found." + cat ${EXPORT_DIR}/metainfo/dataFileDescriptor.json + + step "Run export-data-status" + export_data_status + + expected_file="${TEST_DIR}/export_data_status-report.json" + actual_file="${EXPORT_DIR}/reports/export-data-status-report.json" + + if [ "${EXPORT_TABLE_LIST}" != "" ] + then + expected_file="${TEST_DIR}/export-data-status-with-table-list-report.json" + fi + + step "Verify export-data-status report" + verify_report ${expected_file} ${actual_file} + + step "Import data." + import_data + + step "Import remaining schema (FK, index, and trigger) and Refreshing MViews if present." + import_schema --post-snapshot-import true --refresh-mviews=true + run_ysql ${TARGET_DB_NAME} "\di" + run_ysql ${TARGET_DB_NAME} "\dft" + + step "Run import-data-status" + import_data_status + + expected_file="${TEST_DIR}/import_data_status-report.json" + actual_file="${EXPORT_DIR}/reports/import-data-status-report.json" + + step "Verify import-data-status report" + verify_report ${expected_file} ${actual_file} + + step "Run validations." + if [ -x "${TEST_DIR}/validate" ] + then + "${TEST_DIR}/validate" + fi diff --git a/migtests/tests/upgrade-tests/schema-data/before b/migtests/tests/upgrade-tests/schema-data/before new file mode 100644 index 0000000000..20af61bccf --- /dev/null +++ b/migtests/tests/upgrade-tests/schema-data/before @@ -0,0 +1,34 @@ + step "Assess Migration" + if [ "${SOURCE_DB_TYPE}" = "postgresql" ] || [ "${SOURCE_DB_TYPE}" == "oracle" ]; then + assess_migration || { + cat_log_file "yb-voyager-assess-migration.log" + cat_file ${EXPORT_DIR}/assessment/metadata/yb-voyager-assessment.log + } + post_assess_migration + fi + + step "Export schema." + export_schema + + step "Analyze schema." + analyze_schema + + step "Fix schema." + if [ -x "${TEST_DIR}/fix-schema" ] + then + "${TEST_DIR}/fix-schema" + fi + + if [ "${MOVE_PK_FROM_ALTER_TO_CREATE}" = true ] ; then + "${SCRIPTS}/add-pk-from-alter-to-create" + fi + + step "Import schema." + import_schema + run_ysql ${TARGET_DB_NAME} "\dt" + + step "Run Schema validations." + if [ -x "${TEST_DIR}/validate-schema" ] + then + "${TEST_DIR}/validate-schema" + fi \ No newline at end of file