From f4da0f49db8c7225b5a534fb00a186a36d1895b7 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Sun, 1 Dec 2024 07:55:34 +0000 Subject: [PATCH 01/21] Initial changes to enable parallel tests --- migtests/lib/yb.py | 3 +- migtests/scripts/add-pk-from-alter-to-create | 2 +- migtests/scripts/functions.sh | 44 ++++++++++++++----- .../scripts/live-migration-fallb-run-test.sh | 18 +++++++- .../scripts/live-migration-fallf-run-test.sh | 17 +++++-- migtests/scripts/live-migration-run-test.sh | 18 +++++++- migtests/scripts/run-schema-migration.sh | 2 +- migtests/scripts/run-test-export-data.sh | 2 +- migtests/scripts/run-test.sh | 26 +++++++++-- .../scripts/run-validate-assessment-report.sh | 2 +- .../run-validate-bulk-assessment-report.sh | 3 ++ migtests/scripts/yugabytedb/env.sh | 9 ++-- migtests/tests/mysql/datatypes/env.sh | 1 - .../oracle/bulk-assessment-test/cleanup-db | 11 +++++ .../tests/oracle/bulk-assessment-test/init-db | 23 ++++++---- migtests/tests/oracle/partitions/env.sh | 2 - migtests/tests/oracle/sequences/env.sh | 2 - migtests/tests/pg/datatypes/env.sh | 1 - migtests/tests/pg/partitions/env.sh | 1 - migtests/tests/pg/partitions/fix-schema | 18 ++++---- 20 files changed, 151 insertions(+), 54 deletions(-) create mode 100644 migtests/tests/oracle/bulk-assessment-test/cleanup-db diff --git a/migtests/lib/yb.py b/migtests/lib/yb.py index 854de33fe4..c9351928d8 100644 --- a/migtests/lib/yb.py +++ b/migtests/lib/yb.py @@ -52,7 +52,8 @@ def new_source_db(): def verify_colocation(tgt, source_db_type): print("Verifying the colocation of the tables") - json_file = "export-dir/assessment/reports/migration_assessment_report.json" + export_dir = os.getenv("EXPORT_DIR", "export-dir") + json_file = f"{export_dir}/assessment/reports/migration_assessment_report.json" sharded_tables, colocated_tables = fetch_sharded_and_colocated_tables(json_file) diff --git a/migtests/scripts/add-pk-from-alter-to-create b/migtests/scripts/add-pk-from-alter-to-create index 83d0ef3eb2..77f1ab5b7b 100755 --- a/migtests/scripts/add-pk-from-alter-to-create +++ b/migtests/scripts/add-pk-from-alter-to-create @@ -5,7 +5,7 @@ import os import re import shutil -table_file_path = os.getenv('TEST_DIR')+'/export-dir/schema/tables/table.sql' +table_file_path = os.getenv('EXPORT_DIR')+'/schema/tables/table.sql' #copy the table_file_path to table_file_path+'.old' src = table_file_path diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 5b10574cc3..4abd4f5e75 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -134,7 +134,8 @@ grant_user_permission_oracle(){ */ GRANT FLASHBACK ANY TABLE TO ybvoyager; EOF - run_sqlplus_as_sys ${db_name} "oracle-inputs.sql" + run_sqlplus_as_sys ${db_name} "oracle-inputs.sql" + rm oracle-inputs.sql } @@ -149,6 +150,8 @@ EOF run_sqlplus_as_sys ${pdb_name} "create-pdb-tablespace.sql" cp ${SCRIPTS}/oracle/live-grants.sql oracle-inputs.sql run_sqlplus_as_sys ${cdb_name} "oracle-inputs.sql" + rm create-pdb-tablespace.sql + rm oracle-inputs.sql } grant_permissions_for_live_migration_pg() { @@ -191,7 +194,7 @@ run_sqlplus_as_sys() { run_sqlplus_as_schema_owner() { db_name=$1 sql=$2 - conn_string="${SOURCE_DB_USER_SCHEMA_OWNER}/${SOURCE_DB_USER_SCHEMA_OWNER_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${db_name}" + conn_string="${SOURCE_DB_SCHEMA}/${SOURCE_DB_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${db_name}" echo exit | sqlplus -f "${conn_string}" @"${sql}" } @@ -623,16 +626,6 @@ get_value_from_msr(){ echo $val } -create_ff_schema(){ - db_name=$1 - - cat > create-ff-schema.sql << EOF - CREATE USER FF_SCHEMA IDENTIFIED BY "password"; - GRANT all privileges to FF_SCHEMA; -EOF - run_sqlplus_as_sys ${db_name} "create-ff-schema.sql" -} - set_replica_identity(){ db_schema=$1 cat > alter_replica_identity.sql < create-oracle-schema.sql << EOF + CREATE USER ${source_db} IDENTIFIED BY "password"; + GRANT all privileges to ${source_db}; +EOF + run_sqlplus_as_sys ${SOURCE_DB_NAME} "create-oracle-schema.sql" + rm create-oracle-schema.sql + ;; + *) + echo "ERROR: Source DB not created for ${SOURCE_DB_TYPE}" + exit 1 + ;; + esac +} \ No newline at end of file diff --git a/migtests/scripts/live-migration-fallb-run-test.sh b/migtests/scripts/live-migration-fallb-run-test.sh index fc4fb886a5..2bc49b7bc7 100755 --- a/migtests/scripts/live-migration-fallb-run-test.sh +++ b/migtests/scripts/live-migration-fallb-run-test.sh @@ -18,7 +18,8 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"} +export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" +export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_fallb_export-dir"} export QUEUE_SEGMENT_MAX_BYTES=400 export PYTHONPATH="${REPO_ROOT}/migtests/lib" @@ -37,6 +38,15 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi +if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then + export SOURCE_DB_NAME="${NORMALIZED_TEST_NAME}_fallb" +elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then + export SOURCE_DB_SCHEMA="${NORMALIZED_TEST_NAME}_fallb" +else + echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" + exit 1 +fi + source ${SCRIPTS}/yugabytedb/env.sh source ${SCRIPTS}/functions.sh @@ -56,6 +66,10 @@ main() { pushd ${TEST_DIR} step "Initialise source database." + if [ "${SOURCE_DB_TYPE}" = "oracle" ] + then + create_source_db ${SOURCE_DB_SCHEMA} + fi ./init-db step "Grant source database user permissions for live migration" @@ -264,7 +278,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/live-migration-fallf-run-test.sh b/migtests/scripts/live-migration-fallf-run-test.sh index 175630f70a..14356e4536 100755 --- a/migtests/scripts/live-migration-fallf-run-test.sh +++ b/migtests/scripts/live-migration-fallf-run-test.sh @@ -18,7 +18,8 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"} +export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" +export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_fallf_export-dir"} export PYTHONPATH="${REPO_ROOT}/migtests/lib" export PATH="${PATH}:/usr/lib/oracle/21/client64/bin" @@ -38,6 +39,15 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi +if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then + export SOURCE_DB_NAME="${NORMALIZED_TEST_NAME}_fallf" +elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then + export SOURCE_DB_SCHEMA="${NORMALIZED_TEST_NAME}_fallf" +else + echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" + exit 1 +fi + source ${SCRIPTS}/${SOURCE_DB_TYPE}/ff_env.sh source ${SCRIPTS}/yugabytedb/env.sh @@ -62,7 +72,8 @@ main() { if [ "${SOURCE_DB_TYPE}" = "oracle" ] then - create_ff_schema ${SOURCE_REPLICA_DB_NAME} + create_source_db ${SOURCE_DB_SCHEMA} + create_source_db ${SOURCE_REPLICA_DB_SCHEMA} run_sqlplus_as_sys ${SOURCE_REPLICA_DB_NAME} ${SCRIPTS}/oracle/create_metadata_tables.sql fi ./init-db @@ -286,7 +297,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/live-migration-run-test.sh b/migtests/scripts/live-migration-run-test.sh index 1594f1449c..2207b28c57 100755 --- a/migtests/scripts/live-migration-run-test.sh +++ b/migtests/scripts/live-migration-run-test.sh @@ -18,7 +18,8 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"} +export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" +export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_live_export-dir"} export QUEUE_SEGMENT_MAX_BYTES=400 export PYTHONPATH="${REPO_ROOT}/migtests/lib" @@ -37,6 +38,15 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi +if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then + export SOURCE_DB_NAME="${NORMALIZED_TEST_NAME}_live" +elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then + export SOURCE_DB_SCHEMA="${NORMALIZED_TEST_NAME}_live" +else + echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" + exit 1 +fi + source ${SCRIPTS}/yugabytedb/env.sh source ${SCRIPTS}/functions.sh @@ -56,6 +66,10 @@ main() { pushd ${TEST_DIR} step "Initialise source database." + if [ "${SOURCE_DB_TYPE}" = "oracle" ] + then + create_source_db ${SOURCE_DB_SCHEMA} + fi ./init-db step "Grant source database user permissions for live migration" @@ -218,7 +232,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/run-schema-migration.sh b/migtests/scripts/run-schema-migration.sh index 0f37bab501..8a142cf152 100755 --- a/migtests/scripts/run-schema-migration.sh +++ b/migtests/scripts/run-schema-migration.sh @@ -139,7 +139,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/run-test-export-data.sh b/migtests/scripts/run-test-export-data.sh index 04e84bdd22..dcbeecd362 100755 --- a/migtests/scripts/run-test-export-data.sh +++ b/migtests/scripts/run-test-export-data.sh @@ -74,7 +74,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index f0d9408cf8..5f3d75e26c 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -17,7 +17,8 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/export-dir"} +export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" +export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_offline_export-dir"} export PYTHONPATH="${REPO_ROOT}/migtests/lib" @@ -33,9 +34,20 @@ then else source ${TEST_DIR}/env.sh fi + source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh -source ${SCRIPTS}/yugabytedb/env.sh + +if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then + export SOURCE_DB_NAME="${NORMALIZED_TEST_NAME}_offline" +elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then + export SOURCE_DB_SCHEMA="${NORMALIZED_TEST_NAME}_offline" +else + echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" + exit 1 +fi + +source ${SCRIPTS}/yugabytedb/env.sh source ${SCRIPTS}/functions.sh main() { @@ -52,6 +64,14 @@ main() { pushd ${TEST_DIR} step "Initialise source database." + if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then + create_source_db ${SOURCE_DB_NAME} +elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then + create_source_db ${SOURCE_DB_SCHEMA} +else + echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" + exit 1 +fi ./init-db step "Grant source database user permissions" @@ -190,7 +210,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" run_ysql yugabyte "DROP DATABASE IF EXISTS ${TARGET_DB_NAME};" } diff --git a/migtests/scripts/run-validate-assessment-report.sh b/migtests/scripts/run-validate-assessment-report.sh index 5cdac1d392..75e3d8875c 100755 --- a/migtests/scripts/run-validate-assessment-report.sh +++ b/migtests/scripts/run-validate-assessment-report.sh @@ -94,7 +94,7 @@ main() { step "Clean up" ./cleanup-db - rm -rf "${EXPORT_DIR}/*" + rm -rf "${EXPORT_DIR}" } main diff --git a/migtests/scripts/run-validate-bulk-assessment-report.sh b/migtests/scripts/run-validate-bulk-assessment-report.sh index 05b43c87ab..ac67773b1d 100755 --- a/migtests/scripts/run-validate-bulk-assessment-report.sh +++ b/migtests/scripts/run-validate-bulk-assessment-report.sh @@ -54,6 +54,7 @@ main() { echo "Assigning permissions to the export-dir to execute init-db, cleanup-db scripts" chmod +x ${TEST_DIR}/init-db + chmod +x ${TEST_DIR}/cleanup-db step "START: ${TEST_NAME}" print_env @@ -109,6 +110,8 @@ main() { fi step "Clean up" + ./cleanup-db + rm -rf "${BULK_ASSESSMENT_DIR}" } diff --git a/migtests/scripts/yugabytedb/env.sh b/migtests/scripts/yugabytedb/env.sh index a288b14ffc..10b9e70e5f 100644 --- a/migtests/scripts/yugabytedb/env.sh +++ b/migtests/scripts/yugabytedb/env.sh @@ -7,7 +7,10 @@ export TARGET_DB_ADMIN_PASSWORD=${TARGET_DB_ADMIN_PASSWORD:-''} export TARGET_DB_SCHEMA=${TARGET_DB_SCHEMA:-'public'} # The PG driver, used to connect to YB, is case-sensitive about database name. -if [ "${TARGET_DB_NAME}" == "" ] -then - export TARGET_DB_NAME=`echo ${SOURCE_DB_NAME} | tr [A-Z] [a-z]` +if [ "${TARGET_DB_NAME}" == "" ]; then + if [ "${SOURCE_DB_TYPE}" == "oracle" ]; then + export TARGET_DB_NAME=$(echo ${SOURCE_DB_SCHEMA} | tr [A-Z] [a-z]) + else + export TARGET_DB_NAME=$(echo ${SOURCE_DB_NAME} | tr [A-Z] [a-z]) + fi fi diff --git a/migtests/tests/mysql/datatypes/env.sh b/migtests/tests/mysql/datatypes/env.sh index 1d4759f090..aedc7cffd6 100644 --- a/migtests/tests/mysql/datatypes/env.sh +++ b/migtests/tests/mysql/datatypes/env.sh @@ -1,2 +1 @@ export SOURCE_DB_TYPE="mysql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"datatypes"} diff --git a/migtests/tests/oracle/bulk-assessment-test/cleanup-db b/migtests/tests/oracle/bulk-assessment-test/cleanup-db new file mode 100644 index 0000000000..07ebc07a8e --- /dev/null +++ b/migtests/tests/oracle/bulk-assessment-test/cleanup-db @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -e +set -x + +source ${SCRIPTS}/functions.sh + +for schema in TEST_SCHEMA TEST_SCHEMA2; do + export SOURCE_DB_SCHEMA=${schema} + run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TESTS_DIR}/oracle/utils/delete_full_schema +done \ No newline at end of file diff --git a/migtests/tests/oracle/bulk-assessment-test/init-db b/migtests/tests/oracle/bulk-assessment-test/init-db index b8acab5175..97add76d37 100755 --- a/migtests/tests/oracle/bulk-assessment-test/init-db +++ b/migtests/tests/oracle/bulk-assessment-test/init-db @@ -5,21 +5,28 @@ set -x source ${SCRIPTS}/functions.sh -echo "Deleting existing data in source database" -run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TESTS_DIR}/oracle/utils/delete_full_schema +export SOURCE_DB_SCHEMA="TEST_SCHEMA" + +create_source_db ${SOURCE_DB_SCHEMA} + +./cleanup-db +# echo "Deleting existing data in source database" +# run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TESTS_DIR}/oracle/utils/delete_full_schema echo "Initialising source database 1 & inserting data" run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TEST_DIR}/../assessment-report-test/oracle_assessment_report.sql echo "Initialising source database 2 & inserting data" -cat > create-schema2.sql << EOF - CREATE USER TEST_SCHEMA2 identified by "password"; - GRANT ALL PRIVILEGES TO TEST_SCHEMA2; -EOF - run_sqlplus_as_sys ${SOURCE_DB_NAME} "create-schema2.sql" +# cat > create-schema2.sql << EOF +# CREATE USER TEST_SCHEMA2 identified by "password"; +# GRANT ALL PRIVILEGES TO TEST_SCHEMA2; +# EOF +# run_sqlplus_as_sys ${SOURCE_DB_NAME} "create-schema2.sql" + +export SOURCE_DB_SCHEMA="TEST_SCHEMA2" -export SOURCE_DB_USER_SCHEMA_OWNER="TEST_SCHEMA2" +create_source_db ${SOURCE_DB_SCHEMA} run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TEST_DIR}/../partitions/partition_schema.sql run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TEST_DIR}/../partitions/partition_data.sql diff --git a/migtests/tests/oracle/partitions/env.sh b/migtests/tests/oracle/partitions/env.sh index 05489d8a99..515cf7abbe 100644 --- a/migtests/tests/oracle/partitions/env.sh +++ b/migtests/tests/oracle/partitions/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"partition_test"} diff --git a/migtests/tests/oracle/sequences/env.sh b/migtests/tests/oracle/sequences/env.sh index ef3b4aa439..0357fe1d0e 100644 --- a/migtests/tests/oracle/sequences/env.sh +++ b/migtests/tests/oracle/sequences/env.sh @@ -1,3 +1 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"oracle_sequences_test"} diff --git a/migtests/tests/pg/datatypes/env.sh b/migtests/tests/pg/datatypes/env.sh index 328c0615f3..3606e020cc 100644 --- a/migtests/tests/pg/datatypes/env.sh +++ b/migtests/tests/pg/datatypes/env.sh @@ -1,3 +1,2 @@ export SOURCE_DB_TYPE="postgresql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"pg_datatypes"} export SOURCE_DB_SCHEMA="public" diff --git a/migtests/tests/pg/partitions/env.sh b/migtests/tests/pg/partitions/env.sh index 9ba7757697..bbca0e5017 100644 --- a/migtests/tests/pg/partitions/env.sh +++ b/migtests/tests/pg/partitions/env.sh @@ -1,4 +1,3 @@ export SOURCE_DB_TYPE="postgresql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"partitions"} export SOURCE_DB_SCHEMA="public,p1,p2" export MOVE_PK_FROM_ALTER_TO_CREATE="true" \ No newline at end of file diff --git a/migtests/tests/pg/partitions/fix-schema b/migtests/tests/pg/partitions/fix-schema index 57007db3f5..8a341cc602 100755 --- a/migtests/tests/pg/partitions/fix-schema +++ b/migtests/tests/pg/partitions/fix-schema @@ -3,15 +3,15 @@ set -e set -x -sed -i 's/p2\.boston/p2.boston_region/g' $TEST_DIR/export-dir/schema/tables/table.sql -sed -i 's/p2\.london/p2.london_region/g' $TEST_DIR/export-dir/schema/tables/table.sql -sed -i 's/p2\.sydney/p2.sydney_region/g' $TEST_DIR/export-dir/schema/tables/table.sql +sed -i 's/p2\.boston/p2.boston_region/g' ${EXPORT_DIR}/schema/tables/table.sql +sed -i 's/p2\.london/p2.london_region/g' ${EXPORT_DIR}/schema/tables/table.sql +sed -i 's/p2\.sydney/p2.sydney_region/g' ${EXPORT_DIR}/schema/tables/table.sql -sed -i 's/p2\.boston/p2.boston_region/g' $TEST_DIR/export-dir/schema/tables/INDEXES_table.sql -sed -i 's/p2\.london/p2.london_region/g' $TEST_DIR/export-dir/schema/tables/INDEXES_table.sql -sed -i 's/p2\.sydney/p2.sydney_region/g' $TEST_DIR/export-dir/schema/tables/INDEXES_table.sql +sed -i 's/p2\.boston/p2.boston_region/g' ${EXPORT_DIR}/schema/tables/INDEXES_table.sql +sed -i 's/p2\.london/p2.london_region/g' ${EXPORT_DIR}/schema/tables/INDEXES_table.sql +sed -i 's/p2\.sydney/p2.sydney_region/g' ${EXPORT_DIR}/schema/tables/INDEXES_table.sql # Added so that the validations work(to check the recommendations in target YB applied or not) -sed -i 's/p2\.boston/p2.boston_region/g' $TEST_DIR/export-dir/assessment/reports/migration_assessment_report.json -sed -i 's/p2\.london/p2.london_region/g' $TEST_DIR/export-dir/assessment/reports/migration_assessment_report.json -sed -i 's/p2\.sydney/p2.sydney_region/g' $TEST_DIR/export-dir/assessment/reports/migration_assessment_report.json \ No newline at end of file +sed -i 's/p2\.boston/p2.boston_region/g' ${EXPORT_DIR}/assessment/reports/migration_assessment_report.json +sed -i 's/p2\.london/p2.london_region/g' ${EXPORT_DIR}/assessment/reports/migration_assessment_report.json +sed -i 's/p2\.sydney/p2.sydney_region/g' ${EXPORT_DIR}/assessment/reports/migration_assessment_report.json \ No newline at end of file From ad054cbce75c75e1e251e1ed0cec7c0b91525bc9 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Sun, 1 Dec 2024 08:42:32 +0000 Subject: [PATCH 02/21] Fixed Sakila and Chinook --- migtests/tests/mysql/chinook/Chinook.sql | 30 ++++++++++++------------ migtests/tests/mysql/chinook/env.sh | 4 +--- migtests/tests/mysql/sakila/env.sh | 1 - migtests/tests/mysql/sakila/init-db | 4 ++++ 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/migtests/tests/mysql/chinook/Chinook.sql b/migtests/tests/mysql/chinook/Chinook.sql index c761f0d041..867f6a5eb6 100644 --- a/migtests/tests/mysql/chinook/Chinook.sql +++ b/migtests/tests/mysql/chinook/Chinook.sql @@ -1,26 +1,26 @@  /******************************************************************************* - Chinook Database - Version 1.4 - Script: Chinook_MySql_AutoIncrementPKs.sql - Description: Creates and populates the Chinook database. + mysql_chinook_offline Database - Version 1.4 + Script: mysql_chinook_offline_MySql_AutoIncrementPKs.sql + Description: Creates and populates the mysql_chinook_offline database. DB Server: MySql Author: Luis Rocha - License: http://www.codeplex.com/ChinookDatabase/license + License: http://www.codeplex.com/mysql_chinook_offlineDatabase/license ********************************************************************************/ /******************************************************************************* Drop database if it exists ********************************************************************************/ -DROP DATABASE IF EXISTS `Chinook`; +DROP DATABASE IF EXISTS `mysql_chinook_offline`; /******************************************************************************* Create database ********************************************************************************/ -CREATE DATABASE `Chinook`; +CREATE DATABASE `mysql_chinook_offline`; -USE `Chinook`; +USE `mysql_chinook_offline`; /******************************************************************************* @@ -4371,14 +4371,14 @@ INSERT INTO `Track` (`Name`, `AlbumId`, `MediaTypeId`, `GenreId`, `Composer`, `M INSERT INTO `Track` (`Name`, `AlbumId`, `MediaTypeId`, `GenreId`, `Composer`, `Milliseconds`, `Bytes`, `UnitPrice`) VALUES (N'Quintet for Horn, Violin, 2 Violas, and Cello in E Flat Major, K. 407/386c: III. Allegro', 346, 2, 24, N'Wolfgang Amadeus Mozart', 221331, 3665114, 0.99); INSERT INTO `Track` (`Name`, `AlbumId`, `MediaTypeId`, `GenreId`, `Composer`, `Milliseconds`, `Bytes`, `UnitPrice`) VALUES (N'Koyaanisqatsi', 347, 2, 10, N'Philip Glass', 206005, 3305164, 0.99); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Adams', N'Andrew', N'General Manager', '1962/2/18', '2002/8/14', N'11120 Jasper Ave NW', N'Edmonton', N'AB', N'Canada', N'T5K 2N1', N'+1 (780) 428-9482', N'+1 (780) 428-3457', N'andrew@chinookcorp.com'); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Edwards', N'Nancy', N'Sales Manager', 1, '1958/12/8', '2002/5/1', N'825 8 Ave SW', N'Calgary', N'AB', N'Canada', N'T2P 2T3', N'+1 (403) 262-3443', N'+1 (403) 262-3322', N'nancy@chinookcorp.com'); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Peacock', N'Jane', N'Sales Support Agent', 2, '1973/8/29', '2002/4/1', N'1111 6 Ave SW', N'Calgary', N'AB', N'Canada', N'T2P 5M5', N'+1 (403) 262-3443', N'+1 (403) 262-6712', N'jane@chinookcorp.com'); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Park', N'Margaret', N'Sales Support Agent', 2, '1947/9/19', '2003/5/3', N'683 10 Street SW', N'Calgary', N'AB', N'Canada', N'T2P 5G3', N'+1 (403) 263-4423', N'+1 (403) 263-4289', N'margaret@chinookcorp.com'); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Johnson', N'Steve', N'Sales Support Agent', 2, '1965/3/3', '2003/10/17', N'7727B 41 Ave', N'Calgary', N'AB', N'Canada', N'T3B 1Y7', N'1 (780) 836-9987', N'1 (780) 836-9543', N'steve@chinookcorp.com'); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Mitchell', N'Michael', N'IT Manager', 1, '1973/7/1', '2003/10/17', N'5827 Bowness Road NW', N'Calgary', N'AB', N'Canada', N'T3B 0C5', N'+1 (403) 246-9887', N'+1 (403) 246-9899', N'michael@chinookcorp.com'); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'King', N'Robert', N'IT Staff', 6, '1970/5/29', '2004/1/2', N'590 Columbia Boulevard West', N'Lethbridge', N'AB', N'Canada', N'T1K 5N8', N'+1 (403) 456-9986', N'+1 (403) 456-8485', N'robert@chinookcorp.com'); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Callahan', N'Laura', N'IT Staff', 6, '1968/1/9', '2004/3/4', N'923 7 ST NW', N'Lethbridge', N'AB', N'Canada', N'T1H 1Y8', N'+1 (403) 467-3351', N'+1 (403) 467-8772', N'laura@chinookcorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Adams', N'Andrew', N'General Manager', '1962/2/18', '2002/8/14', N'11120 Jasper Ave NW', N'Edmonton', N'AB', N'Canada', N'T5K 2N1', N'+1 (780) 428-9482', N'+1 (780) 428-3457', N'andrew@mysql_chinook_offlinecorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Edwards', N'Nancy', N'Sales Manager', 1, '1958/12/8', '2002/5/1', N'825 8 Ave SW', N'Calgary', N'AB', N'Canada', N'T2P 2T3', N'+1 (403) 262-3443', N'+1 (403) 262-3322', N'nancy@mysql_chinook_offlinecorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Peacock', N'Jane', N'Sales Support Agent', 2, '1973/8/29', '2002/4/1', N'1111 6 Ave SW', N'Calgary', N'AB', N'Canada', N'T2P 5M5', N'+1 (403) 262-3443', N'+1 (403) 262-6712', N'jane@mysql_chinook_offlinecorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Park', N'Margaret', N'Sales Support Agent', 2, '1947/9/19', '2003/5/3', N'683 10 Street SW', N'Calgary', N'AB', N'Canada', N'T2P 5G3', N'+1 (403) 263-4423', N'+1 (403) 263-4289', N'margaret@mysql_chinook_offlinecorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Johnson', N'Steve', N'Sales Support Agent', 2, '1965/3/3', '2003/10/17', N'7727B 41 Ave', N'Calgary', N'AB', N'Canada', N'T3B 1Y7', N'1 (780) 836-9987', N'1 (780) 836-9543', N'steve@mysql_chinook_offlinecorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Mitchell', N'Michael', N'IT Manager', 1, '1973/7/1', '2003/10/17', N'5827 Bowness Road NW', N'Calgary', N'AB', N'Canada', N'T3B 0C5', N'+1 (403) 246-9887', N'+1 (403) 246-9899', N'michael@mysql_chinook_offlinecorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'King', N'Robert', N'IT Staff', 6, '1970/5/29', '2004/1/2', N'590 Columbia Boulevard West', N'Lethbridge', N'AB', N'Canada', N'T1K 5N8', N'+1 (403) 456-9986', N'+1 (403) 456-8485', N'robert@mysql_chinook_offlinecorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Callahan', N'Laura', N'IT Staff', 6, '1968/1/9', '2004/3/4', N'923 7 ST NW', N'Lethbridge', N'AB', N'Canada', N'T1H 1Y8', N'+1 (403) 467-3351', N'+1 (403) 467-8772', N'laura@mysql_chinook_offlinecorp.com'); INSERT INTO `Customer` (`FirstName`, `LastName`, `Company`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`, `SupportRepId`) VALUES (N'Luís', N'Gonçalves', N'Embraer - Empresa Brasileira de Aeronáutica S.A.', N'Av. Brigadeiro Faria Lima, 2170', N'São José dos Campos', N'SP', N'Brazil', N'12227-000', N'+55 (12) 3923-5555', N'+55 (12) 3923-5566', N'luisg@embraer.com.br', 3); INSERT INTO `Customer` (`FirstName`, `LastName`, `Address`, `City`, `Country`, `PostalCode`, `Phone`, `Email`, `SupportRepId`) VALUES (N'Leonie', N'Köhler', N'Theodor-Heuss-Straße 34', N'Stuttgart', N'Germany', N'70174', N'+49 0711 2842222', N'leonekohler@surfeu.de', 5); diff --git a/migtests/tests/mysql/chinook/env.sh b/migtests/tests/mysql/chinook/env.sh index ff571a1c0e..62ddf80b40 100644 --- a/migtests/tests/mysql/chinook/env.sh +++ b/migtests/tests/mysql/chinook/env.sh @@ -1,3 +1 @@ -export SOURCE_DB_TYPE="mysql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"Chinook"} -export TARGET_DB_SCHEMA="TEST_SAMPLE_CHINOOK" \ No newline at end of file +export SOURCE_DB_TYPE="mysql" \ No newline at end of file diff --git a/migtests/tests/mysql/sakila/env.sh b/migtests/tests/mysql/sakila/env.sh index 620640ea43..aedc7cffd6 100644 --- a/migtests/tests/mysql/sakila/env.sh +++ b/migtests/tests/mysql/sakila/env.sh @@ -1,2 +1 @@ export SOURCE_DB_TYPE="mysql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"sakila"} diff --git a/migtests/tests/mysql/sakila/init-db b/migtests/tests/mysql/sakila/init-db index f6eb9c5bd6..a5ba9535d9 100755 --- a/migtests/tests/mysql/sakila/init-db +++ b/migtests/tests/mysql/sakila/init-db @@ -16,6 +16,10 @@ echo "Creating ${SOURCE_DB_NAME} database on source" run_mysql mysql "DROP DATABASE IF EXISTS ${SOURCE_DB_NAME};" run_mysql mysql "CREATE DATABASE ${SOURCE_DB_NAME};" +# Replace 'sakila' with the value of SOURCE_DB_NAME in both SQL files +sed -i "s/sakila/${SOURCE_DB_NAME}/g" sakila-db/sakila-schema.sql +sed -i "s/sakila/${SOURCE_DB_NAME}/g" sakila-db/sakila-data.sql + echo "Initialising source database." run_mysql ${SOURCE_DB_NAME} "SOURCE sakila-db/sakila-schema.sql;" run_mysql ${SOURCE_DB_NAME} "SOURCE sakila-db/sakila-data.sql;" From 19187137f0d855403236cb378b9cf8b736008bf0 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Sun, 1 Dec 2024 16:36:12 +0000 Subject: [PATCH 03/21] Test fixes --- migtests/tests/mysql/basic-live-test/env.sh | 3 +-- migtests/tests/mysql/chinook/validate | 9 ++++++--- .../tests/oracle/case-sensitivity-reserved-words/env.sh | 2 -- migtests/tests/oracle/unique-key-conflicts-test/env.sh | 2 -- migtests/tests/pg/unique-key-conflicts-test/env.sh | 2 -- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/migtests/tests/mysql/basic-live-test/env.sh b/migtests/tests/mysql/basic-live-test/env.sh index dfe82e6660..62ddf80b40 100644 --- a/migtests/tests/mysql/basic-live-test/env.sh +++ b/migtests/tests/mysql/basic-live-test/env.sh @@ -1,2 +1 @@ -export SOURCE_DB_TYPE="mysql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"live_test"} \ No newline at end of file +export SOURCE_DB_TYPE="mysql" \ No newline at end of file diff --git a/migtests/tests/mysql/chinook/validate b/migtests/tests/mysql/chinook/validate index c0b9e4826d..8fd855812d 100755 --- a/migtests/tests/mysql/chinook/validate +++ b/migtests/tests/mysql/chinook/validate @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import yb +import os def main(): yb.run_checks(migration_completed_checks) @@ -37,16 +38,18 @@ EXPECTED_INDEX_COUNT = { } def migration_completed_checks(tgt): - table_list = tgt.get_table_names("test_sample_chinook") + target_db_name = os.getenv("TARGET_DB_NAME") + + table_list = tgt.get_table_names(target_db_name) print("table_list:", table_list) assert len(table_list) == 11 - got_row_count = tgt.row_count_of_all_tables("test_sample_chinook") + got_row_count = tgt.row_count_of_all_tables(target_db_name) for table_name, row_count in EXPECTED_ROW_COUNT.items(): print(f"table_name: {table_name}, row_count: {got_row_count[table_name]}") assert row_count == got_row_count[table_name] - get_index_count = tgt.get_count_index_on_table("test_sample_chinook") + get_index_count = tgt.get_count_index_on_table(target_db_name) for table_name, index_count in EXPECTED_INDEX_COUNT.items(): print(f"table_name: {table_name}, index_count: {get_index_count[table_name]}") assert index_count == get_index_count[table_name] diff --git a/migtests/tests/oracle/case-sensitivity-reserved-words/env.sh b/migtests/tests/oracle/case-sensitivity-reserved-words/env.sh index 4f724929c1..429f19fb3a 100644 --- a/migtests/tests/oracle/case-sensitivity-reserved-words/env.sh +++ b/migtests/tests/oracle/case-sensitivity-reserved-words/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"live_test"} export TARGET_DB_SCHEMA="test_schema2" diff --git a/migtests/tests/oracle/unique-key-conflicts-test/env.sh b/migtests/tests/oracle/unique-key-conflicts-test/env.sh index 722796b6a8..998f6081f6 100644 --- a/migtests/tests/oracle/unique-key-conflicts-test/env.sh +++ b/migtests/tests/oracle/unique-key-conflicts-test/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"live_test"} export TARGET_DB_SCHEMA="test_schema" diff --git a/migtests/tests/pg/unique-key-conflicts-test/env.sh b/migtests/tests/pg/unique-key-conflicts-test/env.sh index f812fc984c..4615850a1d 100644 --- a/migtests/tests/pg/unique-key-conflicts-test/env.sh +++ b/migtests/tests/pg/unique-key-conflicts-test/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="postgresql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"unique_key_conflict_cases"} export SOURCE_DB_SCHEMA="public,non_public" -export SOURCE_REPLICA_DB_NAME="unique_key_conflict_cases_replica" \ No newline at end of file From 335247aa471416c5d3c7c58ef1dff031bb9a6473 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Sun, 1 Dec 2024 17:48:33 +0000 Subject: [PATCH 04/21] Test fixes --- .github/workflows/mysql-migtests.yml | 1 + migtests/scripts/functions.sh | 11 +++++++ migtests/scripts/run-test.sh | 16 ++++++---- migtests/tests/mysql/chinook/env.sh | 3 +- migtests/tests/mysql/chinook/validate | 29 +++++++++---------- migtests/tests/mysql/sakila/init-db | 2 +- migtests/tests/oracle/co-db/env.sh | 1 + .../oracle/sequences/validateAfterChanges | 5 ++-- 8 files changed, 42 insertions(+), 26 deletions(-) diff --git a/.github/workflows/mysql-migtests.yml b/.github/workflows/mysql-migtests.yml index 7872bea0f5..c8d4ee0550 100644 --- a/.github/workflows/mysql-migtests.yml +++ b/.github/workflows/mysql-migtests.yml @@ -134,6 +134,7 @@ jobs: # Placeholder for now so that a basic test can run - name: Create the live migration user + if: always() run: | mysql -uroot -proot -e 'GRANT SELECT, RELOAD, SHOW DATABASES, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'ybvoyager'@'127.0.0.1';' diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 4abd4f5e75..8883eea47c 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -668,6 +668,17 @@ setup_fallback_environment() { if [ "${SOURCE_DB_TYPE}" = "oracle" ]; then run_sqlplus_as_sys ${SOURCE_DB_NAME} ${SCRIPTS}/oracle/create_metadata_tables.sql run_sqlplus_as_sys ${SOURCE_DB_NAME} ${SCRIPTS}/oracle/fall_back_prep.sql + # Define the temporary file + TEMP_SCRIPT="/tmp/fall_back_prep.sql" + + # Replace TEST_SCHEMA with the value of SOURCE_DB_SCHEMA in the script + sed "s/TEST_SCHEMA/${SOURCE_DB_SCHEMA}/g" ${SCRIPTS}/oracle/fall_back_prep.sql > $TEMP_SCRIPT + + # Run the updated script using the run_sqlplus_as_sys function + run_sqlplus_as_sys ${SOURCE_DB_NAME} $TEMP_SCRIPT + + # Clean up the temporary file after execution + rm -f $TEMP_SCRIPT elif [ "${SOURCE_DB_TYPE}" = "postgresql" ]; then conn_string="postgresql://${SOURCE_DB_ADMIN_USER}:${SOURCE_DB_ADMIN_PASSWORD}@${SOURCE_DB_HOST}:${SOURCE_DB_PORT}/${SOURCE_DB_NAME}" psql "${conn_string}" -v voyager_user="${SOURCE_DB_USER}" -v schema_list="${SOURCE_DB_SCHEMA}" -v replication_group='replication_group' -v original_owner_of_tables="${SOURCE_DB_ADMIN_USER}" -v is_live_migration=1 -v is_live_migration_fall_back=1 -f /opt/yb-voyager/guardrails-scripts/yb-voyager-pg-grant-migration-permissions.sql diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index 5f3d75e26c..f1190db819 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -64,13 +64,17 @@ main() { pushd ${TEST_DIR} step "Initialise source database." - if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then - create_source_db ${SOURCE_DB_NAME} -elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then - create_source_db ${SOURCE_DB_SCHEMA} + if [[ "${SKIP_DB_CREATION}" != "true" ]]; then + if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then + create_source_db "${SOURCE_DB_NAME}" + elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then + create_source_db "${SOURCE_DB_SCHEMA}" + else + echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" + exit 1 + fi else - echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" - exit 1 + echo "Skipping database creation as SKIP_DB_CREATION is set to true." fi ./init-db diff --git a/migtests/tests/mysql/chinook/env.sh b/migtests/tests/mysql/chinook/env.sh index 62ddf80b40..cf6ed2aabf 100644 --- a/migtests/tests/mysql/chinook/env.sh +++ b/migtests/tests/mysql/chinook/env.sh @@ -1 +1,2 @@ -export SOURCE_DB_TYPE="mysql" \ No newline at end of file +export SOURCE_DB_TYPE="mysql" +export TARGET_DB_SCHEMA="TEST_SAMPLE_CHINOOK" diff --git a/migtests/tests/mysql/chinook/validate b/migtests/tests/mysql/chinook/validate index 8fd855812d..d06060f8eb 100755 --- a/migtests/tests/mysql/chinook/validate +++ b/migtests/tests/mysql/chinook/validate @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import yb -import os def main(): yb.run_checks(migration_completed_checks) @@ -38,21 +37,19 @@ EXPECTED_INDEX_COUNT = { } def migration_completed_checks(tgt): - target_db_name = os.getenv("TARGET_DB_NAME") - - table_list = tgt.get_table_names(target_db_name) - print("table_list:", table_list) - assert len(table_list) == 11 - - got_row_count = tgt.row_count_of_all_tables(target_db_name) - for table_name, row_count in EXPECTED_ROW_COUNT.items(): - print(f"table_name: {table_name}, row_count: {got_row_count[table_name]}") - assert row_count == got_row_count[table_name] - - get_index_count = tgt.get_count_index_on_table(target_db_name) - for table_name, index_count in EXPECTED_INDEX_COUNT.items(): - print(f"table_name: {table_name}, index_count: {get_index_count[table_name]}") - assert index_count == get_index_count[table_name] + table_list = tgt.get_table_names("test_sample_chinook") + print("table_list:", table_list) + assert len(table_list) == 11 + + got_row_count = tgt.row_count_of_all_tables("test_sample_chinook") + for table_name, row_count in EXPECTED_ROW_COUNT.items(): + print(f"table_name: {table_name}, row_count: {got_row_count[table_name]}") + assert row_count == got_row_count[table_name] + + get_index_count = tgt.get_count_index_on_table("test_sample_chinook") + for table_name, index_count in EXPECTED_INDEX_COUNT.items(): + print(f"table_name: {table_name}, index_count: {get_index_count[table_name]}") + assert index_count == get_index_count[table_name] if __name__ == "__main__": diff --git a/migtests/tests/mysql/sakila/init-db b/migtests/tests/mysql/sakila/init-db index a5ba9535d9..190654c2e0 100755 --- a/migtests/tests/mysql/sakila/init-db +++ b/migtests/tests/mysql/sakila/init-db @@ -18,7 +18,7 @@ run_mysql mysql "CREATE DATABASE ${SOURCE_DB_NAME};" # Replace 'sakila' with the value of SOURCE_DB_NAME in both SQL files sed -i "s/sakila/${SOURCE_DB_NAME}/g" sakila-db/sakila-schema.sql -sed -i "s/sakila/${SOURCE_DB_NAME}/g" sakila-db/sakila-data.sql +sed -i "s/USE sakila/USE ${SOURCE_DB_NAME}/g" sakila-db/sakila-data.sql echo "Initialising source database." run_mysql ${SOURCE_DB_NAME} "SOURCE sakila-db/sakila-schema.sql;" diff --git a/migtests/tests/oracle/co-db/env.sh b/migtests/tests/oracle/co-db/env.sh index f0f431a669..15d6f15d90 100644 --- a/migtests/tests/oracle/co-db/env.sh +++ b/migtests/tests/oracle/co-db/env.sh @@ -1,4 +1,5 @@ export SOURCE_DB_TYPE="oracle" export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"CO"} export TARGET_DB_NAME=${TARGET_DB_NAME:-"co_db_test"} +export SKIP_DB_CREATION="true" diff --git a/migtests/tests/oracle/sequences/validateAfterChanges b/migtests/tests/oracle/sequences/validateAfterChanges index cc6c2705e3..a549b591c0 100755 --- a/migtests/tests/oracle/sequences/validateAfterChanges +++ b/migtests/tests/oracle/sequences/validateAfterChanges @@ -166,14 +166,15 @@ def migration_completed_checks_yb(): def migration_completed_checks_ff(): print("Running tests on Oracle source replica") global db_schema - db_schema = os.environ.get("SOURCE_REPLICA_DB_SCHEMA") + db_schema = os.environ.get("SOURCE_REPLICA_DB_SCHEMA").upper() change_expected_values_ff_fb() oracle.run_checks(migration_completed_checks, db_type="source_replica") def migration_completed_checks_fb(): print("Running tests on Oracle source") global db_schema - db_schema = os.environ.get("SOURCE_DB_SCHEMA") + db_schema = os.environ.get("SOURCE_DB_SCHEMA").upper() + print(db_schema) change_expected_values_ff_fb() oracle.run_checks(migration_completed_checks, db_type="source") From 1ea24a73d677a694d0d3ee6c1471cdc868fe54e2 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Sun, 1 Dec 2024 18:28:19 +0000 Subject: [PATCH 05/21] Added the usage of SKIP_DB_CREATION --- .../scripts/live-migration-fallb-run-test.sh | 4 +-- .../scripts/live-migration-fallf-run-test.sh | 4 +-- migtests/scripts/live-migration-run-test.sh | 4 +-- migtests/scripts/run-test.sh | 26 ++++++++-------- migtests/tests/mysql/chinook/Chinook.sql | 31 +++++++++---------- migtests/tests/mysql/chinook/env.sh | 3 ++ migtests/tests/mysql/chinook/validate | 26 ++++++++-------- migtests/tests/mysql/sakila/env.sh | 2 ++ migtests/tests/mysql/sakila/init-db | 6 +--- 9 files changed, 53 insertions(+), 53 deletions(-) diff --git a/migtests/scripts/live-migration-fallb-run-test.sh b/migtests/scripts/live-migration-fallb-run-test.sh index 2bc49b7bc7..27ee347d2a 100755 --- a/migtests/scripts/live-migration-fallb-run-test.sh +++ b/migtests/scripts/live-migration-fallb-run-test.sh @@ -39,9 +39,9 @@ else fi if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then - export SOURCE_DB_NAME="${NORMALIZED_TEST_NAME}_fallb" + export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_fallb"} elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then - export SOURCE_DB_SCHEMA="${NORMALIZED_TEST_NAME}_fallb" + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME}_fallb"} else echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" exit 1 diff --git a/migtests/scripts/live-migration-fallf-run-test.sh b/migtests/scripts/live-migration-fallf-run-test.sh index 14356e4536..3ea0041274 100755 --- a/migtests/scripts/live-migration-fallf-run-test.sh +++ b/migtests/scripts/live-migration-fallf-run-test.sh @@ -40,9 +40,9 @@ else fi if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then - export SOURCE_DB_NAME="${NORMALIZED_TEST_NAME}_fallf" + export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_fallf"} elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then - export SOURCE_DB_SCHEMA="${NORMALIZED_TEST_NAME}_fallf" + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME}_fallf"} else echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" exit 1 diff --git a/migtests/scripts/live-migration-run-test.sh b/migtests/scripts/live-migration-run-test.sh index 2207b28c57..bac82c0917 100755 --- a/migtests/scripts/live-migration-run-test.sh +++ b/migtests/scripts/live-migration-run-test.sh @@ -39,9 +39,9 @@ else fi if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then - export SOURCE_DB_NAME="${NORMALIZED_TEST_NAME}_live" + export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_live"} elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then - export SOURCE_DB_SCHEMA="${NORMALIZED_TEST_NAME}_live" + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME}_live"} else echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" exit 1 diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index f1190db819..4f66543ebc 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -39,9 +39,9 @@ source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then - export SOURCE_DB_NAME="${NORMALIZED_TEST_NAME}_offline" + export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_offline"} elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then - export SOURCE_DB_SCHEMA="${NORMALIZED_TEST_NAME}_offline" + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME}_offline"} else echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" exit 1 @@ -65,17 +65,17 @@ main() { step "Initialise source database." if [[ "${SKIP_DB_CREATION}" != "true" ]]; then - if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then - create_source_db "${SOURCE_DB_NAME}" - elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then - create_source_db "${SOURCE_DB_SCHEMA}" - else - echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" - exit 1 - fi -else - echo "Skipping database creation as SKIP_DB_CREATION is set to true." -fi + if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then + create_source_db "${SOURCE_DB_NAME}" + elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then + create_source_db "${SOURCE_DB_SCHEMA}" + else + echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" + exit 1 + fi + else + echo "Skipping database creation as SKIP_DB_CREATION is set to true." + fi ./init-db step "Grant source database user permissions" diff --git a/migtests/tests/mysql/chinook/Chinook.sql b/migtests/tests/mysql/chinook/Chinook.sql index 867f6a5eb6..f3309ff09b 100644 --- a/migtests/tests/mysql/chinook/Chinook.sql +++ b/migtests/tests/mysql/chinook/Chinook.sql @@ -1,26 +1,26 @@  /******************************************************************************* - mysql_chinook_offline Database - Version 1.4 - Script: mysql_chinook_offline_MySql_AutoIncrementPKs.sql - Description: Creates and populates the mysql_chinook_offline database. + Chinook Database - Version 1.4 + Script: Chinook_MySql_AutoIncrementPKs.sql + Description: Creates and populates the Chinook database. DB Server: MySql Author: Luis Rocha - License: http://www.codeplex.com/mysql_chinook_offlineDatabase/license + License: http://www.codeplex.com/ChinookDatabase/license ********************************************************************************/ /******************************************************************************* Drop database if it exists ********************************************************************************/ -DROP DATABASE IF EXISTS `mysql_chinook_offline`; +DROP DATABASE IF EXISTS `Chinook`; /******************************************************************************* Create database ********************************************************************************/ -CREATE DATABASE `mysql_chinook_offline`; +CREATE DATABASE `Chinook`; -USE `mysql_chinook_offline`; +USE `Chinook`; /******************************************************************************* @@ -4371,14 +4371,14 @@ INSERT INTO `Track` (`Name`, `AlbumId`, `MediaTypeId`, `GenreId`, `Composer`, `M INSERT INTO `Track` (`Name`, `AlbumId`, `MediaTypeId`, `GenreId`, `Composer`, `Milliseconds`, `Bytes`, `UnitPrice`) VALUES (N'Quintet for Horn, Violin, 2 Violas, and Cello in E Flat Major, K. 407/386c: III. Allegro', 346, 2, 24, N'Wolfgang Amadeus Mozart', 221331, 3665114, 0.99); INSERT INTO `Track` (`Name`, `AlbumId`, `MediaTypeId`, `GenreId`, `Composer`, `Milliseconds`, `Bytes`, `UnitPrice`) VALUES (N'Koyaanisqatsi', 347, 2, 10, N'Philip Glass', 206005, 3305164, 0.99); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Adams', N'Andrew', N'General Manager', '1962/2/18', '2002/8/14', N'11120 Jasper Ave NW', N'Edmonton', N'AB', N'Canada', N'T5K 2N1', N'+1 (780) 428-9482', N'+1 (780) 428-3457', N'andrew@mysql_chinook_offlinecorp.com'); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Edwards', N'Nancy', N'Sales Manager', 1, '1958/12/8', '2002/5/1', N'825 8 Ave SW', N'Calgary', N'AB', N'Canada', N'T2P 2T3', N'+1 (403) 262-3443', N'+1 (403) 262-3322', N'nancy@mysql_chinook_offlinecorp.com'); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Peacock', N'Jane', N'Sales Support Agent', 2, '1973/8/29', '2002/4/1', N'1111 6 Ave SW', N'Calgary', N'AB', N'Canada', N'T2P 5M5', N'+1 (403) 262-3443', N'+1 (403) 262-6712', N'jane@mysql_chinook_offlinecorp.com'); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Park', N'Margaret', N'Sales Support Agent', 2, '1947/9/19', '2003/5/3', N'683 10 Street SW', N'Calgary', N'AB', N'Canada', N'T2P 5G3', N'+1 (403) 263-4423', N'+1 (403) 263-4289', N'margaret@mysql_chinook_offlinecorp.com'); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Johnson', N'Steve', N'Sales Support Agent', 2, '1965/3/3', '2003/10/17', N'7727B 41 Ave', N'Calgary', N'AB', N'Canada', N'T3B 1Y7', N'1 (780) 836-9987', N'1 (780) 836-9543', N'steve@mysql_chinook_offlinecorp.com'); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Mitchell', N'Michael', N'IT Manager', 1, '1973/7/1', '2003/10/17', N'5827 Bowness Road NW', N'Calgary', N'AB', N'Canada', N'T3B 0C5', N'+1 (403) 246-9887', N'+1 (403) 246-9899', N'michael@mysql_chinook_offlinecorp.com'); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'King', N'Robert', N'IT Staff', 6, '1970/5/29', '2004/1/2', N'590 Columbia Boulevard West', N'Lethbridge', N'AB', N'Canada', N'T1K 5N8', N'+1 (403) 456-9986', N'+1 (403) 456-8485', N'robert@mysql_chinook_offlinecorp.com'); -INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Callahan', N'Laura', N'IT Staff', 6, '1968/1/9', '2004/3/4', N'923 7 ST NW', N'Lethbridge', N'AB', N'Canada', N'T1H 1Y8', N'+1 (403) 467-3351', N'+1 (403) 467-8772', N'laura@mysql_chinook_offlinecorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Adams', N'Andrew', N'General Manager', '1962/2/18', '2002/8/14', N'11120 Jasper Ave NW', N'Edmonton', N'AB', N'Canada', N'T5K 2N1', N'+1 (780) 428-9482', N'+1 (780) 428-3457', N'andrew@chinookcorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Edwards', N'Nancy', N'Sales Manager', 1, '1958/12/8', '2002/5/1', N'825 8 Ave SW', N'Calgary', N'AB', N'Canada', N'T2P 2T3', N'+1 (403) 262-3443', N'+1 (403) 262-3322', N'nancy@chinookcorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Peacock', N'Jane', N'Sales Support Agent', 2, '1973/8/29', '2002/4/1', N'1111 6 Ave SW', N'Calgary', N'AB', N'Canada', N'T2P 5M5', N'+1 (403) 262-3443', N'+1 (403) 262-6712', N'jane@chinookcorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Park', N'Margaret', N'Sales Support Agent', 2, '1947/9/19', '2003/5/3', N'683 10 Street SW', N'Calgary', N'AB', N'Canada', N'T2P 5G3', N'+1 (403) 263-4423', N'+1 (403) 263-4289', N'margaret@chinookcorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Johnson', N'Steve', N'Sales Support Agent', 2, '1965/3/3', '2003/10/17', N'7727B 41 Ave', N'Calgary', N'AB', N'Canada', N'T3B 1Y7', N'1 (780) 836-9987', N'1 (780) 836-9543', N'steve@chinookcorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Mitchell', N'Michael', N'IT Manager', 1, '1973/7/1', '2003/10/17', N'5827 Bowness Road NW', N'Calgary', N'AB', N'Canada', N'T3B 0C5', N'+1 (403) 246-9887', N'+1 (403) 246-9899', N'michael@chinookcorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'King', N'Robert', N'IT Staff', 6, '1970/5/29', '2004/1/2', N'590 Columbia Boulevard West', N'Lethbridge', N'AB', N'Canada', N'T1K 5N8', N'+1 (403) 456-9986', N'+1 (403) 456-8485', N'robert@chinookcorp.com'); +INSERT INTO `Employee` (`LastName`, `FirstName`, `Title`, `ReportsTo`, `BirthDate`, `HireDate`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`) VALUES (N'Callahan', N'Laura', N'IT Staff', 6, '1968/1/9', '2004/3/4', N'923 7 ST NW', N'Lethbridge', N'AB', N'Canada', N'T1H 1Y8', N'+1 (403) 467-3351', N'+1 (403) 467-8772', N'laura@chinookcorp.com'); INSERT INTO `Customer` (`FirstName`, `LastName`, `Company`, `Address`, `City`, `State`, `Country`, `PostalCode`, `Phone`, `Fax`, `Email`, `SupportRepId`) VALUES (N'Luís', N'Gonçalves', N'Embraer - Empresa Brasileira de Aeronáutica S.A.', N'Av. Brigadeiro Faria Lima, 2170', N'São José dos Campos', N'SP', N'Brazil', N'12227-000', N'+55 (12) 3923-5555', N'+55 (12) 3923-5566', N'luisg@embraer.com.br', 3); INSERT INTO `Customer` (`FirstName`, `LastName`, `Address`, `City`, `Country`, `PostalCode`, `Phone`, `Email`, `SupportRepId`) VALUES (N'Leonie', N'Köhler', N'Theodor-Heuss-Straße 34', N'Stuttgart', N'Germany', N'70174', N'+49 0711 2842222', N'leonekohler@surfeu.de', 5); @@ -15829,4 +15829,3 @@ INSERT INTO `PlaylistTrack` (`PlaylistId`, `TrackId`) VALUES (17, 2096); INSERT INTO `PlaylistTrack` (`PlaylistId`, `TrackId`) VALUES (17, 3290); INSERT INTO `PlaylistTrack` (`PlaylistId`, `TrackId`) VALUES (18, 597); - diff --git a/migtests/tests/mysql/chinook/env.sh b/migtests/tests/mysql/chinook/env.sh index cf6ed2aabf..c640e171cd 100644 --- a/migtests/tests/mysql/chinook/env.sh +++ b/migtests/tests/mysql/chinook/env.sh @@ -1,2 +1,5 @@ export SOURCE_DB_TYPE="mysql" +export SOURCE_DB_NAME="Chinook" export TARGET_DB_SCHEMA="TEST_SAMPLE_CHINOOK" +export SKIP_DB_CREATION="true" + diff --git a/migtests/tests/mysql/chinook/validate b/migtests/tests/mysql/chinook/validate index d06060f8eb..c0b9e4826d 100755 --- a/migtests/tests/mysql/chinook/validate +++ b/migtests/tests/mysql/chinook/validate @@ -37,19 +37,19 @@ EXPECTED_INDEX_COUNT = { } def migration_completed_checks(tgt): - table_list = tgt.get_table_names("test_sample_chinook") - print("table_list:", table_list) - assert len(table_list) == 11 - - got_row_count = tgt.row_count_of_all_tables("test_sample_chinook") - for table_name, row_count in EXPECTED_ROW_COUNT.items(): - print(f"table_name: {table_name}, row_count: {got_row_count[table_name]}") - assert row_count == got_row_count[table_name] - - get_index_count = tgt.get_count_index_on_table("test_sample_chinook") - for table_name, index_count in EXPECTED_INDEX_COUNT.items(): - print(f"table_name: {table_name}, index_count: {get_index_count[table_name]}") - assert index_count == get_index_count[table_name] + table_list = tgt.get_table_names("test_sample_chinook") + print("table_list:", table_list) + assert len(table_list) == 11 + + got_row_count = tgt.row_count_of_all_tables("test_sample_chinook") + for table_name, row_count in EXPECTED_ROW_COUNT.items(): + print(f"table_name: {table_name}, row_count: {got_row_count[table_name]}") + assert row_count == got_row_count[table_name] + + get_index_count = tgt.get_count_index_on_table("test_sample_chinook") + for table_name, index_count in EXPECTED_INDEX_COUNT.items(): + print(f"table_name: {table_name}, index_count: {get_index_count[table_name]}") + assert index_count == get_index_count[table_name] if __name__ == "__main__": diff --git a/migtests/tests/mysql/sakila/env.sh b/migtests/tests/mysql/sakila/env.sh index aedc7cffd6..5fa4dc4acb 100644 --- a/migtests/tests/mysql/sakila/env.sh +++ b/migtests/tests/mysql/sakila/env.sh @@ -1 +1,3 @@ export SOURCE_DB_TYPE="mysql" +export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"sakila"} +export SKIP_DB_CREATION="true" \ No newline at end of file diff --git a/migtests/tests/mysql/sakila/init-db b/migtests/tests/mysql/sakila/init-db index 190654c2e0..b783d909f1 100755 --- a/migtests/tests/mysql/sakila/init-db +++ b/migtests/tests/mysql/sakila/init-db @@ -16,10 +16,6 @@ echo "Creating ${SOURCE_DB_NAME} database on source" run_mysql mysql "DROP DATABASE IF EXISTS ${SOURCE_DB_NAME};" run_mysql mysql "CREATE DATABASE ${SOURCE_DB_NAME};" -# Replace 'sakila' with the value of SOURCE_DB_NAME in both SQL files -sed -i "s/sakila/${SOURCE_DB_NAME}/g" sakila-db/sakila-schema.sql -sed -i "s/USE sakila/USE ${SOURCE_DB_NAME}/g" sakila-db/sakila-data.sql - echo "Initialising source database." run_mysql ${SOURCE_DB_NAME} "SOURCE sakila-db/sakila-schema.sql;" run_mysql ${SOURCE_DB_NAME} "SOURCE sakila-db/sakila-data.sql;" @@ -27,4 +23,4 @@ run_mysql ${SOURCE_DB_NAME} "SOURCE sakila-db/sakila-data.sql;" run_mysql ${SOURCE_DB_NAME} "ALTER TABLE address DROP COLUMN location;" echo "Check source database." -run_mysql ${SOURCE_DB_NAME} "SELECT count(*) FROM payment;" +run_mysql ${SOURCE_DB_NAME} "SELECT count(*) FROM payment;" \ No newline at end of file From 23942d01e3694756644cdda7bc5ad92d354b35db Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Mon, 2 Dec 2024 12:01:09 +0000 Subject: [PATCH 06/21] Added the limit to the schema names for Oracle --- migtests/scripts/functions.sh | 8 ++++---- migtests/scripts/live-migration-fallb-run-test.sh | 2 +- migtests/scripts/live-migration-fallf-run-test.sh | 2 +- migtests/scripts/live-migration-run-test.sh | 4 ++-- migtests/scripts/run-test.sh | 2 +- migtests/tests/oracle/basic-live-test/env.sh | 2 -- 6 files changed, 9 insertions(+), 11 deletions(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 8883eea47c..fd894ca10e 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -667,16 +667,16 @@ grant_permissions_for_live_migration() { setup_fallback_environment() { if [ "${SOURCE_DB_TYPE}" = "oracle" ]; then run_sqlplus_as_sys ${SOURCE_DB_NAME} ${SCRIPTS}/oracle/create_metadata_tables.sql - run_sqlplus_as_sys ${SOURCE_DB_NAME} ${SCRIPTS}/oracle/fall_back_prep.sql + # run_sqlplus_as_sys ${SOURCE_DB_NAME} ${SCRIPTS}/oracle/fall_back_prep.sql # Define the temporary file TEMP_SCRIPT="/tmp/fall_back_prep.sql" - + # Replace TEST_SCHEMA with the value of SOURCE_DB_SCHEMA in the script sed "s/TEST_SCHEMA/${SOURCE_DB_SCHEMA}/g" ${SCRIPTS}/oracle/fall_back_prep.sql > $TEMP_SCRIPT - + # Run the updated script using the run_sqlplus_as_sys function run_sqlplus_as_sys ${SOURCE_DB_NAME} $TEMP_SCRIPT - + # Clean up the temporary file after execution rm -f $TEMP_SCRIPT elif [ "${SOURCE_DB_TYPE}" = "postgresql" ]; then diff --git a/migtests/scripts/live-migration-fallb-run-test.sh b/migtests/scripts/live-migration-fallb-run-test.sh index 27ee347d2a..cc5a08625c 100755 --- a/migtests/scripts/live-migration-fallb-run-test.sh +++ b/migtests/scripts/live-migration-fallb-run-test.sh @@ -41,7 +41,7 @@ fi if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_fallb"} elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then - export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME}_fallb"} + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_fallb"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} else echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" exit 1 diff --git a/migtests/scripts/live-migration-fallf-run-test.sh b/migtests/scripts/live-migration-fallf-run-test.sh index 3ea0041274..ce43355f8d 100755 --- a/migtests/scripts/live-migration-fallf-run-test.sh +++ b/migtests/scripts/live-migration-fallf-run-test.sh @@ -42,7 +42,7 @@ fi if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_fallf"} elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then - export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME}_fallf"} + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_fallf"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} else echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" exit 1 diff --git a/migtests/scripts/live-migration-run-test.sh b/migtests/scripts/live-migration-run-test.sh index bac82c0917..509d16710e 100755 --- a/migtests/scripts/live-migration-run-test.sh +++ b/migtests/scripts/live-migration-run-test.sh @@ -41,7 +41,7 @@ fi if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_live"} elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then - export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME}_live"} + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_live"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} else echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" exit 1 @@ -142,7 +142,7 @@ main() { step "Export data." # false if exit code of export_data is non-zero - export_data --export-type "snapshot-and-changes" || { + export_data --export-type "snapshot-and-changes" --log-level debug || { tail_log_file "yb-voyager-export-data.log" tail_log_file "debezium-source_db_exporter.log" exit 1 diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index 4f66543ebc..12c7a6661c 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -41,7 +41,7 @@ source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_offline"} elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then - export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME}_offline"} + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_offline"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} else echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" exit 1 diff --git a/migtests/tests/oracle/basic-live-test/env.sh b/migtests/tests/oracle/basic-live-test/env.sh index 722796b6a8..998f6081f6 100644 --- a/migtests/tests/oracle/basic-live-test/env.sh +++ b/migtests/tests/oracle/basic-live-test/env.sh @@ -1,4 +1,2 @@ export SOURCE_DB_TYPE="oracle" -export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"TEST_SCHEMA"} -export TARGET_DB_NAME=${TARGET_DB_NAME:-"live_test"} export TARGET_DB_SCHEMA="test_schema" From 7835cd3d39a83e4a2b5fcc56bdc6edc42936a72a Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Tue, 3 Dec 2024 14:10:21 +0000 Subject: [PATCH 07/21] Added extra logs for debugging --- migtests/scripts/live-migration-fallb-run-test.sh | 6 +++++- migtests/scripts/live-migration-fallf-run-test.sh | 7 ++++++- migtests/tests/oracle/sequences/validate | 8 ++++---- .../tests/oracle/sequences/validateAfterChanges | 14 +++++++------- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/migtests/scripts/live-migration-fallb-run-test.sh b/migtests/scripts/live-migration-fallb-run-test.sh index cc5a08625c..c63b3f73df 100755 --- a/migtests/scripts/live-migration-fallb-run-test.sh +++ b/migtests/scripts/live-migration-fallb-run-test.sh @@ -183,7 +183,11 @@ main() { import_schema --post-snapshot-import true --refresh-mviews=true step "Run snapshot validations." - "${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'false' --fb_enabled 'true' + "${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'false' --fb_enabled 'true' || { + tail_log_file "yb-voyager-import-data-to-target.log" + tail_log_file "yb-voyager-export-data-from-source.log" + exit 1 + } step "Inserting new events" run_sql_file source_delta.sql diff --git a/migtests/scripts/live-migration-fallf-run-test.sh b/migtests/scripts/live-migration-fallf-run-test.sh index ce43355f8d..ddf9d23dfc 100755 --- a/migtests/scripts/live-migration-fallf-run-test.sh +++ b/migtests/scripts/live-migration-fallf-run-test.sh @@ -217,7 +217,12 @@ main() { import_schema --post-snapshot-import true --refresh-mviews true step "Run snapshot validations." - "${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'true' --fb_enabled 'false' + "${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'true' --fb_enabled 'false' || { + tail_log_file "yb-voyager-import-data-to-target.log" + tail_log_file "yb-voyager-export-data-from-source.log" + tail_log_file "yb-voyager-import-data-to-source-replica.log" + exit 1 + } step "Inserting new events to source" run_sql_file source_delta.sql diff --git a/migtests/tests/oracle/sequences/validate b/migtests/tests/oracle/sequences/validate index 3abd78a3b4..e7918b8567 100755 --- a/migtests/tests/oracle/sequences/validate +++ b/migtests/tests/oracle/sequences/validate @@ -23,7 +23,7 @@ EXPECTED_ROW_COUNT = { 'identity_demo_generated_always_start_with': 1, 'identity_demo_generated_by_def_st_with_inc_by': 3, 'identity_demo_generated_by_def_start_with': 3, - 'identity_demo_with_null': 2, + # 'identity_demo_with_null': 2, 'case_sensitive_always': 2 } @@ -34,7 +34,7 @@ EXPECTED_TABLE_SUM = { 'identity_demo_generated_always_start_with': 101, 'identity_demo_generated_by_def_st_with_inc_by': 115, 'identity_demo_generated_by_def_start_with': 205, - 'identity_demo_with_null': 3, + # 'identity_demo_with_null': 3, 'case_sensitive_always': 3 } @@ -45,7 +45,7 @@ EXPECTED_TABLE_SUM_AFTER_INSERT = { 'identity_demo_generated_always_start_with': 223, 'identity_demo_generated_by_def_st_with_inc_by': 2241, 'identity_demo_generated_by_def_start_with': 327, - 'identity_demo_with_null': 25, + # 'identity_demo_with_null': 25, 'case_sensitive_always': 25 } @@ -57,7 +57,7 @@ if os.environ.get('BETA_FAST_DATA_EXPORT') == '1': 'identity_demo_generated_always_start_with': 203, 'identity_demo_generated_by_def_st_with_inc_by': 322, 'identity_demo_generated_by_def_start_with': 308, - 'identity_demo_with_null': 6, + # 'identity_demo_with_null': 6, 'case_sensitive_always': 6 } diff --git a/migtests/tests/oracle/sequences/validateAfterChanges b/migtests/tests/oracle/sequences/validateAfterChanges index a549b591c0..f44d9a1ab6 100755 --- a/migtests/tests/oracle/sequences/validateAfterChanges +++ b/migtests/tests/oracle/sequences/validateAfterChanges @@ -29,7 +29,7 @@ EXPECTED_ROW_COUNT = { 'identity_demo_generated_always_start_with': 1, 'identity_demo_generated_by_def_st_with_inc_by': 3, 'identity_demo_generated_by_def_start_with': 3, - 'identity_demo_with_null': 3, + # 'identity_demo_with_null': 3, 'empty_identity_def': 1, 'empty_identity_always': 3, 'case_sensitive_always': 3 @@ -42,7 +42,7 @@ EXPECTED_TABLE_SUM = { 'identity_demo_generated_always_start_with': 101, 'identity_demo_generated_by_def_st_with_inc_by': 115, 'identity_demo_generated_by_def_start_with': 205, - 'identity_demo_with_null': 9, + # 'identity_demo_with_null': 9, 'empty_identity_def': 2, 'empty_identity_always': 9, 'case_sensitive_always': 9 @@ -55,7 +55,7 @@ EXPECTED_TABLE_SUM_AFTER_INSERT = { 'identity_demo_generated_always_start_with': 203, 'identity_demo_generated_by_def_st_with_inc_by': 322, 'identity_demo_generated_by_def_start_with': 308, - 'identity_demo_with_null': 14, + # 'identity_demo_with_null': 14, 'empty_identity_def': 5, 'empty_identity_always': 14, 'case_sensitive_always': 14 @@ -69,7 +69,7 @@ EXPECTED_ROW_COUNT_FF = { 'identity_demo_generated_always_start_with': 1, 'identity_demo_generated_by_def_st_with_inc_by': 3, 'identity_demo_generated_by_def_start_with': 3, - 'identity_demo_with_null': 4, + # 'identity_demo_with_null': 4, 'empty_identity_def': 1, 'empty_identity_always': 4, 'case_sensitive_always': 5 @@ -82,7 +82,7 @@ EXPECTED_TABLE_SUM_FF = { 'identity_demo_generated_always_start_with': 101, 'identity_demo_generated_by_def_st_with_inc_by': 115, 'identity_demo_generated_by_def_start_with': 205, - 'identity_demo_with_null': 17, + # 'identity_demo_with_null': 17, 'empty_identity_def': 3, 'empty_identity_always': 18, 'case_sensitive_always': 25 @@ -95,7 +95,7 @@ EXPECTED_TABLE_SUM_AFTER_INSERT_FF = { 'identity_demo_generated_always_start_with': 203, 'identity_demo_generated_by_def_st_with_inc_by': 322, 'identity_demo_generated_by_def_start_with': 308, - 'identity_demo_with_null': 122, + # 'identity_demo_with_null': 122, 'empty_identity_def': 106, 'empty_identity_always': 123, 'case_sensitive_always': 130 @@ -108,7 +108,7 @@ EXPECTED_TABLE_SUM_AFTER_INSERT_ORA = { 'identity_demo_generated_always_start_with': 203, 'identity_demo_generated_by_def_st_with_inc_by': 222, 'identity_demo_generated_by_def_start_with': 308, - 'identity_demo_with_null': 24, + # 'identity_demo_with_null': 24, 'empty_identity_def': 7, 'empty_identity_always': 25, 'case_sensitive_always': 33 From 41e0b18f657e147fd11ebcbe2a51bc6545313057 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Tue, 3 Dec 2024 20:03:37 +0000 Subject: [PATCH 08/21] Fix log name --- migtests/scripts/live-migration-fallb-run-test.sh | 2 +- migtests/scripts/live-migration-fallf-run-test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/migtests/scripts/live-migration-fallb-run-test.sh b/migtests/scripts/live-migration-fallb-run-test.sh index c63b3f73df..7e11453274 100755 --- a/migtests/scripts/live-migration-fallb-run-test.sh +++ b/migtests/scripts/live-migration-fallb-run-test.sh @@ -184,7 +184,7 @@ main() { step "Run snapshot validations." "${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'false' --fb_enabled 'true' || { - tail_log_file "yb-voyager-import-data-to-target.log" + tail_log_file "yb-voyager-import-data.log" tail_log_file "yb-voyager-export-data-from-source.log" exit 1 } diff --git a/migtests/scripts/live-migration-fallf-run-test.sh b/migtests/scripts/live-migration-fallf-run-test.sh index ddf9d23dfc..9d58241e49 100755 --- a/migtests/scripts/live-migration-fallf-run-test.sh +++ b/migtests/scripts/live-migration-fallf-run-test.sh @@ -218,7 +218,7 @@ main() { step "Run snapshot validations." "${TEST_DIR}/validate" --live_migration 'true' --ff_enabled 'true' --fb_enabled 'false' || { - tail_log_file "yb-voyager-import-data-to-target.log" + tail_log_file "yb-voyager-import-data.log" tail_log_file "yb-voyager-export-data-from-source.log" tail_log_file "yb-voyager-import-data-to-source-replica.log" exit 1 From 4962c249ba4c2e4821376b922d5d5c331437ea82 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Wed, 4 Dec 2024 13:41:28 +0000 Subject: [PATCH 09/21] Added the NOCACHE clause to the sequences tables --- .../oracle/sequences/sequence_schema.sql | 22 +++++++++---------- migtests/tests/oracle/sequences/validate | 8 +++---- .../oracle/sequences/validateAfterChanges | 14 ++++++------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/migtests/tests/oracle/sequences/sequence_schema.sql b/migtests/tests/oracle/sequences/sequence_schema.sql index f3c34ea7b8..310edc2501 100644 --- a/migtests/tests/oracle/sequences/sequence_schema.sql +++ b/migtests/tests/oracle/sequences/sequence_schema.sql @@ -1,70 +1,70 @@ drop table identity_demo_generated_always; CREATE TABLE identity_demo_generated_always ( - id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + id NUMBER GENERATED ALWAYS AS IDENTITY NOCACHE PRIMARY KEY, description VARCHAR2(100) NOT NULL ); drop table identity_demo_generated_by_def; CREATE TABLE identity_demo_generated_by_def ( - id NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + id NUMBER GENERATED BY DEFAULT AS IDENTITY NOCACHE PRIMARY KEY, description VARCHAR2(100) not null ); DROP TABLE identity_demo_with_null; CREATE TABLE identity_demo_with_null ( - id NUMBER GENERATED BY DEFAULT ON NULL AS IDENTITY PRIMARY KEY, + id NUMBER GENERATED BY DEFAULT ON NULL AS IDENTITY NOCACHE PRIMARY KEY, description VARCHAR2(100) not null ); drop table identity_demo_generated_always_start_with; CREATE TABLE identity_demo_generated_always_start_with ( - id NUMBER GENERATED ALWAYS AS IDENTITY start with 101 PRIMARY KEY, + id NUMBER GENERATED ALWAYS AS IDENTITY NOCACHE start with 101 PRIMARY KEY, description VARCHAR2(100) NOT NULL ); drop table identity_demo_generated_by_def_start_with; CREATE TABLE identity_demo_generated_by_def_start_with ( - id NUMBER GENERATED BY DEFAULT AS IDENTITY start with 101 PRIMARY KEY, + id NUMBER GENERATED BY DEFAULT AS IDENTITY NOCACHE start with 101 PRIMARY KEY, description VARCHAR2(100) not null ); drop table identity_demo_generated_by_def_inc_by; CREATE TABLE identity_demo_generated_by_def_inc_by ( - id NUMBER GENERATED BY DEFAULT AS IDENTITY increment by 101 PRIMARY KEY, + id NUMBER GENERATED BY DEFAULT AS IDENTITY NOCACHE increment by 101 PRIMARY KEY, description VARCHAR2(100) not null ); drop table identity_demo_generated_by_def_st_with_inc_by; CREATE TABLE identity_demo_generated_by_def_st_with_inc_by ( - id NUMBER GENERATED BY DEFAULT AS IDENTITY start with 5 increment by 101 PRIMARY KEY, + id NUMBER GENERATED BY DEFAULT AS IDENTITY NOCACHE start with 5 increment by 101 PRIMARY KEY, description VARCHAR2(100) not null ); drop table empty_identity_always; CREATE TABLE empty_identity_always ( - id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + id NUMBER GENERATED ALWAYS AS IDENTITY NOCACHE PRIMARY KEY, description VARCHAR2(100) NOT NULL ); drop table empty_identity_def; CREATE TABLE empty_identity_def ( - id NUMBER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + id NUMBER GENERATED BY DEFAULT AS IDENTITY NOCACHE PRIMARY KEY, description VARCHAR2(100) not null ); - drop table "Case_Sensitive_always"; +drop table "Case_Sensitive_always"; CREATE TABLE "Case_Sensitive_always" ( - id NUMBER GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + id NUMBER GENERATED ALWAYS AS IDENTITY NOCACHE PRIMARY KEY, description VARCHAR2(100) not null ); diff --git a/migtests/tests/oracle/sequences/validate b/migtests/tests/oracle/sequences/validate index e7918b8567..3abd78a3b4 100755 --- a/migtests/tests/oracle/sequences/validate +++ b/migtests/tests/oracle/sequences/validate @@ -23,7 +23,7 @@ EXPECTED_ROW_COUNT = { 'identity_demo_generated_always_start_with': 1, 'identity_demo_generated_by_def_st_with_inc_by': 3, 'identity_demo_generated_by_def_start_with': 3, - # 'identity_demo_with_null': 2, + 'identity_demo_with_null': 2, 'case_sensitive_always': 2 } @@ -34,7 +34,7 @@ EXPECTED_TABLE_SUM = { 'identity_demo_generated_always_start_with': 101, 'identity_demo_generated_by_def_st_with_inc_by': 115, 'identity_demo_generated_by_def_start_with': 205, - # 'identity_demo_with_null': 3, + 'identity_demo_with_null': 3, 'case_sensitive_always': 3 } @@ -45,7 +45,7 @@ EXPECTED_TABLE_SUM_AFTER_INSERT = { 'identity_demo_generated_always_start_with': 223, 'identity_demo_generated_by_def_st_with_inc_by': 2241, 'identity_demo_generated_by_def_start_with': 327, - # 'identity_demo_with_null': 25, + 'identity_demo_with_null': 25, 'case_sensitive_always': 25 } @@ -57,7 +57,7 @@ if os.environ.get('BETA_FAST_DATA_EXPORT') == '1': 'identity_demo_generated_always_start_with': 203, 'identity_demo_generated_by_def_st_with_inc_by': 322, 'identity_demo_generated_by_def_start_with': 308, - # 'identity_demo_with_null': 6, + 'identity_demo_with_null': 6, 'case_sensitive_always': 6 } diff --git a/migtests/tests/oracle/sequences/validateAfterChanges b/migtests/tests/oracle/sequences/validateAfterChanges index f44d9a1ab6..a549b591c0 100755 --- a/migtests/tests/oracle/sequences/validateAfterChanges +++ b/migtests/tests/oracle/sequences/validateAfterChanges @@ -29,7 +29,7 @@ EXPECTED_ROW_COUNT = { 'identity_demo_generated_always_start_with': 1, 'identity_demo_generated_by_def_st_with_inc_by': 3, 'identity_demo_generated_by_def_start_with': 3, - # 'identity_demo_with_null': 3, + 'identity_demo_with_null': 3, 'empty_identity_def': 1, 'empty_identity_always': 3, 'case_sensitive_always': 3 @@ -42,7 +42,7 @@ EXPECTED_TABLE_SUM = { 'identity_demo_generated_always_start_with': 101, 'identity_demo_generated_by_def_st_with_inc_by': 115, 'identity_demo_generated_by_def_start_with': 205, - # 'identity_demo_with_null': 9, + 'identity_demo_with_null': 9, 'empty_identity_def': 2, 'empty_identity_always': 9, 'case_sensitive_always': 9 @@ -55,7 +55,7 @@ EXPECTED_TABLE_SUM_AFTER_INSERT = { 'identity_demo_generated_always_start_with': 203, 'identity_demo_generated_by_def_st_with_inc_by': 322, 'identity_demo_generated_by_def_start_with': 308, - # 'identity_demo_with_null': 14, + 'identity_demo_with_null': 14, 'empty_identity_def': 5, 'empty_identity_always': 14, 'case_sensitive_always': 14 @@ -69,7 +69,7 @@ EXPECTED_ROW_COUNT_FF = { 'identity_demo_generated_always_start_with': 1, 'identity_demo_generated_by_def_st_with_inc_by': 3, 'identity_demo_generated_by_def_start_with': 3, - # 'identity_demo_with_null': 4, + 'identity_demo_with_null': 4, 'empty_identity_def': 1, 'empty_identity_always': 4, 'case_sensitive_always': 5 @@ -82,7 +82,7 @@ EXPECTED_TABLE_SUM_FF = { 'identity_demo_generated_always_start_with': 101, 'identity_demo_generated_by_def_st_with_inc_by': 115, 'identity_demo_generated_by_def_start_with': 205, - # 'identity_demo_with_null': 17, + 'identity_demo_with_null': 17, 'empty_identity_def': 3, 'empty_identity_always': 18, 'case_sensitive_always': 25 @@ -95,7 +95,7 @@ EXPECTED_TABLE_SUM_AFTER_INSERT_FF = { 'identity_demo_generated_always_start_with': 203, 'identity_demo_generated_by_def_st_with_inc_by': 322, 'identity_demo_generated_by_def_start_with': 308, - # 'identity_demo_with_null': 122, + 'identity_demo_with_null': 122, 'empty_identity_def': 106, 'empty_identity_always': 123, 'case_sensitive_always': 130 @@ -108,7 +108,7 @@ EXPECTED_TABLE_SUM_AFTER_INSERT_ORA = { 'identity_demo_generated_always_start_with': 203, 'identity_demo_generated_by_def_st_with_inc_by': 222, 'identity_demo_generated_by_def_start_with': 308, - # 'identity_demo_with_null': 24, + 'identity_demo_with_null': 24, 'empty_identity_def': 7, 'empty_identity_always': 25, 'case_sensitive_always': 33 From eecd0e97100e683dc5c468481439598e641683ca Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Sun, 8 Dec 2024 16:11:35 +0000 Subject: [PATCH 10/21] Added uniqueness to ssl tests --- migtests/scripts/live-migration-fallb-run-test.sh | 6 +++++- migtests/scripts/live-migration-fallf-run-test.sh | 6 +++++- migtests/scripts/live-migration-run-test.sh | 6 +++++- migtests/scripts/run-test.sh | 5 ++++- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/migtests/scripts/live-migration-fallb-run-test.sh b/migtests/scripts/live-migration-fallb-run-test.sh index 7e11453274..5333d52071 100755 --- a/migtests/scripts/live-migration-fallb-run-test.sh +++ b/migtests/scripts/live-migration-fallb-run-test.sh @@ -19,7 +19,6 @@ export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_fallb_export-dir"} export QUEUE_SEGMENT_MAX_BYTES=400 export PYTHONPATH="${REPO_ROOT}/migtests/lib" @@ -31,6 +30,11 @@ else source "${TEST_DIR}/env.sh" fi +export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_fallb_export-dir"} +if [ -n "${SOURCE_DB_SSL_MODE}" ]; then + EXPORT_DIR="${EXPORT_DIR}_ssl" +fi + if [ "${SOURCE_DB_TYPE}" = "oracle" ] then source ${SCRIPTS}/${SOURCE_DB_TYPE}/live_env.sh diff --git a/migtests/scripts/live-migration-fallf-run-test.sh b/migtests/scripts/live-migration-fallf-run-test.sh index 9d58241e49..d11f41ce73 100755 --- a/migtests/scripts/live-migration-fallf-run-test.sh +++ b/migtests/scripts/live-migration-fallf-run-test.sh @@ -19,7 +19,6 @@ export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_fallf_export-dir"} export PYTHONPATH="${REPO_ROOT}/migtests/lib" export PATH="${PATH}:/usr/lib/oracle/21/client64/bin" @@ -32,6 +31,11 @@ else source "${TEST_DIR}/env.sh" fi +export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_fallf_export-dir"} +if [ -n "${SOURCE_DB_SSL_MODE}" ]; then + EXPORT_DIR="${EXPORT_DIR}_ssl" +fi + if [ "${SOURCE_DB_TYPE}" = "oracle" ] then source ${SCRIPTS}/${SOURCE_DB_TYPE}/live_env.sh diff --git a/migtests/scripts/live-migration-run-test.sh b/migtests/scripts/live-migration-run-test.sh index 509d16710e..b1b24550cc 100755 --- a/migtests/scripts/live-migration-run-test.sh +++ b/migtests/scripts/live-migration-run-test.sh @@ -19,7 +19,6 @@ export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_live_export-dir"} export QUEUE_SEGMENT_MAX_BYTES=400 export PYTHONPATH="${REPO_ROOT}/migtests/lib" @@ -38,6 +37,11 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi +export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_live_export-dir"} +if [ -n "${SOURCE_DB_SSL_MODE}" ]; then + EXPORT_DIR="${EXPORT_DIR}_ssl" +fi + if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_live"} elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index 12c7a6661c..9f0f4a7832 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -18,7 +18,6 @@ export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_offline_export-dir"} export PYTHONPATH="${REPO_ROOT}/migtests/lib" @@ -37,6 +36,10 @@ fi source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh +export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_offline_export-dir"} +if [ -n "${SOURCE_DB_SSL_MODE}" ]; then + EXPORT_DIR="${EXPORT_DIR}_ssl" +fi if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_offline"} From 901d7741977021edba3906b6e3c7f739b985d680 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Wed, 18 Dec 2024 21:29:11 +0000 Subject: [PATCH 11/21] Fix for missing operand basename error --- .../oracle/yb-voyager-oracle-gather-assessment-metadata.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh b/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh index 7550d8f765..da02606224 100755 --- a/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh +++ b/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh @@ -14,8 +14,8 @@ set -e -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE:-$0}" )" &> /dev/null && pwd ) -SCRIPT_NAME=$(basename $0) +SCRIPT_DIR=$(cd -- "$(dirname "${BASH_SOURCE[0]:-$0}")" &> /dev/null && pwd) +SCRIPT_NAME=$(basename "${BASH_SOURCE[0]:-$0}") HELP_TEXT=" Usage: $SCRIPT_NAME From d478e311ea4bd6adfda4d38bda9ea1f28c971336 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Wed, 18 Dec 2024 22:00:38 +0000 Subject: [PATCH 12/21] Added verbose to ora2pg output --- .../oracle/yb-voyager-oracle-gather-assessment-metadata.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh b/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh index da02606224..99eb0e242e 100755 --- a/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh +++ b/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh @@ -14,8 +14,8 @@ set -e -SCRIPT_DIR=$(cd -- "$(dirname "${BASH_SOURCE[0]:-$0}")" &> /dev/null && pwd) -SCRIPT_NAME=$(basename "${BASH_SOURCE[0]:-$0}") +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE:-$0}" )" &> /dev/null && pwd ) +SCRIPT_NAME=$(basename $0) HELP_TEXT=" Usage: $SCRIPT_NAME @@ -269,7 +269,7 @@ main() { run_command "$ora2pg_cmd" done - ora2pg_report_cmd="ora2pg -t show_report --estimate_cost -c $OUTPUT_FILE_PATH --dump_as_sheet --quiet > $assessment_metadata_dir/schema/ora2pg_report.csv" + ora2pg_report_cmd="ora2pg -t -v show_report --estimate_cost -c $OUTPUT_FILE_PATH --dump_as_sheet > $assessment_metadata_dir/schema/ora2pg_report.csv" log "INFO" "executing ora2pg command for report: $ora2pg_report_cmd" run_command "$ora2pg_report_cmd" From ccece7e301d08463e05605ec92f3095a890e2472 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Thu, 19 Dec 2024 07:45:09 +0000 Subject: [PATCH 13/21] Changed the source db name for PG Partitions with Index test --- migtests/tests/pg/partitions-with-indexes/env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migtests/tests/pg/partitions-with-indexes/env.sh b/migtests/tests/pg/partitions-with-indexes/env.sh index 6b22305323..7f216dc8da 100644 --- a/migtests/tests/pg/partitions-with-indexes/env.sh +++ b/migtests/tests/pg/partitions-with-indexes/env.sh @@ -1,3 +1,3 @@ export SOURCE_DB_TYPE="postgresql" -export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"partitions"} +export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"partitions_with_index"} export SOURCE_DB_SCHEMA="public,p1,p2" From fa676102033521ffee1eeaf24a5761e73fab037a Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Fri, 20 Dec 2024 06:21:25 +0000 Subject: [PATCH 14/21] Use debug with ora2pg commands --- .../oracle/yb-voyager-oracle-gather-assessment-metadata.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh b/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh index 99eb0e242e..15fcab5560 100755 --- a/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh +++ b/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh @@ -269,7 +269,7 @@ main() { run_command "$ora2pg_cmd" done - ora2pg_report_cmd="ora2pg -t -v show_report --estimate_cost -c $OUTPUT_FILE_PATH --dump_as_sheet > $assessment_metadata_dir/schema/ora2pg_report.csv" + ora2pg_report_cmd="ora2pg -t -d show_report --estimate_cost -c $OUTPUT_FILE_PATH --dump_as_sheet > $assessment_metadata_dir/schema/ora2pg_report.csv" log "INFO" "executing ora2pg command for report: $ora2pg_report_cmd" run_command "$ora2pg_report_cmd" From bf311563b54915224331fd0ce5c37972d2c74c84 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Fri, 20 Dec 2024 07:21:11 +0000 Subject: [PATCH 15/21] Use debug with ora2pg commands --- .../oracle/yb-voyager-oracle-gather-assessment-metadata.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh b/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh index 15fcab5560..9525bc77ba 100755 --- a/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh +++ b/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh @@ -269,7 +269,7 @@ main() { run_command "$ora2pg_cmd" done - ora2pg_report_cmd="ora2pg -t -d show_report --estimate_cost -c $OUTPUT_FILE_PATH --dump_as_sheet > $assessment_metadata_dir/schema/ora2pg_report.csv" + ora2pg_report_cmd="ora2pg -t show_report --estimate_cost -d -c $OUTPUT_FILE_PATH --dump_as_sheet > $assessment_metadata_dir/schema/ora2pg_report.csv" log "INFO" "executing ora2pg command for report: $ora2pg_report_cmd" run_command "$ora2pg_report_cmd" From ff7373aba1f4d45c7bc96104d278a2684952ac36 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Tue, 31 Dec 2024 10:31:30 +0000 Subject: [PATCH 16/21] Cleanup --- migtests/scripts/functions.sh | 5 +---- migtests/scripts/live-migration-run-test.sh | 2 +- migtests/tests/oracle/bulk-assessment-test/init-db | 8 -------- migtests/tests/oracle/sequences/validateAfterChanges | 1 - .../yb-voyager-oracle-gather-assessment-metadata.sh | 2 +- 5 files changed, 3 insertions(+), 15 deletions(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index c70bb0592f..18c7b87872 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -667,14 +667,11 @@ grant_permissions_for_live_migration() { setup_fallback_environment() { if [ "${SOURCE_DB_TYPE}" = "oracle" ]; then run_sqlplus_as_sys ${SOURCE_DB_NAME} ${SCRIPTS}/oracle/create_metadata_tables.sql - # run_sqlplus_as_sys ${SOURCE_DB_NAME} ${SCRIPTS}/oracle/fall_back_prep.sql - # Define the temporary file + TEMP_SCRIPT="/tmp/fall_back_prep.sql" - # Replace TEST_SCHEMA with the value of SOURCE_DB_SCHEMA in the script sed "s/TEST_SCHEMA/${SOURCE_DB_SCHEMA}/g" ${SCRIPTS}/oracle/fall_back_prep.sql > $TEMP_SCRIPT - # Run the updated script using the run_sqlplus_as_sys function run_sqlplus_as_sys ${SOURCE_DB_NAME} $TEMP_SCRIPT # Clean up the temporary file after execution diff --git a/migtests/scripts/live-migration-run-test.sh b/migtests/scripts/live-migration-run-test.sh index b1b24550cc..5ed2ab9f8b 100755 --- a/migtests/scripts/live-migration-run-test.sh +++ b/migtests/scripts/live-migration-run-test.sh @@ -146,7 +146,7 @@ main() { step "Export data." # false if exit code of export_data is non-zero - export_data --export-type "snapshot-and-changes" --log-level debug || { + export_data --export-type "snapshot-and-changes" || { tail_log_file "yb-voyager-export-data.log" tail_log_file "debezium-source_db_exporter.log" exit 1 diff --git a/migtests/tests/oracle/bulk-assessment-test/init-db b/migtests/tests/oracle/bulk-assessment-test/init-db index 97add76d37..0cd3340a0d 100755 --- a/migtests/tests/oracle/bulk-assessment-test/init-db +++ b/migtests/tests/oracle/bulk-assessment-test/init-db @@ -10,20 +10,12 @@ export SOURCE_DB_SCHEMA="TEST_SCHEMA" create_source_db ${SOURCE_DB_SCHEMA} ./cleanup-db -# echo "Deleting existing data in source database" -# run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TESTS_DIR}/oracle/utils/delete_full_schema echo "Initialising source database 1 & inserting data" run_sqlplus_as_schema_owner ${SOURCE_DB_NAME} ${TEST_DIR}/../assessment-report-test/oracle_assessment_report.sql echo "Initialising source database 2 & inserting data" -# cat > create-schema2.sql << EOF -# CREATE USER TEST_SCHEMA2 identified by "password"; -# GRANT ALL PRIVILEGES TO TEST_SCHEMA2; -# EOF -# run_sqlplus_as_sys ${SOURCE_DB_NAME} "create-schema2.sql" - export SOURCE_DB_SCHEMA="TEST_SCHEMA2" create_source_db ${SOURCE_DB_SCHEMA} diff --git a/migtests/tests/oracle/sequences/validateAfterChanges b/migtests/tests/oracle/sequences/validateAfterChanges index a549b591c0..95f0832da3 100755 --- a/migtests/tests/oracle/sequences/validateAfterChanges +++ b/migtests/tests/oracle/sequences/validateAfterChanges @@ -174,7 +174,6 @@ def migration_completed_checks_fb(): print("Running tests on Oracle source") global db_schema db_schema = os.environ.get("SOURCE_DB_SCHEMA").upper() - print(db_schema) change_expected_values_ff_fb() oracle.run_checks(migration_completed_checks, db_type="source") diff --git a/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh b/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh index 9525bc77ba..5f30e93410 100755 --- a/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh +++ b/yb-voyager/src/srcdb/data/gather-assessment-metadata/oracle/yb-voyager-oracle-gather-assessment-metadata.sh @@ -269,7 +269,7 @@ main() { run_command "$ora2pg_cmd" done - ora2pg_report_cmd="ora2pg -t show_report --estimate_cost -d -c $OUTPUT_FILE_PATH --dump_as_sheet > $assessment_metadata_dir/schema/ora2pg_report.csv" + ora2pg_report_cmd="ora2pg -t show_report --estimate_cost -c $OUTPUT_FILE_PATH --dump_as_sheet --quiet > $assessment_metadata_dir/schema/ora2pg_report.csv" log "INFO" "executing ora2pg command for report: $ora2pg_report_cmd" run_command "$ora2pg_report_cmd" From c49b3d115243ba6f6ed72082ca80845d09b148a6 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Tue, 7 Jan 2025 09:14:01 +0000 Subject: [PATCH 17/21] Review comments --- migtests/scripts/functions.sh | 31 +++++++++++++++- .../scripts/live-migration-fallb-run-test.sh | 25 ++++++------- .../scripts/live-migration-fallf-run-test.sh | 36 ++++++++++--------- migtests/scripts/live-migration-run-test.sh | 30 ++++++++-------- migtests/scripts/run-test.sh | 29 ++++++++------- 5 files changed, 95 insertions(+), 56 deletions(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 18c7b87872..e2e7060d7a 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -1071,4 +1071,33 @@ EOF exit 1 ;; esac -} \ No newline at end of file +} + +normalize_and_export_vars() { + local test_suffix=$1 + + # Normalize TEST_NAME + export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" + + # Set EXPORT_DIR + export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_${test_suffix}_export-dir"} + if [ -n "${SOURCE_DB_SSL_MODE}" ]; then + EXPORT_DIR="${EXPORT_DIR}_ssl" + fi + + # Set database-specific variables + case "${SOURCE_DB_TYPE}" in + postgresql|mysql) + export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_${test_suffix}"} + ;; + oracle) + # Limit schema name to 10 characters for Oracle/Debezium due to 30 character limit + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_${test_suffix}"} + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} + ;; + *) + echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" + exit 1 + ;; + esac +} diff --git a/migtests/scripts/live-migration-fallb-run-test.sh b/migtests/scripts/live-migration-fallb-run-test.sh index 5333d52071..362ecd5bf0 100755 --- a/migtests/scripts/live-migration-fallb-run-test.sh +++ b/migtests/scripts/live-migration-fallb-run-test.sh @@ -18,9 +18,7 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" export QUEUE_SEGMENT_MAX_BYTES=400 - export PYTHONPATH="${REPO_ROOT}/migtests/lib" # Order of env.sh import matters. @@ -42,19 +40,22 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi -if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then - export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_fallb"} -elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then - export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_fallb"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} -else - echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" - exit 1 -fi - -source ${SCRIPTS}/yugabytedb/env.sh +# export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" +# if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then +# export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_fallb"} +# elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then +# export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_fallb"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} +# else +# echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" +# exit 1 +# fi source ${SCRIPTS}/functions.sh +normalize_and_export_vars "fallb" + +source ${SCRIPTS}/yugabytedb/env.sh + main() { echo "Deleting the parent export-dir present in the test directory" diff --git a/migtests/scripts/live-migration-fallf-run-test.sh b/migtests/scripts/live-migration-fallf-run-test.sh index d11f41ce73..af6813e619 100755 --- a/migtests/scripts/live-migration-fallf-run-test.sh +++ b/migtests/scripts/live-migration-fallf-run-test.sh @@ -18,7 +18,6 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" export PYTHONPATH="${REPO_ROOT}/migtests/lib" export PATH="${PATH}:/usr/lib/oracle/21/client64/bin" @@ -31,11 +30,6 @@ else source "${TEST_DIR}/env.sh" fi -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_fallf_export-dir"} -if [ -n "${SOURCE_DB_SSL_MODE}" ]; then - EXPORT_DIR="${EXPORT_DIR}_ssl" -fi - if [ "${SOURCE_DB_TYPE}" = "oracle" ] then source ${SCRIPTS}/${SOURCE_DB_TYPE}/live_env.sh @@ -43,21 +37,30 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi -if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then - export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_fallf"} -elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then - export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_fallf"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} -else - echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" - exit 1 -fi +# export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" +# export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_fallf_export-dir"} +# if [ -n "${SOURCE_DB_SSL_MODE}" ]; then +# EXPORT_DIR="${EXPORT_DIR}_ssl" +# fi -source ${SCRIPTS}/${SOURCE_DB_TYPE}/ff_env.sh +# if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then +# export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_fallf"} +# elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then +# export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_fallf"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} +# else +# echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" +# exit 1 +# fi -source ${SCRIPTS}/yugabytedb/env.sh source ${SCRIPTS}/functions.sh +normalize_and_export_vars "fallf" + +source ${SCRIPTS}/${SOURCE_DB_TYPE}/ff_env.sh + +source ${SCRIPTS}/yugabytedb/env.sh + main() { echo "Deleting the parent export-dir present in the test directory" @@ -77,6 +80,7 @@ main() { if [ "${SOURCE_DB_TYPE}" = "oracle" ] then create_source_db ${SOURCE_DB_SCHEMA} + # TODO: Add dynamic Fall Forward schema creation create_source_db ${SOURCE_REPLICA_DB_SCHEMA} run_sqlplus_as_sys ${SOURCE_REPLICA_DB_NAME} ${SCRIPTS}/oracle/create_metadata_tables.sql fi diff --git a/migtests/scripts/live-migration-run-test.sh b/migtests/scripts/live-migration-run-test.sh index 5ed2ab9f8b..2a59849659 100755 --- a/migtests/scripts/live-migration-run-test.sh +++ b/migtests/scripts/live-migration-run-test.sh @@ -37,23 +37,25 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_live_export-dir"} -if [ -n "${SOURCE_DB_SSL_MODE}" ]; then - EXPORT_DIR="${EXPORT_DIR}_ssl" -fi +# export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_live_export-dir"} +# if [ -n "${SOURCE_DB_SSL_MODE}" ]; then +# EXPORT_DIR="${EXPORT_DIR}_ssl" +# fi + +# if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then +# export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_live"} +# elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then +# export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_live"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} +# else +# echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" +# exit 1 +# fi -if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then - export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_live"} -elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then - export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_live"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} -else - echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" - exit 1 -fi +source ${SCRIPTS}/functions.sh -source ${SCRIPTS}/yugabytedb/env.sh +normalize_and_export_vars "live" -source ${SCRIPTS}/functions.sh +source ${SCRIPTS}/yugabytedb/env.sh main() { diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index 9f0f4a7832..bc4f57cbd0 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -36,22 +36,25 @@ fi source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_offline_export-dir"} -if [ -n "${SOURCE_DB_SSL_MODE}" ]; then - EXPORT_DIR="${EXPORT_DIR}_ssl" -fi +# export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_offline_export-dir"} +# if [ -n "${SOURCE_DB_SSL_MODE}" ]; then +# EXPORT_DIR="${EXPORT_DIR}_ssl" +# fi + +# if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then +# export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_offline"} +# elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then +# export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_offline"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} +# else +# echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" +# exit 1 +# fi -if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then - export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_offline"} -elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then - export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_offline"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} -else - echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" - exit 1 -fi +source ${SCRIPTS}/functions.sh + +normalize_and_export_vars "offline" source ${SCRIPTS}/yugabytedb/env.sh -source ${SCRIPTS}/functions.sh main() { echo "Deleting the parent export-dir present in the test directory" From a582746248aaec21fa4b29054ae497ce257644cc Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Tue, 7 Jan 2025 15:45:20 +0000 Subject: [PATCH 18/21] Cleanup --- .../scripts/live-migration-fallb-run-test.sh | 15 --------------- .../scripts/live-migration-fallf-run-test.sh | 18 +----------------- migtests/scripts/live-migration-run-test.sh | 15 --------------- migtests/scripts/run-test.sh | 14 -------------- 4 files changed, 1 insertion(+), 61 deletions(-) diff --git a/migtests/scripts/live-migration-fallb-run-test.sh b/migtests/scripts/live-migration-fallb-run-test.sh index 362ecd5bf0..bac1a1e968 100755 --- a/migtests/scripts/live-migration-fallb-run-test.sh +++ b/migtests/scripts/live-migration-fallb-run-test.sh @@ -28,11 +28,6 @@ else source "${TEST_DIR}/env.sh" fi -export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_fallb_export-dir"} -if [ -n "${SOURCE_DB_SSL_MODE}" ]; then - EXPORT_DIR="${EXPORT_DIR}_ssl" -fi - if [ "${SOURCE_DB_TYPE}" = "oracle" ] then source ${SCRIPTS}/${SOURCE_DB_TYPE}/live_env.sh @@ -40,16 +35,6 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi -# export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" -# if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then -# export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_fallb"} -# elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then -# export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_fallb"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} -# else -# echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" -# exit 1 -# fi - source ${SCRIPTS}/functions.sh normalize_and_export_vars "fallb" diff --git a/migtests/scripts/live-migration-fallf-run-test.sh b/migtests/scripts/live-migration-fallf-run-test.sh index af6813e619..54dbe7bd78 100755 --- a/migtests/scripts/live-migration-fallf-run-test.sh +++ b/migtests/scripts/live-migration-fallf-run-test.sh @@ -37,22 +37,6 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi -# export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" -# export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_fallf_export-dir"} -# if [ -n "${SOURCE_DB_SSL_MODE}" ]; then -# EXPORT_DIR="${EXPORT_DIR}_ssl" -# fi - -# if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then -# export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_fallf"} -# elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then -# export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_fallf"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} -# else -# echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" -# exit 1 -# fi - - source ${SCRIPTS}/functions.sh normalize_and_export_vars "fallf" @@ -80,7 +64,7 @@ main() { if [ "${SOURCE_DB_TYPE}" = "oracle" ] then create_source_db ${SOURCE_DB_SCHEMA} - # TODO: Add dynamic Fall Forward schema creation + # TODO: Add dynamic Fall Forward schema creation. Currently using the same name for all tests. create_source_db ${SOURCE_REPLICA_DB_SCHEMA} run_sqlplus_as_sys ${SOURCE_REPLICA_DB_NAME} ${SCRIPTS}/oracle/create_metadata_tables.sql fi diff --git a/migtests/scripts/live-migration-run-test.sh b/migtests/scripts/live-migration-run-test.sh index 2a59849659..98f99e6beb 100755 --- a/migtests/scripts/live-migration-run-test.sh +++ b/migtests/scripts/live-migration-run-test.sh @@ -18,7 +18,6 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" export QUEUE_SEGMENT_MAX_BYTES=400 export PYTHONPATH="${REPO_ROOT}/migtests/lib" @@ -37,20 +36,6 @@ else source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh fi -# export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_live_export-dir"} -# if [ -n "${SOURCE_DB_SSL_MODE}" ]; then -# EXPORT_DIR="${EXPORT_DIR}_ssl" -# fi - -# if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then -# export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_live"} -# elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then -# export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_live"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} -# else -# echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" -# exit 1 -# fi - source ${SCRIPTS}/functions.sh normalize_and_export_vars "live" diff --git a/migtests/scripts/run-test.sh b/migtests/scripts/run-test.sh index bc4f57cbd0..e40cc60d10 100755 --- a/migtests/scripts/run-test.sh +++ b/migtests/scripts/run-test.sh @@ -17,7 +17,6 @@ export REPO_ROOT="${PWD}" export SCRIPTS="${REPO_ROOT}/migtests/scripts" export TESTS_DIR="${REPO_ROOT}/migtests/tests" export TEST_DIR="${TESTS_DIR}/${TEST_NAME}" -export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" export PYTHONPATH="${REPO_ROOT}/migtests/lib" @@ -36,19 +35,6 @@ fi source ${SCRIPTS}/${SOURCE_DB_TYPE}/env.sh -# export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_offline_export-dir"} -# if [ -n "${SOURCE_DB_SSL_MODE}" ]; then -# EXPORT_DIR="${EXPORT_DIR}_ssl" -# fi - -# if [[ "${SOURCE_DB_TYPE}" == "postgresql" || "${SOURCE_DB_TYPE}" == "mysql" ]]; then -# export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_offline"} -# elif [[ "${SOURCE_DB_TYPE}" == "oracle" ]]; then -# export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_offline"} && export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} -# else -# echo "ERROR: Unsupported SOURCE_DB_TYPE: ${SOURCE_DB_TYPE}" -# exit 1 -# fi source ${SCRIPTS}/functions.sh From 0c88b8a84e00e6c04bf0495432c237fcf5ca522e Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Thu, 9 Jan 2025 06:32:26 +0000 Subject: [PATCH 19/21] Addressed review comment --- migtests/scripts/functions.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index e2e7060d7a..297c713f91 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -1077,10 +1077,10 @@ normalize_and_export_vars() { local test_suffix=$1 # Normalize TEST_NAME - export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" + export NORMALIZED_TEST_NAME="$(echo "${TEST_NAME}_${test_suffix}" | tr '/-' '_')" # Set EXPORT_DIR - export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_${test_suffix}_export-dir"} + export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_export-dir"} if [ -n "${SOURCE_DB_SSL_MODE}" ]; then EXPORT_DIR="${EXPORT_DIR}_ssl" fi @@ -1088,11 +1088,11 @@ normalize_and_export_vars() { # Set database-specific variables case "${SOURCE_DB_TYPE}" in postgresql|mysql) - export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_${test_suffix}"} + export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}"} ;; oracle) # Limit schema name to 10 characters for Oracle/Debezium due to 30 character limit - export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_${test_suffix}"} + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}"} export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} ;; *) From c6fa66228c9229115002ff22cd252d82bf08f93b Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Thu, 9 Jan 2025 09:12:18 +0000 Subject: [PATCH 20/21] Revert "Addressed review comment" This reverts commit 0c88b8a84e00e6c04bf0495432c237fcf5ca522e. --- migtests/scripts/functions.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index 297c713f91..e2e7060d7a 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -1077,10 +1077,10 @@ normalize_and_export_vars() { local test_suffix=$1 # Normalize TEST_NAME - export NORMALIZED_TEST_NAME="$(echo "${TEST_NAME}_${test_suffix}" | tr '/-' '_')" + export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" # Set EXPORT_DIR - export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_export-dir"} + export EXPORT_DIR=${EXPORT_DIR:-"${TEST_DIR}/${NORMALIZED_TEST_NAME}_${test_suffix}_export-dir"} if [ -n "${SOURCE_DB_SSL_MODE}" ]; then EXPORT_DIR="${EXPORT_DIR}_ssl" fi @@ -1088,11 +1088,11 @@ normalize_and_export_vars() { # Set database-specific variables case "${SOURCE_DB_TYPE}" in postgresql|mysql) - export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}"} + export SOURCE_DB_NAME=${SOURCE_DB_NAME:-"${NORMALIZED_TEST_NAME}_${test_suffix}"} ;; oracle) # Limit schema name to 10 characters for Oracle/Debezium due to 30 character limit - export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}"} + export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_${test_suffix}"} export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} ;; *) From dcdc03569b9b06172bd1afa79089c50ed0235bb1 Mon Sep 17 00:00:00 2001 From: shubham-yb Date: Thu, 9 Jan 2025 09:24:35 +0000 Subject: [PATCH 21/21] Added comments --- migtests/scripts/functions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/migtests/scripts/functions.sh b/migtests/scripts/functions.sh index e2e7060d7a..49275ad445 100644 --- a/migtests/scripts/functions.sh +++ b/migtests/scripts/functions.sh @@ -1077,6 +1077,7 @@ normalize_and_export_vars() { local test_suffix=$1 # Normalize TEST_NAME + # Keeping the full name for PG and MySQL to test out large schema/export dir names export NORMALIZED_TEST_NAME="$(echo "$TEST_NAME" | tr '/-' '_')" # Set EXPORT_DIR @@ -1092,6 +1093,7 @@ normalize_and_export_vars() { ;; oracle) # Limit schema name to 10 characters for Oracle/Debezium due to 30 character limit + # Since test_suffix is the unique identifying factor, we need to add it post all the normalization export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA:-"${NORMALIZED_TEST_NAME:0:10}_${test_suffix}"} export SOURCE_DB_SCHEMA=${SOURCE_DB_SCHEMA^^} ;;