From 536a14907084d18d58fdb7061db9b6dc167e1bc1 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Fri, 9 Feb 2024 16:01:18 +0530 Subject: [PATCH 01/11] Integration test for all datatype mappings & conversions --- .../DataStreamToSpannerSimpleIT.java | 414 ++++++++++++++++++ .../mysql-backfill-AllDatatypeColumns.jsonl | 2 + .../mysql-backfill-AllDatatypeColumns2.jsonl | 2 + .../mysql-backfill-Category.jsonl | 2 + .../mysql-cdc-AllDatatypeColumns.jsonl | 0 .../mysql-cdc-AllDatatypeColumns2.jsonl | 2 + .../mysql-cdc-Category.jsonl | 4 + .../mysql-schema.sql | 73 +++ .../spanner-schema.sql | 82 +++- 9 files changed, 574 insertions(+), 7 deletions(-) create mode 100644 v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns.jsonl create mode 100644 v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns2.jsonl create mode 100644 v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-Category.jsonl create mode 100644 v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl create mode 100644 v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns2.jsonl create mode 100644 v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-Category.jsonl diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java index 0f609dd0ff..289b7fe275 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java @@ -58,6 +58,9 @@ public class DataStreamToSpannerSimpleIT extends DataStreamToSpannerITBase { private static final String TABLE1 = "Users"; private static final String TABLE2 = "Movie"; + private static final String TABLE3 = "Category"; + private static final String TABLE4 = "AllDatatypeColumns"; + private static final String TABLE5 = "AllDatatypeColumns2"; private static final String SESSION_FILE_RESOURCE = "DataStreamToSpannerSimpleIT/mysql-session.json"; @@ -237,6 +240,178 @@ public void interleavedAndFKAndIndexTest() { assertArticlesTable(); } + @Test + public void migrationTestWithAllDatatypeConversionsWithInsertsOnly() { + // Construct a ChainedConditionCheck with 4 stages. + // 1. Send initial wave of events + // 2. Wait on Spanner to have events + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE4, + "backfill.jsonl", + "DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns.jsonl"), + SpannerRowsCheck.builder(spannerResourceManager, TABLE4) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + // Wait for conditions + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + // Assert Conditions + // assertThatResult(result).meetsConditions(); + + assertAllDatatypeColumnsTableBackfillContents(); + } + + @Test + public void migrationTestWithAllDatatypeMappingsWithInsertsOnly() { + // Construct a ChainedConditionCheck with 4 stages. + // 1. Send initial wave of events + // 2. Wait on Spanner to have events + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE5, + "backfill.jsonl", + "DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns2.jsonl"), + SpannerRowsCheck.builder(spannerResourceManager, TABLE5) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + // Wait for conditions + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + // Assert Conditions + assertThatResult(result).meetsConditions(); + + assertAllDatatypeColumns2TableBackfillContents(); + } + + @Test + public void migrationTestWithAllDatatypeMappingsWithUpdatesAndDeletes() { + // Construct a ChainedConditionCheck with 4 stages. + // 1. Send initial wave of events + // 2. Wait on Spanner to have events + // 3. Send second wave of events + // 4. Wait on Spanner to merge second wave of events + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE5, + "backfill.jsonl", + "DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns2.jsonl"), + SpannerRowsCheck.builder(spannerResourceManager, TABLE5) + .setMinRows(2) + .setMaxRows(2) + .build(), + uploadDataStreamFile( + jobInfo, + TABLE5, + "cdc1.jsonl", + "DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns2.jsonl"), + SpannerRowsCheck.builder(spannerResourceManager, TABLE5) + .setMinRows(1) + .setMaxRows(1) + .build())) + .build(); + + // Wait for conditions + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + // Assert Conditions + assertThatResult(result).meetsConditions(); + + assertAllDatatypeColumns2TableCdcContents(); + } + + @Test + public void migrationTestWithRenameAndDropColumnWithInsertsOnly() { + // Construct a ChainedConditionCheck with 4 stages. + // 1. Send initial wave of events + // 2. Wait on Spanner to have events + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE3, + "backfill.jsonl", + "DataStreamToSpannerSimpleIT/mysql-backfill-Category.jsonl"), + SpannerRowsCheck.builder(spannerResourceManager, TABLE3) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + // Wait for conditions + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + // Assert Conditions + assertThatResult(result).meetsConditions(); + + assertCategoryTableBackfillContents(); + } + + @Test + public void migrationTestWithRenameAndDropColumnWithUpdatesAndDeletes() { + // Construct a ChainedConditionCheck with 4 stages. + // 1. Send initial wave of events + // 2. Wait on Spanner to have events + // 3. Send second wave of events + // 4. Wait on Spanner to merge second wave of events + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE3, + "backfill.jsonl", + "DataStreamToSpannerSimpleIT/mysql-backfill-Category.jsonl"), + SpannerRowsCheck.builder(spannerResourceManager, TABLE3) + .setMinRows(2) + .setMaxRows(2) + .build(), + uploadDataStreamFile( + jobInfo, + TABLE3, + "cdc1.jsonl", + "DataStreamToSpannerSimpleIT/mysql-cdc-Category.jsonl"), + SpannerRowsCheck.builder(spannerResourceManager, TABLE3) + .setMinRows(3) + .setMaxRows(3) + .build())) + .build(); + + // Wait for conditions + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + // Assert Conditions + assertThatResult(result).meetsConditions(); + + assertCategoryTableCdcContents(); + } + private void assertUsersTableContents() { List> events = new ArrayList<>(); @@ -299,6 +474,245 @@ private void assertMovieTableContents() { Assert.assertEquals(931.512, numericVals.get(1).getBigDecimal(0).doubleValue(), 0.001); } + private void assertAllDatatypeColumnsTableBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", "10"); + row.put("text_column", "dGV4dF9kYXRhXzEK"); + row.put("date_column", "2024-02-08T00:00:00.000Z"); + row.put("smallint_column", "50"); + row.put("mediumint_column", "1000"); + row.put("int_column", "50000"); + row.put("bigint_column", "987654321"); + row.put("float_column", "45.67"); + row.put("double_column", "123.789"); + row.put("datetime_column", "2024-02-08T08:15:30.000Z"); + row.put("timestamp_column", "2024-02-08T08:15:30.000Z"); + row.put("time_column", "29730000000"); + row.put("year_column", "2022"); + row.put("char_column", "Y2hhcjEK"); + row.put("tinyblob_column", "74696e79626c6f625f646174615f31"); + row.put("tinytext_column", "dGlueXRleHRfZGF0YV8xCg=="); + row.put("blob_column", "626c6f625f646174615f31"); + row.put("mediumblob_column", "6d656469756d626c6f625f646174615f31"); + row.put("mediumtext_column", "bWVkaXVtdGV4dF9kYXRhXzE="); + row.put("longblob_column", "6c6f6e67626c6f625f646174615f31"); + row.put("longtext_column", "bG9uZ3RleHRfZGF0YV8x"); + row.put("enum_column", "2"); + row.put("bool_column", 0); + row.put("other_bool_column", "1"); + row.put("binary_column", "62696e6172795f31"); + row.put("varbinary_column", "76617262696e6172795f646174615f31"); + row.put("bit_column", "102"); + events.add(row); + + row.put("varchar_column", "value2"); + row.put("tinyint_column", "5"); + row.put("text_column", "dGV4dF9kYXRhXzIK"); + row.put("date_column", "2024-02-09T00:00:00.000Z"); + row.put("smallint_column", "25"); + row.put("mediumint_column", "500"); + row.put("int_column", "25000"); + row.put("bigint_column", "987654"); + row.put("float_column", "12.34"); + row.put("double_column", "56.789"); + row.put("datetime_column", "2024-02-09T15:30:45.000Z"); + row.put("timestamp_column", "2024-02-09T15:30:45.000Z"); + row.put("time_column", "55845000000"); + row.put("year_column", "2023"); + row.put("char_column", "Y2hhcjIK"); + row.put("tinyblob_column", "74696e79626c6f625f646174615f32"); + row.put("tinytext_column", "dGlueXRleHRfZGF0YV8yCg=="); + row.put("blob_column", "626c6f625f646174615f32"); + row.put("mediumblob_column", "6d656469756d626c6f625f646174615f32"); + row.put("mediumtext_column", "bWVkaXVtdGV4dF9kYXRhXzI="); + row.put("longblob_column", "6c6f6e67626c6f625f646174615f32"); + row.put("longtext_column", "bG9uZ3RleHRfZGF0YV8y"); + row.put("enum_column", "3"); + row.put("bool_column", 1); + row.put("other_bool_column", "0"); + row.put("binary_column", "62696e6172795f32"); + row.put("varbinary_column", "76617262696e6172795f646174615f32"); + row.put("bit_column", "25"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column" + + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" + + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" + + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column, " + + " longblob_column, longtext_column, enum_column, bool_column, other_bool_column, binary_column" + + ", varbinary_column, bit_column from AllDatatypeColumns")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumns2TableBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", 10); + row.put("text_column", "text1"); + row.put("date_column", "2024-02-08"); + row.put("smallint_column", 50); + row.put("mediumint_column", 1000); + row.put("int_column", 50000); + row.put("bigint_column", 987654321); + row.put("float_column", 45.67); + row.put("double_column", 123.789); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "29730000000"); + row.put("year_column", "2022"); + row.put("char_column", "char_1"); + // Source column value: 74696e79626c6f625f646174615f31 ( in BYTES, "tinyblob_data_1" in STRING) + // results in dGlueWJsb2JfZGF0YV8x base64 encoded string + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8x"); + row.put("tinytext_column", "tinytext_data_1"); + row.put("blob_column", "YmxvYl9kYXRhXzE="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzE="); + row.put("mediumtext_column", "mediumtext_data_1"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8x"); + row.put("longtext_column", "longtext_data_1"); + row.put("enum_column", "2"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "AQI="); + events.add(row); + + row.put("varchar_column", "value2"); + row.put("tinyint_column", 5); + row.put("text_column", "text2"); + row.put("date_column", "2024-02-09"); + row.put("smallint_column", 25); + row.put("mediumint_column", 500); + row.put("int_column", 25000); + row.put("bigint_column", 987654); + row.put("float_column", 12.34); + row.put("double_column", 56.789); + row.put("datetime_column", "2024-02-09T15:30:45Z"); + row.put("timestamp_column", "2024-02-09T15:30:45Z"); + row.put("time_column", "55845000000"); + row.put("year_column", "2023"); + row.put("char_column", "char_2"); + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8y"); + row.put("tinytext_column", "tinytext_data_2"); + row.put("blob_column", "YmxvYl9kYXRhXzI="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzI="); + row.put("mediumtext_column", "mediumtext_data_2"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8y"); + row.put("longtext_column", "longtext_data_2"); + row.put("enum_column", "3"); + row.put("bool_column", true); + row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); + row.put("bit_column", "JQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column" + + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" + + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" + + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column, " + + " longblob_column, longtext_column, enum_column, bool_column, binary_column" + + ", varbinary_column, bit_column from AllDatatypeColumns2")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumns2TableCdcContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", 15); + row.put("text_column", "text1"); + row.put("date_column", "2024-02-08"); + row.put("smallint_column", 50); + row.put("mediumint_column", 1000); + row.put("int_column", 50000); + row.put("bigint_column", 987654321); + row.put("float_column", 45.67); + row.put("double_column", 123.789); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "29730000000"); + row.put("year_column", "2022"); + row.put("char_column", "char_1"); + // Source column value: 74696e79626c6f625f646174615f31 ( in BYTES, "tinyblob_data_1" in STRING) + // results in dGlueWJsb2JfZGF0YV8x base64 encoded string + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8x"); + row.put("tinytext_column", "tinytext_data_1"); + row.put("blob_column", "YmxvYl9kYXRhXzE="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzE="); + row.put("mediumtext_column", "mediumtext_data_1"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8x"); + row.put("longtext_column", "longtext_data_1"); + row.put("enum_column", "2"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "AQI="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column" + + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" + + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" + + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column, " + + " longblob_column, longtext_column, enum_column, bool_column, binary_column" + + ", varbinary_column, bit_column from AllDatatypeColumns2")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertCategoryTableBackfillContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("category_id", 1); + row1.put("full_name", "xyz"); + + Map row2 = new HashMap<>(); + row2.put("category_id", 2); + row2.put("full_name", "abc"); + + events.add(row1); + events.add(row2); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select category_id, full_name from Category")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertCategoryTableCdcContents() { + List> events = new ArrayList<>(); + + Map row1 = new HashMap<>(); + row1.put("category_id", 2); + row1.put("full_name", "abc1"); + + Map row2 = new HashMap<>(); + row2.put("category_id", 3); + row2.put("full_name", "def"); + + Map row3 = new HashMap<>(); + row3.put("category_id", 4); + row3.put("full_name", "ghi"); + + events.add(row1); + events.add(row2); + events.add(row3); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery("select category_id, full_name from Category")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + private void assertAuthorsTable() { List> events = new ArrayList<>(); diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns.jsonl b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns.jsonl new file mode 100644 index 0000000000..791c47b7c7 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns.jsonl @@ -0,0 +1,2 @@ +{"uuid":"5deaf70b-2f02-486c-831a-b92400000000","read_timestamp":"2024-02-07T07:03:46.226Z","source_timestamp":"2024-02-07T07:02:44.000Z","object":"it_AllDatatypeColumns","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"0df5741bfd9144e7b58ce8cd018728d345ed6acd","sort_keys":[1707289364000,"mysql-bin.000029",9610730],"source_metadata":{"table":"AllDatatypeColumns","database":"it","primary_keys":["varchar_column"],"log_file":"mysql-bin.000029","log_position":9610730,"change_type":"INSERT","is_deleted":false},"payload":{"varchar_column":"value1","tinyint_column":10,"text_column":"746578745f646174615f310a","date_column":"2024-02-08T00:00:00.000Z","smallint_column":50,"mediumint_column":1000,"int_column":50000,"bigint_column":987654321,"float_column":45.67,"double_column":123.789,"decimal_column":456.12,"datetime_column":"2024-02-08T08:15:30.000Z","timestamp_column":"2024-02-08T08:15:30.000Z","time_column":29730000000,"year_column":2022,"char_column":"63686172310a","tinyblob_column":"74696e79626c6f625f646174615f31","tinytext_column":"74696e79746578745f646174615f310a","blob_column":"626c6f625f646174615f31","mediumblob_column":"6d656469756d626c6f625f646174615f31","mediumtext_column":"6d656469756d746578745f646174615f31","longblob_column":"6c6f6e67626c6f625f646174615f31","longtext_column":"6c6f6e67746578745f646174615f31","enum_column":"2","bool_column":0,"other_bool_column":1,"binary_column":"62696e6172795f31","varbinary_column":"76617262696e6172795f646174615f31","bit_column":102}} +{"uuid":"5deaf70b-2f02-486c-831a-b92400000001","read_timestamp":"2024-02-07T07:03:46.226Z","source_timestamp":"2024-02-07T07:02:55.000Z","object":"it_AllDatatypeColumns","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"0df5741bfd9144e7b58ce8cd018728d345ed6acd","sort_keys":[1707289375000,"mysql-bin.000029",9611373],"source_metadata":{"table":"AllDatatypeColumns","database":"it","primary_keys":["varchar_column"],"log_file":"mysql-bin.000029","log_position":9611373,"change_type":"INSERT","is_deleted":false},"payload":{"varchar_column":"value2","tinyint_column":5,"text_column":"746578745f646174615f320a","date_column":"2024-02-09T00:00:00.000Z","smallint_column":25,"mediumint_column":500,"int_column":25000,"bigint_column":987654,"float_column":12.34,"double_column":56.789,"decimal_column":123.45,"datetime_column":"2024-02-09T15:30:45.000Z","timestamp_column":"2024-02-09T15:30:45.000Z","time_column":55845000000,"year_column":2023,"char_column":"63686172320a","tinyblob_column":"74696e79626c6f625f646174615f32","tinytext_column":"74696e79746578745f646174615f320a","blob_column":"626c6f625f646174615f32","mediumblob_column":"6d656469756d626c6f625f646174615f32","mediumtext_column":"6d656469756d746578745f646174615f32","longblob_column":"6c6f6e67626c6f625f646174615f32","longtext_column":"6c6f6e67746578745f646174615f32","enum_column":"3","bool_column":1,"other_bool_column":0,"binary_column":"62696e6172795f32","varbinary_column":"76617262696e6172795f646174615f32","bit_column":25}} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns2.jsonl b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns2.jsonl new file mode 100644 index 0000000000..ee35c320b7 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns2.jsonl @@ -0,0 +1,2 @@ +{"uuid":"2a646066-40f2-44b7-bec0-61d100000000","read_timestamp":"2024-02-08T09:32:05.695Z","source_timestamp":"2024-02-08T09:30:59.000Z","object":"it_AllDatatypeColumns2","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"20f68e0ee69f7b864b730a1273b6f1b698ab4c28","sort_keys":[1707384659000,"mysql-bin.000029",9615213],"source_metadata":{"table":"AllDatatypeColumns2","database":"it","primary_keys":["varchar_column"],"log_file":"mysql-bin.000029","log_position":9615213,"change_type":"INSERT","is_deleted":false},"payload":{"varchar_column":"value1","tinyint_column":10,"text_column":"text1","date_column":"2024-02-08T00:00:00.000Z","smallint_column":50,"mediumint_column":1000,"int_column":50000,"bigint_column":987654321,"float_column":45.67,"double_column":123.789,"decimal_column":456.12,"datetime_column":"2024-02-08T08:15:30.000Z","timestamp_column":"2024-02-08T08:15:30.000Z","time_column":29730000000,"year_column":2022,"char_column":"char_1","tinyblob_column":"74696e79626c6f625f646174615f31","tinytext_column":"tinytext_data_1","blob_column":"626c6f625f646174615f31","mediumblob_column":"6d656469756d626c6f625f646174615f31","mediumtext_column":"mediumtext_data_1","longblob_column":"6c6f6e67626c6f625f646174615f31","longtext_column":"longtext_data_1","enum_column":"2","bool_column":0,"binary_column":"62696e6172795f646174615f3100000000000000","varbinary_column":"76617262696e6172795f646174615f31","bit_column":102}} +{"uuid":"2a646066-40f2-44b7-bec0-61d100000001","read_timestamp":"2024-02-08T09:32:05.695Z","source_timestamp":"2024-02-08T09:30:59.000Z","object":"it_AllDatatypeColumns2","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"20f68e0ee69f7b864b730a1273b6f1b698ab4c28","sort_keys":[1707384659000,"mysql-bin.000029",9615856],"source_metadata":{"table":"AllDatatypeColumns2","database":"it","primary_keys":["varchar_column"],"log_file":"mysql-bin.000029","log_position":9615856,"change_type":"INSERT","is_deleted":false},"payload":{"varchar_column":"value2","tinyint_column":5,"text_column":"text2","date_column":"2024-02-09T00:00:00.000Z","smallint_column":25,"mediumint_column":500,"int_column":25000,"bigint_column":987654,"float_column":12.34,"double_column":56.789,"decimal_column":123.45,"datetime_column":"2024-02-09T15:30:45.000Z","timestamp_column":"2024-02-09T15:30:45.000Z","time_column":55845000000,"year_column":2023,"char_column":"char_2","tinyblob_column":"74696e79626c6f625f646174615f32","tinytext_column":"tinytext_data_2","blob_column":"626c6f625f646174615f32","mediumblob_column":"6d656469756d626c6f625f646174615f32","mediumtext_column":"mediumtext_data_2","longblob_column":"6c6f6e67626c6f625f646174615f32","longtext_column":"longtext_data_2","enum_column":"3","bool_column":1,"binary_column":"62696e6172795f646174615f3200000000000000","varbinary_column":"76617262696e6172795f646174615f32","bit_column":25}} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-Category.jsonl b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-Category.jsonl new file mode 100644 index 0000000000..babd32f55b --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-Category.jsonl @@ -0,0 +1,2 @@ +{"uuid":"a49f1ab1-e39e-4f82-a1af-dd3f00000000","read_timestamp":"2024-02-06T12:27:45.602Z","source_timestamp":"2024-02-06T12:27:45.000Z","object":"it_Category","read_method":"mysql-backfill-fulldump","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"e38807ece1eb1660b3baaafbe2ffb849c6c3023a","sort_keys":[1707222465000,"",0],"source_metadata":{"table":"Category","database":"it","primary_keys":["category_id"],"log_file":"","log_position":0,"change_type":"INSERT","is_deleted":false},"payload":{"category_id":1,"name":"xyz","last_update":"2024-02-06T12:19:37.000Z"}} +{"uuid":"a49f1ab1-e39e-4f82-a1af-dd3f00000001","read_timestamp":"2024-02-06T12:27:45.602Z","source_timestamp":"2024-02-06T12:27:45.000Z","object":"it_Category","read_method":"mysql-backfill-fulldump","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"e38807ece1eb1660b3baaafbe2ffb849c6c3023a","sort_keys":[1707222465000,"",0],"source_metadata":{"table":"Category","database":"it","primary_keys":["category_id"],"log_file":"","log_position":0,"change_type":"INSERT","is_deleted":false},"payload":{"category_id":2,"name":"abc","last_update":"2024-02-06T12:19:47.000Z"}} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl new file mode 100644 index 0000000000..e69de29bb2 diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns2.jsonl b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns2.jsonl new file mode 100644 index 0000000000..86ec409f68 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns2.jsonl @@ -0,0 +1,2 @@ +{"uuid":"631f1c50-33fc-46ea-aa50-61a900000000","read_timestamp":"2024-02-09T05:09:12.430Z","source_timestamp":"2024-02-09T05:08:13.000Z","object":"it_AllDatatypeColumns2","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"20f68e0ee69f7b864b730a1273b6f1b698ab4c28","sort_keys":[1707455293000,"mysql-bin.000029",9616753],"source_metadata":{"table":"AllDatatypeColumns2","database":"it","primary_keys":["varchar_column"],"log_file":"mysql-bin.000029","log_position":9616753,"change_type":"UPDATE-INSERT","is_deleted":false},"payload":{"varchar_column":"value1","tinyint_column":15,"text_column":"text1","date_column":"2024-02-08T00:00:00.000Z","smallint_column":50,"mediumint_column":1000,"int_column":50000,"bigint_column":987654321,"float_column":45.67,"double_column":123.789,"decimal_column":456.12,"datetime_column":"2024-02-08T08:15:30.000Z","timestamp_column":"2024-02-08T08:15:30.000Z","time_column":29730000000,"year_column":2022,"char_column":"char_1","tinyblob_column":"74696e79626c6f625f646174615f31","tinytext_column":"tinytext_data_1","blob_column":"626c6f625f646174615f31","mediumblob_column":"6d656469756d626c6f625f646174615f31","mediumtext_column":"mediumtext_data_1","longblob_column":"6c6f6e67626c6f625f646174615f31","longtext_column":"longtext_data_1","enum_column":"2","bool_column":0,"binary_column":"62696e6172795f646174615f3100000000000000","varbinary_column":"76617262696e6172795f646174615f31","bit_column":102}} +{"uuid":"631f1c50-33fc-46ea-aa50-61a900000001","read_timestamp":"2024-02-09T05:09:12.430Z","source_timestamp":"2024-02-09T05:08:42.000Z","object":"it_AllDatatypeColumns2","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"20f68e0ee69f7b864b730a1273b6f1b698ab4c28","sort_keys":[1707455322000,"mysql-bin.000029",9617388],"source_metadata":{"table":"AllDatatypeColumns2","database":"it","primary_keys":["varchar_column"],"log_file":"mysql-bin.000029","log_position":9617388,"change_type":"DELETE","is_deleted":true},"payload":{"varchar_column":"value2","tinyint_column":5,"text_column":"text2","date_column":"2024-02-09T00:00:00.000Z","smallint_column":25,"mediumint_column":500,"int_column":25000,"bigint_column":987654,"float_column":12.34,"double_column":56.789,"decimal_column":123.45,"datetime_column":"2024-02-09T15:30:45.000Z","timestamp_column":"2024-02-09T15:30:45.000Z","time_column":55845000000,"year_column":2023,"char_column":"char_2","tinyblob_column":"74696e79626c6f625f646174615f32","tinytext_column":"tinytext_data_2","blob_column":"626c6f625f646174615f32","mediumblob_column":"6d656469756d626c6f625f646174615f32","mediumtext_column":"mediumtext_data_2","longblob_column":"6c6f6e67626c6f625f646174615f32","longtext_column":"longtext_data_2","enum_column":"3","bool_column":1,"binary_column":"62696e6172795f646174615f3200000000000000","varbinary_column":"76617262696e6172795f646174615f32","bit_column":25}} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-Category.jsonl b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-Category.jsonl new file mode 100644 index 0000000000..7825ca6e74 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-Category.jsonl @@ -0,0 +1,4 @@ +{"uuid":"05111f37-1fcc-48b2-99bd-8f1c00000000","read_timestamp":"2024-02-07T08:11:50.054Z","source_timestamp":"2024-02-07T08:10:55.000Z","object":"it_Category","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"e38807ece1eb1660b3baaafbe2ffb849c6c3023a","sort_keys":[1707293455000,"mysql-bin.000029",9612272],"source_metadata":{"table":"Category","database":"it","primary_keys":["category_id"],"log_file":"mysql-bin.000029","log_position":9612272,"change_type":"DELETE","is_deleted":true},"payload":{"category_id":1,"name":"xyz","last_update":"2024-02-06T12:19:37.000Z"}} +{"uuid":"3faa3ec0-cedc-4a93-b9ce-50e000000000","read_timestamp":"2024-02-07T08:13:00.421Z","source_timestamp":"2024-02-07T08:12:32.000Z","object":"it_Category","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"e38807ece1eb1660b3baaafbe2ffb849c6c3023a","sort_keys":[1707293552000,"mysql-bin.000029",9612570],"source_metadata":{"table":"Category","database":"it","primary_keys":["category_id"],"log_file":"mysql-bin.000029","log_position":9612570,"change_type":"INSERT","is_deleted":false},"payload":{"category_id":4,"name":"ghi","last_update":"2024-02-07T08:12:32.000Z"}} +{"uuid":"359901f6-96bc-4f68-9f6c-7d4500000001","read_timestamp":"2024-02-07T08:10:39.444Z","source_timestamp":"2024-02-07T08:10:35.000Z","object":"it_Category","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"e38807ece1eb1660b3baaafbe2ffb849c6c3023a","sort_keys":[1707293435000,"mysql-bin.000029",9611982],"source_metadata":{"table":"Category","database":"it","primary_keys":["category_id"],"log_file":"mysql-bin.000029","log_position":9611982,"change_type":"INSERT","is_deleted":false},"payload":{"category_id":3,"name":"def","last_update":"2024-02-07T08:10:35.000Z"}} +{"uuid":"359901f6-96bc-4f68-9f6c-7d4500000000","read_timestamp":"2024-02-07T08:10:39.444Z","source_timestamp":"2024-02-07T08:10:15.000Z","object":"it_Category","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"e38807ece1eb1660b3baaafbe2ffb849c6c3023a","sort_keys":[1707293415000,"mysql-bin.000029",9611684],"source_metadata":{"table":"Category","database":"it","primary_keys":["category_id"],"log_file":"mysql-bin.000029","log_position":9611684,"change_type":"UPDATE-INSERT","is_deleted":false},"payload":{"category_id":2,"name":"abc1","last_update":"2024-02-06T12:19:47.000Z"}} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-schema.sql b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-schema.sql index 4869fff4ac..e4d1f0103a 100644 --- a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-schema.sql +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-schema.sql @@ -16,6 +16,79 @@ CREATE TABLE `Users` ( PRIMARY KEY (`id`) ); +CREATE TABLE `Category` ( + `category_id` tinyint NOT NULL, -- To: category_id INT64 + `name` varchar(25), -- To: full_name STRING(25) Column name renamed + `last_update` timestamp, -- To: Column dropped in spanner + PRIMARY KEY (`category_id`) +); + +-- Database postgres_test in shreya-mysql +CREATE TABLE `AllDatatypeColumns` ( + `varchar_column` varchar(20) NOT NULL, -- To: varchar_column BYTES(20) + `tinyint_column` tinyint, -- To: tinyint_column STRING(MAX) + `text_column` text, -- To: text_column BYTES(MAX) + `date_column` date, -- To: date_column STRING(MAX) + `smallint_column` smallint, -- To: smallint_column STRING(MAX) + `mediumint_column` mediumint, -- To: mediumint_column STRING(MAX) + `int_column` int, -- To: int_column STRING(MAX) + `bigint_column` bigint, -- To: bigint_column STRING(MAX) + `float_column` float(10,2), -- To: float_column STRING(MAX) + `double_column` double, -- To: double_column STRING(MAX) + `decimal_column` decimal(10,2), -- To: decimal_column STRING(MAX) + `datetime_column` datetime, -- To: datetime_column STRING(MAX) + `timestamp_column` timestamp, -- To: timestamp_column STRING(MAX) + `time_column` time, -- To: time_column STRING(MAX) + `year_column` year, -- To: year_column STRING(MAX) + `char_column` char(10), -- To: char_column BYTES(10) + `tinyblob_column` tinyblob, -- To: tinyblob_column STRING(MAX) + `tinytext_column` tinytext, -- To: tinytext_column BYTES(MAX) + `blob_column` blob, -- To: blob_column STRING(MAX) + `mediumblob_column` mediumblob, -- To: mediumblob_column STRING(MAX) + `mediumtext_column` mediumtext, -- To: mediumtext_column BYTES(MAX) + `longblob_column` longblob, -- To: longblob_column STRING(MAX) + `longtext_column` longtext, -- To: longtext_column BYTES(MAX) + `enum_column` enum('1','2','3'), -- To: enum_column STRING(MAX) + `bool_column` tinyint(1), -- To: bool_column INT64 + `other_bool_column` tinyint(1), -- To: other_bool_column STRING(MAX) + `binary_column` binary(20), -- To: binary_column STRING(MAX) + `varbinary_column` varbinary(20), -- To: varbinary_column STRING(MAX) + `bit_column` bit(7), -- To: bit_column STRING(MAX) + PRIMARY KEY (`varchar_column`) +); + +CREATE TABLE `AllDatatypeColumns2` ( + `varchar_column` varchar(20) NOT NULL, -- To: varchar_column BYTES(20) + `tinyint_column` tinyint, -- To: tinyint_column INT64 + `text_column` text, -- To: text_column STRING(MAX) + `date_column` date, -- To: date_column DATE + `smallint_column` smallint, -- To: smallint_column INT64 + `mediumint_column` mediumint, -- To: mediumint_column INT64 + `int_column` int, -- To: int_column INT64 + `bigint_column` bigint, -- To: bigint_column INT64 + `float_column` float(10,2), -- To: float_column FLOAT64 + `double_column` double, -- To: double_column FLOAT64 + `decimal_column` decimal(10,2), -- To: decimal_column NUMERIC + `datetime_column` datetime, -- To: datetime_column TIMESTAMP + `timestamp_column` timestamp, -- To: timestamp_column TIMESTAMP + `time_column` time, -- To: time_column STRING(MAX) + `year_column` year, -- To: year_column STRING(MAX) + `char_column` char(10), -- To: char_column STRING(10) + `tinyblob_column` tinyblob, -- To: tinyblob_column BYTES(MAX) + `tinytext_column` tinytext, -- To: tinytext_column STRING(MAX) + `blob_column` blob, -- To: blob_column BYTES(MAX) + `mediumblob_column` mediumblob, -- To: mediumblob_column BYTES(MAX) + `mediumtext_column` mediumtext, -- To: mediumtext_column STRING(MAX) + `longblob_column` longblob, -- To: longblob_column BYTES(MAX) + `longtext_column` longtext, -- To: longtext_column STRING(MAX) + `enum_column` enum('1','2','3'), -- To: enum_column STRING(MAX) + `bool_column` tinyint(1), -- To: bool_column BOOL + `binary_column` binary(20), -- To: binary_column BYTES(MAX) + `varbinary_column` varbinary(20), -- To: varbinary_column BYTES(MAX) + `bit_column` bit(7), -- To: bit_column BYTES(MAX) + PRIMARY KEY (`varchar_column`) +); + CREATE TABLE `Authors` ( `author_id` int NOT NULL, `name` varchar(200), diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/spanner-schema.sql index a637eb6f0b..8d457ef2f4 100644 --- a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/spanner-schema.sql +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/spanner-schema.sql @@ -1,9 +1,77 @@ -CREATE TABLE IF NOT EXISTS Movie ( - id INT64 NOT NULL, - name STRING(200), - actor NUMERIC, - startTime TIMESTAMP -) PRIMARY KEY (id); +CREATE TABLE AllDatatypeColumns ( + varchar_column STRING(20) NOT NULL, + tinyint_column STRING(MAX), + text_column BYTES(MAX), + date_column STRING(MAX), + smallint_column STRING(MAX), + mediumint_column STRING(MAX), + int_column STRING(MAX), + bigint_column STRING(MAX), + float_column STRING(MAX), + double_column STRING(MAX), + decimal_column STRING(MAX), + datetime_column STRING(MAX), + timestamp_column STRING(MAX), + time_column STRING(MAX), + year_column STRING(MAX), + char_column BYTES(10), + tinyblob_column STRING(MAX), + tinytext_column BYTES(MAX), + blob_column STRING(MAX), + mediumblob_column STRING(MAX), + mediumtext_column BYTES(MAX), + longblob_column STRING(MAX), + longtext_column BYTES(MAX), + enum_column STRING(MAX), + bool_column INT64, + other_bool_column STRING(MAX), + binary_column STRING(MAX), + varbinary_column STRING(MAX), + bit_column STRING(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE AllDatatypeColumns2 ( + varchar_column STRING(20) NOT NULL, + tinyint_column INT64, + text_column STRING(MAX), + date_column DATE, + smallint_column INT64, + mediumint_column INT64, + int_column INT64, + bigint_column INT64, + float_column FLOAT64, + double_column FLOAT64, + decimal_column NUMERIC, + datetime_column TIMESTAMP, + timestamp_column TIMESTAMP, + time_column STRING(MAX), + year_column STRING(MAX), + char_column STRING(10), + tinyblob_column BYTES(MAX), + tinytext_column STRING(MAX), + blob_column BYTES(MAX), + mediumblob_column BYTES(MAX), + mediumtext_column STRING(MAX), + longblob_column BYTES(MAX), + longtext_column STRING(MAX), + enum_column STRING(MAX), + bool_column BOOL, + binary_column BYTES(MAX), + varbinary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE Category ( + category_id INT64 NOT NULL, + full_name STRING(25), +) PRIMARY KEY(category_id); + +CREATE TABLE Movie ( + id INT64 NOT NULL, + name STRING(200), + actor NUMERIC, + startTime TIMESTAMP, +) PRIMARY KEY(id); CREATE TABLE IF NOT EXISTS Users ( id INT64 NOT NULL, @@ -37,4 +105,4 @@ CREATE TABLE IF NOT EXISTS Articles ( ) PRIMARY KEY (author_id, id), INTERLEAVE IN PARENT Authors; -CREATE INDEX author_id ON Articles (author_id); +CREATE INDEX author_id ON Articles (author_id); \ No newline at end of file From 559d864b8d911308be326bd2c1a12539b0ea7c2c Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Fri, 9 Feb 2024 16:13:01 +0530 Subject: [PATCH 02/11] added a cdc test case --- .../DataStreamToSpannerSimpleIT.java | 88 ++++++++++++++++++- .../mysql-cdc-AllDatatypeColumns.jsonl | 2 + 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java index 289b7fe275..ecb1134239 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java @@ -265,11 +265,52 @@ public void migrationTestWithAllDatatypeConversionsWithInsertsOnly() { .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); // Assert Conditions - // assertThatResult(result).meetsConditions(); + assertThatResult(result).meetsConditions(); assertAllDatatypeColumnsTableBackfillContents(); } + @Test + public void migrationTestWithAllDatatypeConversionsWithUpdatesAndDeletes() { + // Construct a ChainedConditionCheck with 4 stages. + // 1. Send initial wave of events + // 2. Wait on Spanner to have events + // 3. Send second wave of events + // 4. Wait on Spanner to merge second wave of events + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE4, + "backfill.jsonl", + "DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns.jsonl"), + SpannerRowsCheck.builder(spannerResourceManager, TABLE4) + .setMinRows(2) + .setMaxRows(2) + .build(), + uploadDataStreamFile( + jobInfo, + TABLE4, + "cdc1.jsonl", + "DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl"), + SpannerRowsCheck.builder(spannerResourceManager, TABLE4) + .setMinRows(1) + .setMaxRows(1) + .build())) + .build(); + + // Wait for conditions + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + // Assert Conditions + assertThatResult(result).meetsConditions(); + + assertAllDatatypeColumnsTableCdcContents(); + } + @Test public void migrationTestWithAllDatatypeMappingsWithInsertsOnly() { // Construct a ChainedConditionCheck with 4 stages. @@ -549,6 +590,51 @@ private void assertAllDatatypeColumnsTableBackfillContents() { .hasRecordsUnorderedCaseInsensitiveColumns(events); } + private void assertAllDatatypeColumnsTableCdcContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", "15"); + row.put("text_column", "dGV4dF9kYXRhXzEK"); + row.put("date_column", "2024-02-08T00:00:00.000Z"); + row.put("smallint_column", "50"); + row.put("mediumint_column", "1000"); + row.put("int_column", "50000"); + row.put("bigint_column", "987654321"); + row.put("float_column", "45.67"); + row.put("double_column", "123.789"); + row.put("datetime_column", "2024-02-08T08:15:30.000Z"); + row.put("timestamp_column", "2024-02-08T08:15:30.000Z"); + row.put("time_column", "29730000000"); + row.put("year_column", "2022"); + row.put("char_column", "Y2hhcjEK"); + row.put("tinyblob_column", "74696e79626c6f625f646174615f31"); + row.put("tinytext_column", "dGlueXRleHRfZGF0YV8xCg=="); + row.put("blob_column", "626c6f625f646174615f31"); + row.put("mediumblob_column", "6d656469756d626c6f625f646174615f31"); + row.put("mediumtext_column", "bWVkaXVtdGV4dF9kYXRhXzE="); + row.put("longblob_column", "6c6f6e67626c6f625f646174615f31"); + row.put("longtext_column", "bG9uZ3RleHRfZGF0YV8x"); + row.put("enum_column", "2"); + row.put("bool_column", 0); + row.put("other_bool_column", "1"); + row.put("binary_column", "62696e6172795f31"); + row.put("varbinary_column", "76617262696e6172795f646174615f31"); + row.put("bit_column", "102"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column" + + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" + + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" + + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column, " + + " longblob_column, longtext_column, enum_column, bool_column, other_bool_column, binary_column" + + ", varbinary_column, bit_column from AllDatatypeColumns")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + private void assertAllDatatypeColumns2TableBackfillContents() { List> events = new ArrayList<>(); diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl index e69de29bb2..8c80829e17 100644 --- a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl @@ -0,0 +1,2 @@ +{"uuid":"45c148ba-a57f-4882-879e-831700000001","read_timestamp":"2024-02-09T09:59:32.777Z","source_timestamp":"2024-02-09T09:59:19.000Z","object":"it_AllDatatypeColumns","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"0df5741bfd9144e7b58ce8cd018728d345ed6acd","sort_keys":[1707472759000,"mysql-bin.000029",9618920],"source_metadata":{"table":"AllDatatypeColumns","database":"it","primary_keys":["varchar_column"],"log_file":"mysql-bin.000029","log_position":9618920,"change_type":"DELETE","is_deleted":true},"payload":{"varchar_column":"value2","tinyint_column":5,"text_column":"Another sample text for the second row.","date_column":"2024-02-09T00:00:00.000Z","smallint_column":25,"mediumint_column":500,"int_column":25000,"bigint_column":987654,"float_column":12.34,"double_column":56.789,"decimal_column":123.45,"datetime_column":"2024-02-09T15:30:45.000Z","timestamp_column":"2024-02-09T15:30:45.000Z","time_column":55845000000,"year_column":2023,"char_column":"char2","tinyblob_column":"74696e79626c6f625f646174615f32","tinytext_column":"tinytext_data_2","blob_column":"626c6f625f646174615f32","mediumblob_column":"6d656469756d626c6f625f646174615f32","mediumtext_column":"mediumtext_data_2","longblob_column":"6c6f6e67626c6f625f646174615f32","longtext_column":"longtext_data_2","enum_column":"3","bool_column":1,"other_bool_column":0,"binary_column":"62696e6172795f646174615f3200000000000000","varbinary_column":"76617262696e6172795f646174615f32","bit_column":25}} +{"uuid":"45c148ba-a57f-4882-879e-831700000000","read_timestamp":"2024-02-09T09:59:32.777Z","source_timestamp":"2024-02-09T09:58:59.000Z","object":"it_AllDatatypeColumns","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"0df5741bfd9144e7b58ce8cd018728d345ed6acd","sort_keys":[1707472739000,"mysql-bin.000029",9618285],"source_metadata":{"table":"AllDatatypeColumns","database":"it","primary_keys":["varchar_column"],"log_file":"mysql-bin.000029","log_position":9618285,"change_type":"UPDATE-INSERT","is_deleted":false},"payload":{"varchar_column":"value1","tinyint_column":15,"text_column":"Sample text for the first row.","date_column":"2024-02-08T00:00:00.000Z","smallint_column":50,"mediumint_column":1000,"int_column":50000,"bigint_column":987654321,"float_column":45.67,"double_column":123.789,"decimal_column":456.12,"datetime_column":"2024-02-08T08:15:30.000Z","timestamp_column":"2024-02-08T08:15:30.000Z","time_column":29730000000,"year_column":2022,"char_column":"char1","tinyblob_column":"74696e79626c6f625f646174615f31","tinytext_column":"tinytext_data_1","blob_column":"626c6f625f646174615f31","mediumblob_column":"6d656469756d626c6f625f646174615f31","mediumtext_column":"mediumtext_data_1","longblob_column":"6c6f6e67626c6f625f646174615f31","longtext_column":"longtext_data_1","enum_column":"2","bool_column":0,"other_bool_column":1,"binary_column":"62696e6172795f646174615f3100000000000000","varbinary_column":"76617262696e6172795f646174615f31","bit_column":102}} \ No newline at end of file From 77ac7ae1e33876946798329f242018fcdeba2e5d Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Tue, 13 Feb 2024 07:49:15 +0530 Subject: [PATCH 03/11] fixing test case --- .../mysql-cdc-AllDatatypeColumns.jsonl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl index 8c80829e17..a6c835c373 100644 --- a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl @@ -1,2 +1,2 @@ -{"uuid":"45c148ba-a57f-4882-879e-831700000001","read_timestamp":"2024-02-09T09:59:32.777Z","source_timestamp":"2024-02-09T09:59:19.000Z","object":"it_AllDatatypeColumns","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"0df5741bfd9144e7b58ce8cd018728d345ed6acd","sort_keys":[1707472759000,"mysql-bin.000029",9618920],"source_metadata":{"table":"AllDatatypeColumns","database":"it","primary_keys":["varchar_column"],"log_file":"mysql-bin.000029","log_position":9618920,"change_type":"DELETE","is_deleted":true},"payload":{"varchar_column":"value2","tinyint_column":5,"text_column":"Another sample text for the second row.","date_column":"2024-02-09T00:00:00.000Z","smallint_column":25,"mediumint_column":500,"int_column":25000,"bigint_column":987654,"float_column":12.34,"double_column":56.789,"decimal_column":123.45,"datetime_column":"2024-02-09T15:30:45.000Z","timestamp_column":"2024-02-09T15:30:45.000Z","time_column":55845000000,"year_column":2023,"char_column":"char2","tinyblob_column":"74696e79626c6f625f646174615f32","tinytext_column":"tinytext_data_2","blob_column":"626c6f625f646174615f32","mediumblob_column":"6d656469756d626c6f625f646174615f32","mediumtext_column":"mediumtext_data_2","longblob_column":"6c6f6e67626c6f625f646174615f32","longtext_column":"longtext_data_2","enum_column":"3","bool_column":1,"other_bool_column":0,"binary_column":"62696e6172795f646174615f3200000000000000","varbinary_column":"76617262696e6172795f646174615f32","bit_column":25}} -{"uuid":"45c148ba-a57f-4882-879e-831700000000","read_timestamp":"2024-02-09T09:59:32.777Z","source_timestamp":"2024-02-09T09:58:59.000Z","object":"it_AllDatatypeColumns","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"0df5741bfd9144e7b58ce8cd018728d345ed6acd","sort_keys":[1707472739000,"mysql-bin.000029",9618285],"source_metadata":{"table":"AllDatatypeColumns","database":"it","primary_keys":["varchar_column"],"log_file":"mysql-bin.000029","log_position":9618285,"change_type":"UPDATE-INSERT","is_deleted":false},"payload":{"varchar_column":"value1","tinyint_column":15,"text_column":"Sample text for the first row.","date_column":"2024-02-08T00:00:00.000Z","smallint_column":50,"mediumint_column":1000,"int_column":50000,"bigint_column":987654321,"float_column":45.67,"double_column":123.789,"decimal_column":456.12,"datetime_column":"2024-02-08T08:15:30.000Z","timestamp_column":"2024-02-08T08:15:30.000Z","time_column":29730000000,"year_column":2022,"char_column":"char1","tinyblob_column":"74696e79626c6f625f646174615f31","tinytext_column":"tinytext_data_1","blob_column":"626c6f625f646174615f31","mediumblob_column":"6d656469756d626c6f625f646174615f31","mediumtext_column":"mediumtext_data_1","longblob_column":"6c6f6e67626c6f625f646174615f31","longtext_column":"longtext_data_1","enum_column":"2","bool_column":0,"other_bool_column":1,"binary_column":"62696e6172795f646174615f3100000000000000","varbinary_column":"76617262696e6172795f646174615f31","bit_column":102}} \ No newline at end of file +{"uuid":"45c148ba-a57f-4882-879e-831700000001","read_timestamp":"2024-02-09T09:59:32.777Z","source_timestamp":"2024-02-09T09:59:19.000Z","object":"it_AllDatatypeColumns","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"0df5741bfd9144e7b58ce8cd018728d345ed6acd","sort_keys":[1707472759000,"mysql-bin.000029",9618920],"source_metadata":{"table":"AllDatatypeColumns","database":"it","primary_keys":["varchar_column"],"log_file":"mysql-bin.000029","log_position":9618920,"change_type":"DELETE","is_deleted":true},"payload":{"varchar_column":"value2","tinyint_column":5,"text_column":"746578745f646174615f320a","date_column":"2024-02-09T00:00:00.000Z","smallint_column":25,"mediumint_column":500,"int_column":25000,"bigint_column":987654,"float_column":12.34,"double_column":56.789,"decimal_column":123.45,"datetime_column":"2024-02-09T15:30:45.000Z","timestamp_column":"2024-02-09T15:30:45.000Z","time_column":55845000000,"year_column":2023,"char_column":"63686172320a","tinyblob_column":"74696e79626c6f625f646174615f32","tinytext_column":"74696e79746578745f646174615f320a","blob_column":"626c6f625f646174615f32","mediumblob_column":"6d656469756d626c6f625f646174615f32","mediumtext_column":"6d656469756d746578745f646174615f32","longblob_column":"6c6f6e67626c6f625f646174615f32","longtext_column":"6c6f6e67746578745f646174615f32","enum_column":"3","bool_column":1,"other_bool_column":0,"binary_column":"62696e6172795f32","varbinary_column":"76617262696e6172795f646174615f32","bit_column":25}} +{"uuid":"45c148ba-a57f-4882-879e-831700000000","read_timestamp":"2024-02-09T09:59:32.777Z","source_timestamp":"2024-02-09T09:58:59.000Z","object":"it_AllDatatypeColumns","read_method":"mysql-cdc-binlog","stream_name":"projects/545418958905/locations/us-central1/streams/int-test-shreya","schema_key":"0df5741bfd9144e7b58ce8cd018728d345ed6acd","sort_keys":[1707472739000,"mysql-bin.000029",9618285],"source_metadata":{"table":"AllDatatypeColumns","database":"it","primary_keys":["varchar_column"],"log_file":"mysql-bin.000029","log_position":9618285,"change_type":"UPDATE-INSERT","is_deleted":false},"payload":{"varchar_column":"value1","tinyint_column":15,"text_column":"746578745f646174615f310a","date_column":"2024-02-08T00:00:00.000Z","smallint_column":50,"mediumint_column":1000,"int_column":50000,"bigint_column":987654321,"float_column":45.67,"double_column":123.789,"decimal_column":456.12,"datetime_column":"2024-02-08T08:15:30.000Z","timestamp_column":"2024-02-08T08:15:30.000Z","time_column":29730000000,"year_column":2022,"char_column":"63686172310a","tinyblob_column":"74696e79626c6f625f646174615f31","tinytext_column":"74696e79746578745f646174615f310a","blob_column":"626c6f625f646174615f31","mediumblob_column":"6d656469756d626c6f625f646174615f31","mediumtext_column":"6d656469756d746578745f646174615f31","longblob_column":"6c6f6e67626c6f625f646174615f31","longtext_column":"6c6f6e67746578745f646174615f31","enum_column":"2","bool_column":0,"other_bool_column":1,"binary_column":"62696e6172795f31","varbinary_column":"76617262696e6172795f646174615f31","bit_column":102}} \ No newline at end of file From a78abd15b5536b1ab21a160f50f695948853521c Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Tue, 20 Feb 2024 10:44:15 +0530 Subject: [PATCH 04/11] merged test cases --- .../DataStreamToSpannerSimpleIT.java | 71 ++++++++----------- 1 file changed, 29 insertions(+), 42 deletions(-) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java index ecb1134239..6a763bbe4a 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java @@ -241,7 +241,7 @@ public void interleavedAndFKAndIndexTest() { } @Test - public void migrationTestWithAllDatatypeConversionsWithInsertsOnly() { + public void migrationTestWithAllDatatypeConversions() { // Construct a ChainedConditionCheck with 4 stages. // 1. Send initial wave of events // 2. Wait on Spanner to have events @@ -268,27 +268,10 @@ public void migrationTestWithAllDatatypeConversionsWithInsertsOnly() { assertThatResult(result).meetsConditions(); assertAllDatatypeColumnsTableBackfillContents(); - } - @Test - public void migrationTestWithAllDatatypeConversionsWithUpdatesAndDeletes() { - // Construct a ChainedConditionCheck with 4 stages. - // 1. Send initial wave of events - // 2. Wait on Spanner to have events - // 3. Send second wave of events - // 4. Wait on Spanner to merge second wave of events - ChainedConditionCheck conditionCheck = + conditionCheck = ChainedConditionCheck.builder( List.of( - uploadDataStreamFile( - jobInfo, - TABLE4, - "backfill.jsonl", - "DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns.jsonl"), - SpannerRowsCheck.builder(spannerResourceManager, TABLE4) - .setMinRows(2) - .setMaxRows(2) - .build(), uploadDataStreamFile( jobInfo, TABLE4, @@ -300,8 +283,7 @@ public void migrationTestWithAllDatatypeConversionsWithUpdatesAndDeletes() { .build())) .build(); - // Wait for conditions - PipelineOperator.Result result = + result = pipelineOperator() .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); @@ -312,7 +294,7 @@ public void migrationTestWithAllDatatypeConversionsWithUpdatesAndDeletes() { } @Test - public void migrationTestWithAllDatatypeMappingsWithInsertsOnly() { + public void migrationTestWithAllDatatypeMappings() { // Construct a ChainedConditionCheck with 4 stages. // 1. Send initial wave of events // 2. Wait on Spanner to have events @@ -339,27 +321,10 @@ public void migrationTestWithAllDatatypeMappingsWithInsertsOnly() { assertThatResult(result).meetsConditions(); assertAllDatatypeColumns2TableBackfillContents(); - } - @Test - public void migrationTestWithAllDatatypeMappingsWithUpdatesAndDeletes() { - // Construct a ChainedConditionCheck with 4 stages. - // 1. Send initial wave of events - // 2. Wait on Spanner to have events - // 3. Send second wave of events - // 4. Wait on Spanner to merge second wave of events - ChainedConditionCheck conditionCheck = + conditionCheck = ChainedConditionCheck.builder( List.of( - uploadDataStreamFile( - jobInfo, - TABLE5, - "backfill.jsonl", - "DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns2.jsonl"), - SpannerRowsCheck.builder(spannerResourceManager, TABLE5) - .setMinRows(2) - .setMaxRows(2) - .build(), uploadDataStreamFile( jobInfo, TABLE5, @@ -371,8 +336,7 @@ public void migrationTestWithAllDatatypeMappingsWithUpdatesAndDeletes() { .build())) .build(); - // Wait for conditions - PipelineOperator.Result result = + result = pipelineOperator() .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); @@ -410,6 +374,29 @@ public void migrationTestWithRenameAndDropColumnWithInsertsOnly() { assertThatResult(result).meetsConditions(); assertCategoryTableBackfillContents(); + + conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE3, + "cdc1.jsonl", + "DataStreamToSpannerSimpleIT/mysql-cdc-Category.jsonl"), + SpannerRowsCheck.builder(spannerResourceManager, TABLE3) + .setMinRows(3) + .setMaxRows(3) + .build())) + .build(); + + result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + // Assert Conditions + assertThatResult(result).meetsConditions(); + + assertCategoryTableCdcContents(); } @Test From 304682df6c1630215685956c56697be2fbaafb4e Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Tue, 20 Feb 2024 11:53:06 +0530 Subject: [PATCH 05/11] addressing coments --- .../DataStreamToSpannerSimpleIT.java | 49 ++----------------- 1 file changed, 3 insertions(+), 46 deletions(-) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java index 6a763bbe4a..efebb8f2f6 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java @@ -347,7 +347,7 @@ public void migrationTestWithAllDatatypeMappings() { } @Test - public void migrationTestWithRenameAndDropColumnWithInsertsOnly() { + public void migrationTestWithRenameAndDropColumn() { // Construct a ChainedConditionCheck with 4 stages. // 1. Send initial wave of events // 2. Wait on Spanner to have events @@ -399,47 +399,6 @@ public void migrationTestWithRenameAndDropColumnWithInsertsOnly() { assertCategoryTableCdcContents(); } - @Test - public void migrationTestWithRenameAndDropColumnWithUpdatesAndDeletes() { - // Construct a ChainedConditionCheck with 4 stages. - // 1. Send initial wave of events - // 2. Wait on Spanner to have events - // 3. Send second wave of events - // 4. Wait on Spanner to merge second wave of events - ChainedConditionCheck conditionCheck = - ChainedConditionCheck.builder( - List.of( - uploadDataStreamFile( - jobInfo, - TABLE3, - "backfill.jsonl", - "DataStreamToSpannerSimpleIT/mysql-backfill-Category.jsonl"), - SpannerRowsCheck.builder(spannerResourceManager, TABLE3) - .setMinRows(2) - .setMaxRows(2) - .build(), - uploadDataStreamFile( - jobInfo, - TABLE3, - "cdc1.jsonl", - "DataStreamToSpannerSimpleIT/mysql-cdc-Category.jsonl"), - SpannerRowsCheck.builder(spannerResourceManager, TABLE3) - .setMinRows(3) - .setMaxRows(3) - .build())) - .build(); - - // Wait for conditions - PipelineOperator.Result result = - pipelineOperator() - .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); - - // Assert Conditions - assertThatResult(result).meetsConditions(); - - assertCategoryTableCdcContents(); - } - private void assertUsersTableContents() { List> events = new ArrayList<>(); @@ -757,8 +716,7 @@ private void assertCategoryTableBackfillContents() { events.add(row1); events.add(row2); - SpannerAsserts.assertThatStructs( - spannerResourceManager.runQuery("select category_id, full_name from Category")) + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Category")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } @@ -781,8 +739,7 @@ private void assertCategoryTableCdcContents() { events.add(row2); events.add(row3); - SpannerAsserts.assertThatStructs( - spannerResourceManager.runQuery("select category_id, full_name from Category")) + SpannerAsserts.assertThatStructs(spannerResourceManager.runQuery("select * from Category")) .hasRecordsUnorderedCaseInsensitiveColumns(events); } From 60cd56ec878e4ada6727da128e307b001df3aea9 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Mon, 11 Mar 2024 12:06:56 +0530 Subject: [PATCH 06/11] addressing comment --- .../teleport/v2/templates/DataStreamToSpannerSimpleIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java index efebb8f2f6..f19a4d8677 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java @@ -241,7 +241,7 @@ public void interleavedAndFKAndIndexTest() { } @Test - public void migrationTestWithAllDatatypeConversions() { + public void migrationTestWithAllDatatypeDefaultMapping() { // Construct a ChainedConditionCheck with 4 stages. // 1. Send initial wave of events // 2. Wait on Spanner to have events @@ -294,7 +294,7 @@ public void migrationTestWithAllDatatypeConversions() { } @Test - public void migrationTestWithAllDatatypeMappings() { + public void migrationTestWithAllDatatypeConversionMapping() { // Construct a ChainedConditionCheck with 4 stages. // 1. Send initial wave of events // 2. Wait on Spanner to have events From c430723c8870697a5189d0fb5bd9ae9779f3ca30 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Mon, 11 Mar 2024 16:29:31 +0530 Subject: [PATCH 07/11] test commit --- .../cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java index f19a4d8677..6331aab8c6 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java @@ -658,7 +658,6 @@ private void assertAllDatatypeColumns2TableBackfillContents() { private void assertAllDatatypeColumns2TableCdcContents() { List> events = new ArrayList<>(); - Map row = new HashMap<>(); row.put("varchar_column", "value1"); row.put("tinyint_column", 15); From 1889ed360376b536b8c2517da29bc8b1871b503d Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Wed, 20 Mar 2024 18:00:48 +0530 Subject: [PATCH 08/11] created a new datatype test class --- .../DataStreamToSpannerDatatypeIT.java | 454 ++++ .../DataStreamToSpannerSimpleIT.java | 348 --- .../mysql-backfill-AllDatatypeColumns.jsonl | 0 .../mysql-backfill-AllDatatypeColumns2.jsonl | 0 .../mysql-cdc-AllDatatypeColumns.jsonl | 0 .../mysql-cdc-AllDatatypeColumns2.jsonl | 0 .../mysql-schema.sql | 65 + .../mysql-session.json | 2059 +++++++++++++++++ .../spanner-schema.sql | 62 + .../mysql-schema.sql | 66 - .../mysql-session.json | 834 ++++--- .../spanner-schema.sql | 63 - 12 files changed, 3155 insertions(+), 796 deletions(-) create mode 100644 v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerDatatypeIT.java rename v2/datastream-to-spanner/src/test/resources/{DataStreamToSpannerSimpleIT => DataStreamToSpannerDatatypeIT}/mysql-backfill-AllDatatypeColumns.jsonl (100%) rename v2/datastream-to-spanner/src/test/resources/{DataStreamToSpannerSimpleIT => DataStreamToSpannerDatatypeIT}/mysql-backfill-AllDatatypeColumns2.jsonl (100%) rename v2/datastream-to-spanner/src/test/resources/{DataStreamToSpannerSimpleIT => DataStreamToSpannerDatatypeIT}/mysql-cdc-AllDatatypeColumns.jsonl (100%) rename v2/datastream-to-spanner/src/test/resources/{DataStreamToSpannerSimpleIT => DataStreamToSpannerDatatypeIT}/mysql-cdc-AllDatatypeColumns2.jsonl (100%) create mode 100644 v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-schema.sql create mode 100644 v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-session.json create mode 100644 v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/spanner-schema.sql diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerDatatypeIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerDatatypeIT.java new file mode 100644 index 0000000000..e715b95c7f --- /dev/null +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerDatatypeIT.java @@ -0,0 +1,454 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates; + +import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatResult; + +import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; +import com.google.cloud.teleport.metadata.TemplateIntegrationTest; +import java.io.IOException; +import java.time.Duration; +import java.util.*; +import org.apache.beam.it.common.PipelineLauncher; +import org.apache.beam.it.common.PipelineOperator; +import org.apache.beam.it.common.utils.ResourceManagerUtils; +import org.apache.beam.it.conditions.ChainedConditionCheck; +import org.apache.beam.it.gcp.pubsub.PubsubResourceManager; +import org.apache.beam.it.gcp.spanner.SpannerResourceManager; +import org.apache.beam.it.gcp.spanner.conditions.SpannerRowsCheck; +import org.apache.beam.it.gcp.spanner.matchers.SpannerAsserts; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Integration test for datatype conversions for {@link DataStreamToSpanner} Flex template. */ +@Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) +@TemplateIntegrationTest(DataStreamToSpanner.class) +@RunWith(JUnit4.class) +public class DataStreamToSpannerDatatypeIT extends DataStreamToSpannerITBase { + private static final Logger LOG = LoggerFactory.getLogger(DataStreamToSpannerDatatypeIT.class); + + private static final String SESSION_FILE_RESOURCE = + "DataStreamToSpannerDatatypeIT/mysql-session.json"; + + private static final String SPANNER_DDL_RESOURCE = + "DataStreamToSpannerDatatypeIT/spanner-schema.sql"; + + private static final String TABLE1 = "AllDatatypeColumns"; + private static final String TABLE2 = "AllDatatypeColumns2"; + + private static HashSet testInstances = new HashSet<>(); + private static PipelineLauncher.LaunchInfo jobInfo; + + public static PubsubResourceManager pubsubResourceManager; + public static SpannerResourceManager spannerResourceManager; + + /** + * Setup resource managers and Launch dataflow job once during the execution of this test class. + * + * @throws IOException + */ + @Before + public void setUp() throws IOException { + // Prevent cleaning up of dataflow job after a test method is executed. + skipBaseCleanup = true; + synchronized (DataStreamToSpannerDatatypeIT.class) { + testInstances.add(this); + if (jobInfo == null) { + spannerResourceManager = setUpSpannerResourceManager(); + pubsubResourceManager = setUpPubSubResourceManager(); + createSpannerDDL(spannerResourceManager, SPANNER_DDL_RESOURCE); + jobInfo = + launchDataflowJob( + getClass().getSimpleName(), + SESSION_FILE_RESOURCE, + null, + "DatatypeIT", + spannerResourceManager, + pubsubResourceManager, + new HashMap<>() { + { + put("inputFileFormat", "json"); + } + }); + } + } + } + + /** + * Cleanup dataflow job and all the resources and resource managers. + * + * @throws IOException + */ + @AfterClass + public static void cleanUp() throws IOException { + for (DataStreamToSpannerDatatypeIT instance : testInstances) { + instance.tearDownBase(); + } + ResourceManagerUtils.cleanResources(spannerResourceManager, pubsubResourceManager); + } + + @Test + public void migrationTestWithAllDatatypeDefaultMapping() { + // Construct a ChainedConditionCheck with 4 stages. + // 1. Send initial wave of events + // 2. Wait on Spanner to have events + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE1, + "backfill.jsonl", + "DataStreamToSpannerDatatypeIT/mysql-backfill-AllDatatypeColumns.jsonl"), + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + // Wait for conditions + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + // Assert Conditions + assertThatResult(result).meetsConditions(); + + assertAllDatatypeColumnsTableBackfillContents(); + + conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE1, + "cdc1.jsonl", + "DataStreamToSpannerDatatypeIT/mysql-cdc-AllDatatypeColumns.jsonl"), + SpannerRowsCheck.builder(spannerResourceManager, TABLE1) + .setMinRows(1) + .setMaxRows(1) + .build())) + .build(); + + result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + // Assert Conditions + assertThatResult(result).meetsConditions(); + + assertAllDatatypeColumnsTableCdcContents(); + } + + @Test + public void migrationTestWithAllDatatypeConversionMapping() { + // Construct a ChainedConditionCheck with 4 stages. + // 1. Send initial wave of events + // 2. Wait on Spanner to have events + ChainedConditionCheck conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE2, + "backfill.jsonl", + "DataStreamToSpannerDatatypeIT/mysql-backfill-AllDatatypeColumns2.jsonl"), + SpannerRowsCheck.builder(spannerResourceManager, TABLE2) + .setMinRows(2) + .setMaxRows(2) + .build())) + .build(); + + // Wait for conditions + PipelineOperator.Result result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + // Assert Conditions + assertThatResult(result).meetsConditions(); + + assertAllDatatypeColumns2TableBackfillContents(); + + conditionCheck = + ChainedConditionCheck.builder( + List.of( + uploadDataStreamFile( + jobInfo, + TABLE2, + "cdc1.jsonl", + "DataStreamToSpannerDatatypeIT/mysql-cdc-AllDatatypeColumns2.jsonl"), + SpannerRowsCheck.builder(spannerResourceManager, TABLE2) + .setMinRows(1) + .setMaxRows(1) + .build())) + .build(); + + result = + pipelineOperator() + .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); + + // Assert Conditions + assertThatResult(result).meetsConditions(); + + assertAllDatatypeColumns2TableCdcContents(); + } + + private void assertAllDatatypeColumnsTableBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", "10"); + row.put("text_column", "dGV4dF9kYXRhXzEK"); + row.put("date_column", "2024-02-08T00:00:00.000Z"); + row.put("smallint_column", "50"); + row.put("mediumint_column", "1000"); + row.put("int_column", "50000"); + row.put("bigint_column", "987654321"); + row.put("float_column", "45.67"); + row.put("double_column", "123.789"); + row.put("datetime_column", "2024-02-08T08:15:30.000Z"); + row.put("timestamp_column", "2024-02-08T08:15:30.000Z"); + row.put("time_column", "29730000000"); + row.put("year_column", "2022"); + row.put("char_column", "Y2hhcjEK"); + row.put("tinyblob_column", "74696e79626c6f625f646174615f31"); + row.put("tinytext_column", "dGlueXRleHRfZGF0YV8xCg=="); + row.put("blob_column", "626c6f625f646174615f31"); + row.put("mediumblob_column", "6d656469756d626c6f625f646174615f31"); + row.put("mediumtext_column", "bWVkaXVtdGV4dF9kYXRhXzE="); + row.put("longblob_column", "6c6f6e67626c6f625f646174615f31"); + row.put("longtext_column", "bG9uZ3RleHRfZGF0YV8x"); + row.put("enum_column", "2"); + row.put("bool_column", 0); + row.put("other_bool_column", "1"); + row.put("binary_column", "62696e6172795f31"); + row.put("varbinary_column", "76617262696e6172795f646174615f31"); + row.put("bit_column", "102"); + events.add(row); + + row.put("varchar_column", "value2"); + row.put("tinyint_column", "5"); + row.put("text_column", "dGV4dF9kYXRhXzIK"); + row.put("date_column", "2024-02-09T00:00:00.000Z"); + row.put("smallint_column", "25"); + row.put("mediumint_column", "500"); + row.put("int_column", "25000"); + row.put("bigint_column", "987654"); + row.put("float_column", "12.34"); + row.put("double_column", "56.789"); + row.put("datetime_column", "2024-02-09T15:30:45.000Z"); + row.put("timestamp_column", "2024-02-09T15:30:45.000Z"); + row.put("time_column", "55845000000"); + row.put("year_column", "2023"); + row.put("char_column", "Y2hhcjIK"); + row.put("tinyblob_column", "74696e79626c6f625f646174615f32"); + row.put("tinytext_column", "dGlueXRleHRfZGF0YV8yCg=="); + row.put("blob_column", "626c6f625f646174615f32"); + row.put("mediumblob_column", "6d656469756d626c6f625f646174615f32"); + row.put("mediumtext_column", "bWVkaXVtdGV4dF9kYXRhXzI="); + row.put("longblob_column", "6c6f6e67626c6f625f646174615f32"); + row.put("longtext_column", "bG9uZ3RleHRfZGF0YV8y"); + row.put("enum_column", "3"); + row.put("bool_column", 1); + row.put("other_bool_column", "0"); + row.put("binary_column", "62696e6172795f32"); + row.put("varbinary_column", "76617262696e6172795f646174615f32"); + row.put("bit_column", "25"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column" + + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" + + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" + + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column, " + + " longblob_column, longtext_column, enum_column, bool_column, other_bool_column, binary_column" + + ", varbinary_column, bit_column from AllDatatypeColumns")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumnsTableCdcContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", "15"); + row.put("text_column", "dGV4dF9kYXRhXzEK"); + row.put("date_column", "2024-02-08T00:00:00.000Z"); + row.put("smallint_column", "50"); + row.put("mediumint_column", "1000"); + row.put("int_column", "50000"); + row.put("bigint_column", "987654321"); + row.put("float_column", "45.67"); + row.put("double_column", "123.789"); + row.put("datetime_column", "2024-02-08T08:15:30.000Z"); + row.put("timestamp_column", "2024-02-08T08:15:30.000Z"); + row.put("time_column", "29730000000"); + row.put("year_column", "2022"); + row.put("char_column", "Y2hhcjEK"); + row.put("tinyblob_column", "74696e79626c6f625f646174615f31"); + row.put("tinytext_column", "dGlueXRleHRfZGF0YV8xCg=="); + row.put("blob_column", "626c6f625f646174615f31"); + row.put("mediumblob_column", "6d656469756d626c6f625f646174615f31"); + row.put("mediumtext_column", "bWVkaXVtdGV4dF9kYXRhXzE="); + row.put("longblob_column", "6c6f6e67626c6f625f646174615f31"); + row.put("longtext_column", "bG9uZ3RleHRfZGF0YV8x"); + row.put("enum_column", "2"); + row.put("bool_column", 0); + row.put("other_bool_column", "1"); + row.put("binary_column", "62696e6172795f31"); + row.put("varbinary_column", "76617262696e6172795f646174615f31"); + row.put("bit_column", "102"); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column" + + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" + + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" + + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column, " + + " longblob_column, longtext_column, enum_column, bool_column, other_bool_column, binary_column" + + ", varbinary_column, bit_column from AllDatatypeColumns")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumns2TableBackfillContents() { + List> events = new ArrayList<>(); + + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", 10); + row.put("text_column", "text1"); + row.put("date_column", "2024-02-08"); + row.put("smallint_column", 50); + row.put("mediumint_column", 1000); + row.put("int_column", 50000); + row.put("bigint_column", 987654321); + row.put("float_column", 45.67); + row.put("double_column", 123.789); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "29730000000"); + row.put("year_column", "2022"); + row.put("char_column", "char_1"); + // Source column value: 74696e79626c6f625f646174615f31 ( in BYTES, "tinyblob_data_1" in STRING) + // results in dGlueWJsb2JfZGF0YV8x base64 encoded string + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8x"); + row.put("tinytext_column", "tinytext_data_1"); + row.put("blob_column", "YmxvYl9kYXRhXzE="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzE="); + row.put("mediumtext_column", "mediumtext_data_1"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8x"); + row.put("longtext_column", "longtext_data_1"); + row.put("enum_column", "2"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "AQI="); + events.add(row); + + row.put("varchar_column", "value2"); + row.put("tinyint_column", 5); + row.put("text_column", "text2"); + row.put("date_column", "2024-02-09"); + row.put("smallint_column", 25); + row.put("mediumint_column", 500); + row.put("int_column", 25000); + row.put("bigint_column", 987654); + row.put("float_column", 12.34); + row.put("double_column", 56.789); + row.put("datetime_column", "2024-02-09T15:30:45Z"); + row.put("timestamp_column", "2024-02-09T15:30:45Z"); + row.put("time_column", "55845000000"); + row.put("year_column", "2023"); + row.put("char_column", "char_2"); + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8y"); + row.put("tinytext_column", "tinytext_data_2"); + row.put("blob_column", "YmxvYl9kYXRhXzI="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzI="); + row.put("mediumtext_column", "mediumtext_data_2"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8y"); + row.put("longtext_column", "longtext_data_2"); + row.put("enum_column", "3"); + row.put("bool_column", true); + row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); + row.put("bit_column", "JQ=="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column" + + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" + + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" + + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column, " + + " longblob_column, longtext_column, enum_column, bool_column, binary_column" + + ", varbinary_column, bit_column from AllDatatypeColumns2")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } + + private void assertAllDatatypeColumns2TableCdcContents() { + List> events = new ArrayList<>(); + Map row = new HashMap<>(); + row.put("varchar_column", "value1"); + row.put("tinyint_column", 15); + row.put("text_column", "text1"); + row.put("date_column", "2024-02-08"); + row.put("smallint_column", 50); + row.put("mediumint_column", 1000); + row.put("int_column", 50000); + row.put("bigint_column", 987654321); + row.put("float_column", 45.67); + row.put("double_column", 123.789); + row.put("datetime_column", "2024-02-08T08:15:30Z"); + row.put("timestamp_column", "2024-02-08T08:15:30Z"); + row.put("time_column", "29730000000"); + row.put("year_column", "2022"); + row.put("char_column", "char_1"); + // Source column value: 74696e79626c6f625f646174615f31 ( in BYTES, "tinyblob_data_1" in STRING) + // results in dGlueWJsb2JfZGF0YV8x base64 encoded string + row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8x"); + row.put("tinytext_column", "tinytext_data_1"); + row.put("blob_column", "YmxvYl9kYXRhXzE="); + row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzE="); + row.put("mediumtext_column", "mediumtext_data_1"); + row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8x"); + row.put("longtext_column", "longtext_data_1"); + row.put("enum_column", "2"); + row.put("bool_column", false); + row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); + row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); + row.put("bit_column", "AQI="); + events.add(row); + + SpannerAsserts.assertThatStructs( + spannerResourceManager.runQuery( + "select varchar_column, tinyint_column, text_column, date_column" + + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" + + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" + + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column, " + + " longblob_column, longtext_column, enum_column, bool_column, binary_column" + + ", varbinary_column, bit_column from AllDatatypeColumns2")) + .hasRecordsUnorderedCaseInsensitiveColumns(events); + } +} diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java index 6331aab8c6..3991883739 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerSimpleIT.java @@ -59,8 +59,6 @@ public class DataStreamToSpannerSimpleIT extends DataStreamToSpannerITBase { private static final String TABLE1 = "Users"; private static final String TABLE2 = "Movie"; private static final String TABLE3 = "Category"; - private static final String TABLE4 = "AllDatatypeColumns"; - private static final String TABLE5 = "AllDatatypeColumns2"; private static final String SESSION_FILE_RESOURCE = "DataStreamToSpannerSimpleIT/mysql-session.json"; @@ -240,112 +238,6 @@ public void interleavedAndFKAndIndexTest() { assertArticlesTable(); } - @Test - public void migrationTestWithAllDatatypeDefaultMapping() { - // Construct a ChainedConditionCheck with 4 stages. - // 1. Send initial wave of events - // 2. Wait on Spanner to have events - ChainedConditionCheck conditionCheck = - ChainedConditionCheck.builder( - List.of( - uploadDataStreamFile( - jobInfo, - TABLE4, - "backfill.jsonl", - "DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns.jsonl"), - SpannerRowsCheck.builder(spannerResourceManager, TABLE4) - .setMinRows(2) - .setMaxRows(2) - .build())) - .build(); - - // Wait for conditions - PipelineOperator.Result result = - pipelineOperator() - .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); - - // Assert Conditions - assertThatResult(result).meetsConditions(); - - assertAllDatatypeColumnsTableBackfillContents(); - - conditionCheck = - ChainedConditionCheck.builder( - List.of( - uploadDataStreamFile( - jobInfo, - TABLE4, - "cdc1.jsonl", - "DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl"), - SpannerRowsCheck.builder(spannerResourceManager, TABLE4) - .setMinRows(1) - .setMaxRows(1) - .build())) - .build(); - - result = - pipelineOperator() - .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); - - // Assert Conditions - assertThatResult(result).meetsConditions(); - - assertAllDatatypeColumnsTableCdcContents(); - } - - @Test - public void migrationTestWithAllDatatypeConversionMapping() { - // Construct a ChainedConditionCheck with 4 stages. - // 1. Send initial wave of events - // 2. Wait on Spanner to have events - ChainedConditionCheck conditionCheck = - ChainedConditionCheck.builder( - List.of( - uploadDataStreamFile( - jobInfo, - TABLE5, - "backfill.jsonl", - "DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns2.jsonl"), - SpannerRowsCheck.builder(spannerResourceManager, TABLE5) - .setMinRows(2) - .setMaxRows(2) - .build())) - .build(); - - // Wait for conditions - PipelineOperator.Result result = - pipelineOperator() - .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); - - // Assert Conditions - assertThatResult(result).meetsConditions(); - - assertAllDatatypeColumns2TableBackfillContents(); - - conditionCheck = - ChainedConditionCheck.builder( - List.of( - uploadDataStreamFile( - jobInfo, - TABLE5, - "cdc1.jsonl", - "DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns2.jsonl"), - SpannerRowsCheck.builder(spannerResourceManager, TABLE5) - .setMinRows(1) - .setMaxRows(1) - .build())) - .build(); - - result = - pipelineOperator() - .waitForCondition(createConfig(jobInfo, Duration.ofMinutes(8)), conditionCheck); - - // Assert Conditions - assertThatResult(result).meetsConditions(); - - assertAllDatatypeColumns2TableCdcContents(); - } - @Test public void migrationTestWithRenameAndDropColumn() { // Construct a ChainedConditionCheck with 4 stages. @@ -461,246 +353,6 @@ private void assertMovieTableContents() { Assert.assertEquals(931.512, numericVals.get(1).getBigDecimal(0).doubleValue(), 0.001); } - private void assertAllDatatypeColumnsTableBackfillContents() { - List> events = new ArrayList<>(); - - Map row = new HashMap<>(); - row.put("varchar_column", "value1"); - row.put("tinyint_column", "10"); - row.put("text_column", "dGV4dF9kYXRhXzEK"); - row.put("date_column", "2024-02-08T00:00:00.000Z"); - row.put("smallint_column", "50"); - row.put("mediumint_column", "1000"); - row.put("int_column", "50000"); - row.put("bigint_column", "987654321"); - row.put("float_column", "45.67"); - row.put("double_column", "123.789"); - row.put("datetime_column", "2024-02-08T08:15:30.000Z"); - row.put("timestamp_column", "2024-02-08T08:15:30.000Z"); - row.put("time_column", "29730000000"); - row.put("year_column", "2022"); - row.put("char_column", "Y2hhcjEK"); - row.put("tinyblob_column", "74696e79626c6f625f646174615f31"); - row.put("tinytext_column", "dGlueXRleHRfZGF0YV8xCg=="); - row.put("blob_column", "626c6f625f646174615f31"); - row.put("mediumblob_column", "6d656469756d626c6f625f646174615f31"); - row.put("mediumtext_column", "bWVkaXVtdGV4dF9kYXRhXzE="); - row.put("longblob_column", "6c6f6e67626c6f625f646174615f31"); - row.put("longtext_column", "bG9uZ3RleHRfZGF0YV8x"); - row.put("enum_column", "2"); - row.put("bool_column", 0); - row.put("other_bool_column", "1"); - row.put("binary_column", "62696e6172795f31"); - row.put("varbinary_column", "76617262696e6172795f646174615f31"); - row.put("bit_column", "102"); - events.add(row); - - row.put("varchar_column", "value2"); - row.put("tinyint_column", "5"); - row.put("text_column", "dGV4dF9kYXRhXzIK"); - row.put("date_column", "2024-02-09T00:00:00.000Z"); - row.put("smallint_column", "25"); - row.put("mediumint_column", "500"); - row.put("int_column", "25000"); - row.put("bigint_column", "987654"); - row.put("float_column", "12.34"); - row.put("double_column", "56.789"); - row.put("datetime_column", "2024-02-09T15:30:45.000Z"); - row.put("timestamp_column", "2024-02-09T15:30:45.000Z"); - row.put("time_column", "55845000000"); - row.put("year_column", "2023"); - row.put("char_column", "Y2hhcjIK"); - row.put("tinyblob_column", "74696e79626c6f625f646174615f32"); - row.put("tinytext_column", "dGlueXRleHRfZGF0YV8yCg=="); - row.put("blob_column", "626c6f625f646174615f32"); - row.put("mediumblob_column", "6d656469756d626c6f625f646174615f32"); - row.put("mediumtext_column", "bWVkaXVtdGV4dF9kYXRhXzI="); - row.put("longblob_column", "6c6f6e67626c6f625f646174615f32"); - row.put("longtext_column", "bG9uZ3RleHRfZGF0YV8y"); - row.put("enum_column", "3"); - row.put("bool_column", 1); - row.put("other_bool_column", "0"); - row.put("binary_column", "62696e6172795f32"); - row.put("varbinary_column", "76617262696e6172795f646174615f32"); - row.put("bit_column", "25"); - events.add(row); - - SpannerAsserts.assertThatStructs( - spannerResourceManager.runQuery( - "select varchar_column, tinyint_column, text_column, date_column" - + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" - + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" - + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column, " - + " longblob_column, longtext_column, enum_column, bool_column, other_bool_column, binary_column" - + ", varbinary_column, bit_column from AllDatatypeColumns")) - .hasRecordsUnorderedCaseInsensitiveColumns(events); - } - - private void assertAllDatatypeColumnsTableCdcContents() { - List> events = new ArrayList<>(); - - Map row = new HashMap<>(); - row.put("varchar_column", "value1"); - row.put("tinyint_column", "15"); - row.put("text_column", "dGV4dF9kYXRhXzEK"); - row.put("date_column", "2024-02-08T00:00:00.000Z"); - row.put("smallint_column", "50"); - row.put("mediumint_column", "1000"); - row.put("int_column", "50000"); - row.put("bigint_column", "987654321"); - row.put("float_column", "45.67"); - row.put("double_column", "123.789"); - row.put("datetime_column", "2024-02-08T08:15:30.000Z"); - row.put("timestamp_column", "2024-02-08T08:15:30.000Z"); - row.put("time_column", "29730000000"); - row.put("year_column", "2022"); - row.put("char_column", "Y2hhcjEK"); - row.put("tinyblob_column", "74696e79626c6f625f646174615f31"); - row.put("tinytext_column", "dGlueXRleHRfZGF0YV8xCg=="); - row.put("blob_column", "626c6f625f646174615f31"); - row.put("mediumblob_column", "6d656469756d626c6f625f646174615f31"); - row.put("mediumtext_column", "bWVkaXVtdGV4dF9kYXRhXzE="); - row.put("longblob_column", "6c6f6e67626c6f625f646174615f31"); - row.put("longtext_column", "bG9uZ3RleHRfZGF0YV8x"); - row.put("enum_column", "2"); - row.put("bool_column", 0); - row.put("other_bool_column", "1"); - row.put("binary_column", "62696e6172795f31"); - row.put("varbinary_column", "76617262696e6172795f646174615f31"); - row.put("bit_column", "102"); - events.add(row); - - SpannerAsserts.assertThatStructs( - spannerResourceManager.runQuery( - "select varchar_column, tinyint_column, text_column, date_column" - + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" - + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" - + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column, " - + " longblob_column, longtext_column, enum_column, bool_column, other_bool_column, binary_column" - + ", varbinary_column, bit_column from AllDatatypeColumns")) - .hasRecordsUnorderedCaseInsensitiveColumns(events); - } - - private void assertAllDatatypeColumns2TableBackfillContents() { - List> events = new ArrayList<>(); - - Map row = new HashMap<>(); - row.put("varchar_column", "value1"); - row.put("tinyint_column", 10); - row.put("text_column", "text1"); - row.put("date_column", "2024-02-08"); - row.put("smallint_column", 50); - row.put("mediumint_column", 1000); - row.put("int_column", 50000); - row.put("bigint_column", 987654321); - row.put("float_column", 45.67); - row.put("double_column", 123.789); - row.put("datetime_column", "2024-02-08T08:15:30Z"); - row.put("timestamp_column", "2024-02-08T08:15:30Z"); - row.put("time_column", "29730000000"); - row.put("year_column", "2022"); - row.put("char_column", "char_1"); - // Source column value: 74696e79626c6f625f646174615f31 ( in BYTES, "tinyblob_data_1" in STRING) - // results in dGlueWJsb2JfZGF0YV8x base64 encoded string - row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8x"); - row.put("tinytext_column", "tinytext_data_1"); - row.put("blob_column", "YmxvYl9kYXRhXzE="); - row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzE="); - row.put("mediumtext_column", "mediumtext_data_1"); - row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8x"); - row.put("longtext_column", "longtext_data_1"); - row.put("enum_column", "2"); - row.put("bool_column", false); - row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); - row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); - row.put("bit_column", "AQI="); - events.add(row); - - row.put("varchar_column", "value2"); - row.put("tinyint_column", 5); - row.put("text_column", "text2"); - row.put("date_column", "2024-02-09"); - row.put("smallint_column", 25); - row.put("mediumint_column", 500); - row.put("int_column", 25000); - row.put("bigint_column", 987654); - row.put("float_column", 12.34); - row.put("double_column", 56.789); - row.put("datetime_column", "2024-02-09T15:30:45Z"); - row.put("timestamp_column", "2024-02-09T15:30:45Z"); - row.put("time_column", "55845000000"); - row.put("year_column", "2023"); - row.put("char_column", "char_2"); - row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8y"); - row.put("tinytext_column", "tinytext_data_2"); - row.put("blob_column", "YmxvYl9kYXRhXzI="); - row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzI="); - row.put("mediumtext_column", "mediumtext_data_2"); - row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8y"); - row.put("longtext_column", "longtext_data_2"); - row.put("enum_column", "3"); - row.put("bool_column", true); - row.put("binary_column", "YmluYXJ5X2RhdGFfMgAAAAAAAAA="); - row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMg=="); - row.put("bit_column", "JQ=="); - events.add(row); - - SpannerAsserts.assertThatStructs( - spannerResourceManager.runQuery( - "select varchar_column, tinyint_column, text_column, date_column" - + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" - + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" - + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column, " - + " longblob_column, longtext_column, enum_column, bool_column, binary_column" - + ", varbinary_column, bit_column from AllDatatypeColumns2")) - .hasRecordsUnorderedCaseInsensitiveColumns(events); - } - - private void assertAllDatatypeColumns2TableCdcContents() { - List> events = new ArrayList<>(); - Map row = new HashMap<>(); - row.put("varchar_column", "value1"); - row.put("tinyint_column", 15); - row.put("text_column", "text1"); - row.put("date_column", "2024-02-08"); - row.put("smallint_column", 50); - row.put("mediumint_column", 1000); - row.put("int_column", 50000); - row.put("bigint_column", 987654321); - row.put("float_column", 45.67); - row.put("double_column", 123.789); - row.put("datetime_column", "2024-02-08T08:15:30Z"); - row.put("timestamp_column", "2024-02-08T08:15:30Z"); - row.put("time_column", "29730000000"); - row.put("year_column", "2022"); - row.put("char_column", "char_1"); - // Source column value: 74696e79626c6f625f646174615f31 ( in BYTES, "tinyblob_data_1" in STRING) - // results in dGlueWJsb2JfZGF0YV8x base64 encoded string - row.put("tinyblob_column", "dGlueWJsb2JfZGF0YV8x"); - row.put("tinytext_column", "tinytext_data_1"); - row.put("blob_column", "YmxvYl9kYXRhXzE="); - row.put("mediumblob_column", "bWVkaXVtYmxvYl9kYXRhXzE="); - row.put("mediumtext_column", "mediumtext_data_1"); - row.put("longblob_column", "bG9uZ2Jsb2JfZGF0YV8x"); - row.put("longtext_column", "longtext_data_1"); - row.put("enum_column", "2"); - row.put("bool_column", false); - row.put("binary_column", "YmluYXJ5X2RhdGFfMQAAAAAAAAA="); - row.put("varbinary_column", "dmFyYmluYXJ5X2RhdGFfMQ=="); - row.put("bit_column", "AQI="); - events.add(row); - - SpannerAsserts.assertThatStructs( - spannerResourceManager.runQuery( - "select varchar_column, tinyint_column, text_column, date_column" - + ", smallint_column, mediumint_column, int_column, bigint_column, float_column" - + ", double_column, datetime_column, timestamp_column, time_column, year_column, char_column" - + ", tinyblob_column, tinytext_column, blob_column, mediumblob_column, mediumtext_column, " - + " longblob_column, longtext_column, enum_column, bool_column, binary_column" - + ", varbinary_column, bit_column from AllDatatypeColumns2")) - .hasRecordsUnorderedCaseInsensitiveColumns(events); - } - private void assertCategoryTableBackfillContents() { List> events = new ArrayList<>(); diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns.jsonl b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-backfill-AllDatatypeColumns.jsonl similarity index 100% rename from v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns.jsonl rename to v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-backfill-AllDatatypeColumns.jsonl diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns2.jsonl b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-backfill-AllDatatypeColumns2.jsonl similarity index 100% rename from v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-backfill-AllDatatypeColumns2.jsonl rename to v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-backfill-AllDatatypeColumns2.jsonl diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-cdc-AllDatatypeColumns.jsonl similarity index 100% rename from v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns.jsonl rename to v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-cdc-AllDatatypeColumns.jsonl diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns2.jsonl b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-cdc-AllDatatypeColumns2.jsonl similarity index 100% rename from v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-cdc-AllDatatypeColumns2.jsonl rename to v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-cdc-AllDatatypeColumns2.jsonl diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-schema.sql b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-schema.sql new file mode 100644 index 0000000000..7988cc7adf --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-schema.sql @@ -0,0 +1,65 @@ +-- Database datatype_test in shreya-mysql +CREATE TABLE `AllDatatypeColumns` ( + `varchar_column` varchar(20) NOT NULL, -- To: varchar_column BYTES(20) + `tinyint_column` tinyint, -- To: tinyint_column STRING(MAX) + `text_column` text, -- To: text_column BYTES(MAX) + `date_column` date, -- To: date_column STRING(MAX) + `smallint_column` smallint, -- To: smallint_column STRING(MAX) + `mediumint_column` mediumint, -- To: mediumint_column STRING(MAX) + `int_column` int, -- To: int_column STRING(MAX) + `bigint_column` bigint, -- To: bigint_column STRING(MAX) + `float_column` float(10,2), -- To: float_column STRING(MAX) + `double_column` double, -- To: double_column STRING(MAX) + `decimal_column` decimal(10,2), -- To: decimal_column STRING(MAX) + `datetime_column` datetime, -- To: datetime_column STRING(MAX) + `timestamp_column` timestamp, -- To: timestamp_column STRING(MAX) + `time_column` time, -- To: time_column STRING(MAX) + `year_column` year, -- To: year_column STRING(MAX) + `char_column` char(10), -- To: char_column BYTES(10) + `tinyblob_column` tinyblob, -- To: tinyblob_column STRING(MAX) + `tinytext_column` tinytext, -- To: tinytext_column BYTES(MAX) + `blob_column` blob, -- To: blob_column STRING(MAX) + `mediumblob_column` mediumblob, -- To: mediumblob_column STRING(MAX) + `mediumtext_column` mediumtext, -- To: mediumtext_column BYTES(MAX) + `longblob_column` longblob, -- To: longblob_column STRING(MAX) + `longtext_column` longtext, -- To: longtext_column BYTES(MAX) + `enum_column` enum('1','2','3'), -- To: enum_column STRING(MAX) + `bool_column` tinyint(1), -- To: bool_column INT64 + `other_bool_column` tinyint(1), -- To: other_bool_column STRING(MAX) + `binary_column` binary(20), -- To: binary_column STRING(MAX) + `varbinary_column` varbinary(20), -- To: varbinary_column STRING(MAX) + `bit_column` bit(7), -- To: bit_column STRING(MAX) + PRIMARY KEY (`varchar_column`) +); + +CREATE TABLE `AllDatatypeColumns2` ( + `varchar_column` varchar(20) NOT NULL, -- To: varchar_column BYTES(20) + `tinyint_column` tinyint, -- To: tinyint_column INT64 + `text_column` text, -- To: text_column STRING(MAX) + `date_column` date, -- To: date_column DATE + `smallint_column` smallint, -- To: smallint_column INT64 + `mediumint_column` mediumint, -- To: mediumint_column INT64 + `int_column` int, -- To: int_column INT64 + `bigint_column` bigint, -- To: bigint_column INT64 + `float_column` float(10,2), -- To: float_column FLOAT64 + `double_column` double, -- To: double_column FLOAT64 + `decimal_column` decimal(10,2), -- To: decimal_column NUMERIC + `datetime_column` datetime, -- To: datetime_column TIMESTAMP + `timestamp_column` timestamp, -- To: timestamp_column TIMESTAMP + `time_column` time, -- To: time_column STRING(MAX) + `year_column` year, -- To: year_column STRING(MAX) + `char_column` char(10), -- To: char_column STRING(10) + `tinyblob_column` tinyblob, -- To: tinyblob_column BYTES(MAX) + `tinytext_column` tinytext, -- To: tinytext_column STRING(MAX) + `blob_column` blob, -- To: blob_column BYTES(MAX) + `mediumblob_column` mediumblob, -- To: mediumblob_column BYTES(MAX) + `mediumtext_column` mediumtext, -- To: mediumtext_column STRING(MAX) + `longblob_column` longblob, -- To: longblob_column BYTES(MAX) + `longtext_column` longtext, -- To: longtext_column STRING(MAX) + `enum_column` enum('1','2','3'), -- To: enum_column STRING(MAX) + `bool_column` tinyint(1), -- To: bool_column BOOL + `binary_column` binary(20), -- To: binary_column BYTES(MAX) + `varbinary_column` varbinary(20), -- To: varbinary_column BYTES(MAX) + `bit_column` bit(7), -- To: bit_column BYTES(MAX) + PRIMARY KEY (`varchar_column`) +); \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-session.json b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-session.json new file mode 100644 index 0000000000..0c73f55b78 --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-session.json @@ -0,0 +1,2059 @@ +{ + "SessionName": "NewSession", + "EditorName": "", + "DatabaseType": "mysql", + "DatabaseName": "datatype_test", + "Dialect": "google_standard_sql", + "Notes": null, + "Tags": null, + "SpSchema": { + "t77": { + "Name": "AllDatatypeColumns", + "ColIds": [ + "c79", + "c80", + "c81", + "c82", + "c83", + "c84", + "c85", + "c86", + "c87", + "c88", + "c89", + "c90", + "c91", + "c92", + "c93", + "c94", + "c95", + "c96", + "c97", + "c98", + "c99", + "c100", + "c101", + "c102", + "c103", + "c104", + "c105", + "c106", + "c107" + ], + "ShardIdColumn": "", + "ColDefs": { + "c100": { + "Name": "longblob_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: longblob_column longblob(4294967295)", + "Id": "c100" + }, + "c101": { + "Name": "longtext_column", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: longtext_column longtext(4294967295)", + "Id": "c101" + }, + "c102": { + "Name": "enum_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: enum_column enum(1)", + "Id": "c102" + }, + "c103": { + "Name": "bool_column", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bool_column tinyint(1)", + "Id": "c103" + }, + "c104": { + "Name": "other_bool_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: other_bool_column tinyint(1)", + "Id": "c104" + }, + "c105": { + "Name": "binary_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: binary_column binary(20)", + "Id": "c105" + }, + "c106": { + "Name": "varbinary_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: varbinary_column varbinary(20)", + "Id": "c106" + }, + "c107": { + "Name": "bit_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bit_column bit(7)", + "Id": "c107" + }, + "c79": { + "Name": "varchar_column", + "T": { + "Name": "BYTES", + "Len": 20, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: varchar_column varchar(20)", + "Id": "c79" + }, + "c80": { + "Name": "tinyint_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinyint_column tinyint(3)", + "Id": "c80" + }, + "c81": { + "Name": "text_column", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: text_column text(65535)", + "Id": "c81" + }, + "c82": { + "Name": "date_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: date_column date", + "Id": "c82" + }, + "c83": { + "Name": "smallint_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: smallint_column smallint(5)", + "Id": "c83" + }, + "c84": { + "Name": "mediumint_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumint_column mediumint(7)", + "Id": "c84" + }, + "c85": { + "Name": "int_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: int_column int(10)", + "Id": "c85" + }, + "c86": { + "Name": "bigint_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bigint_column bigint(19)", + "Id": "c86" + }, + "c87": { + "Name": "float_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: float_column float(10,2)", + "Id": "c87" + }, + "c88": { + "Name": "double_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: double_column double(22)", + "Id": "c88" + }, + "c89": { + "Name": "decimal_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: decimal_column decimal(10,2)", + "Id": "c89" + }, + "c90": { + "Name": "datetime_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: datetime_column datetime", + "Id": "c90" + }, + "c91": { + "Name": "timestamp_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: timestamp_column timestamp", + "Id": "c91" + }, + "c92": { + "Name": "time_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: time_column time", + "Id": "c92" + }, + "c93": { + "Name": "year_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: year_column year", + "Id": "c93" + }, + "c94": { + "Name": "char_column", + "T": { + "Name": "BYTES", + "Len": 10, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: char_column char(10)", + "Id": "c94" + }, + "c95": { + "Name": "tinyblob_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinyblob_column tinyblob(255)", + "Id": "c95" + }, + "c96": { + "Name": "tinytext_column", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinytext_column tinytext(255)", + "Id": "c96" + }, + "c97": { + "Name": "blob_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: blob_column blob(65535)", + "Id": "c97" + }, + "c98": { + "Name": "mediumblob_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumblob_column mediumblob(16777215)", + "Id": "c98" + }, + "c99": { + "Name": "mediumtext_column", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumtext_column mediumtext(16777215)", + "Id": "c99" + } + }, + "PrimaryKeys": [ + { + "ColId": "c79", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table AllDatatypeColumns", + "Id": "t77" + }, + "t78": { + "Name": "AllDatatypeColumns2", + "ColIds": [ + "c108", + "c109", + "c110", + "c111", + "c112", + "c113", + "c114", + "c115", + "c116", + "c117", + "c118", + "c119", + "c120", + "c121", + "c122", + "c123", + "c124", + "c125", + "c126", + "c127", + "c128", + "c129", + "c130", + "c131", + "c132", + "c133", + "c134", + "c135" + ], + "ShardIdColumn": "", + "ColDefs": { + "c108": { + "Name": "varchar_column", + "T": { + "Name": "STRING", + "Len": 20, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: varchar_column varchar(20)", + "Id": "c108" + }, + "c109": { + "Name": "tinyint_column", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinyint_column tinyint(3)", + "Id": "c109" + }, + "c110": { + "Name": "text_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: text_column text(65535)", + "Id": "c110" + }, + "c111": { + "Name": "date_column", + "T": { + "Name": "DATE", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: date_column date", + "Id": "c111" + }, + "c112": { + "Name": "smallint_column", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: smallint_column smallint(5)", + "Id": "c112" + }, + "c113": { + "Name": "mediumint_column", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumint_column mediumint(7)", + "Id": "c113" + }, + "c114": { + "Name": "int_column", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: int_column int(10)", + "Id": "c114" + }, + "c115": { + "Name": "bigint_column", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bigint_column bigint(19)", + "Id": "c115" + }, + "c116": { + "Name": "float_column", + "T": { + "Name": "FLOAT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: float_column float(10,2)", + "Id": "c116" + }, + "c117": { + "Name": "double_column", + "T": { + "Name": "FLOAT64", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: double_column double(22)", + "Id": "c117" + }, + "c118": { + "Name": "decimal_column", + "T": { + "Name": "NUMERIC", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: decimal_column decimal(10,2)", + "Id": "c118" + }, + "c119": { + "Name": "datetime_column", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: datetime_column datetime", + "Id": "c119" + }, + "c120": { + "Name": "timestamp_column", + "T": { + "Name": "TIMESTAMP", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: timestamp_column timestamp", + "Id": "c120" + }, + "c121": { + "Name": "time_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: time_column time", + "Id": "c121" + }, + "c122": { + "Name": "year_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: year_column year", + "Id": "c122" + }, + "c123": { + "Name": "char_column", + "T": { + "Name": "STRING", + "Len": 10, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: char_column char(10)", + "Id": "c123" + }, + "c124": { + "Name": "tinyblob_column", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinyblob_column tinyblob(255)", + "Id": "c124" + }, + "c125": { + "Name": "tinytext_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: tinytext_column tinytext(255)", + "Id": "c125" + }, + "c126": { + "Name": "blob_column", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: blob_column blob(65535)", + "Id": "c126" + }, + "c127": { + "Name": "mediumblob_column", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumblob_column mediumblob(16777215)", + "Id": "c127" + }, + "c128": { + "Name": "mediumtext_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: mediumtext_column mediumtext(16777215)", + "Id": "c128" + }, + "c129": { + "Name": "longblob_column", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: longblob_column longblob(4294967295)", + "Id": "c129" + }, + "c130": { + "Name": "longtext_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: longtext_column longtext(4294967295)", + "Id": "c130" + }, + "c131": { + "Name": "enum_column", + "T": { + "Name": "STRING", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: enum_column enum(1)", + "Id": "c131" + }, + "c132": { + "Name": "bool_column", + "T": { + "Name": "BOOL", + "Len": 0, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bool_column tinyint(1)", + "Id": "c132" + }, + "c133": { + "Name": "binary_column", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: binary_column binary(20)", + "Id": "c133" + }, + "c134": { + "Name": "varbinary_column", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: varbinary_column varbinary(20)", + "Id": "c134" + }, + "c135": { + "Name": "bit_column", + "T": { + "Name": "BYTES", + "Len": 9223372036854775807, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: bit_column bit(7)", + "Id": "c135" + } + }, + "PrimaryKeys": [ + { + "ColId": "c108", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table AllDatatypeColumns2", + "Id": "t78" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t77": { + "Name": "AllDatatypeColumns", + "Schema": "datatype_test", + "ColIds": [ + "c79", + "c80", + "c81", + "c82", + "c83", + "c84", + "c85", + "c86", + "c87", + "c88", + "c89", + "c90", + "c91", + "c92", + "c93", + "c94", + "c95", + "c96", + "c97", + "c98", + "c99", + "c100", + "c101", + "c102", + "c103", + "c104", + "c105", + "c106", + "c107" + ], + "ColDefs": { + "c100": { + "Name": "longblob_column", + "Type": { + "Name": "longblob", + "Mods": [ + 4294967295 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c100" + }, + "c101": { + "Name": "longtext_column", + "Type": { + "Name": "longtext", + "Mods": [ + 4294967295 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c101" + }, + "c102": { + "Name": "enum_column", + "Type": { + "Name": "enum", + "Mods": [ + 1 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c102" + }, + "c103": { + "Name": "bool_column", + "Type": { + "Name": "tinyint", + "Mods": [ + 1 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c103" + }, + "c104": { + "Name": "other_bool_column", + "Type": { + "Name": "tinyint", + "Mods": [ + 1 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c104" + }, + "c105": { + "Name": "binary_column", + "Type": { + "Name": "binary", + "Mods": [ + 20 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c105" + }, + "c106": { + "Name": "varbinary_column", + "Type": { + "Name": "varbinary", + "Mods": [ + 20 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c106" + }, + "c107": { + "Name": "bit_column", + "Type": { + "Name": "bit", + "Mods": [ + 7 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c107" + }, + "c79": { + "Name": "varchar_column", + "Type": { + "Name": "varchar", + "Mods": [ + 20 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c79" + }, + "c80": { + "Name": "tinyint_column", + "Type": { + "Name": "tinyint", + "Mods": [ + 3 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c80" + }, + "c81": { + "Name": "text_column", + "Type": { + "Name": "text", + "Mods": [ + 65535 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c81" + }, + "c82": { + "Name": "date_column", + "Type": { + "Name": "date", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c82" + }, + "c83": { + "Name": "smallint_column", + "Type": { + "Name": "smallint", + "Mods": [ + 5 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c83" + }, + "c84": { + "Name": "mediumint_column", + "Type": { + "Name": "mediumint", + "Mods": [ + 7 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c84" + }, + "c85": { + "Name": "int_column", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c85" + }, + "c86": { + "Name": "bigint_column", + "Type": { + "Name": "bigint", + "Mods": [ + 19 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c86" + }, + "c87": { + "Name": "float_column", + "Type": { + "Name": "float", + "Mods": [ + 10, + 2 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c87" + }, + "c88": { + "Name": "double_column", + "Type": { + "Name": "double", + "Mods": [ + 22 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c88" + }, + "c89": { + "Name": "decimal_column", + "Type": { + "Name": "decimal", + "Mods": [ + 10, + 2 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c89" + }, + "c90": { + "Name": "datetime_column", + "Type": { + "Name": "datetime", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c90" + }, + "c91": { + "Name": "timestamp_column", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c91" + }, + "c92": { + "Name": "time_column", + "Type": { + "Name": "time", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c92" + }, + "c93": { + "Name": "year_column", + "Type": { + "Name": "year", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c93" + }, + "c94": { + "Name": "char_column", + "Type": { + "Name": "char", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c94" + }, + "c95": { + "Name": "tinyblob_column", + "Type": { + "Name": "tinyblob", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c95" + }, + "c96": { + "Name": "tinytext_column", + "Type": { + "Name": "tinytext", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c96" + }, + "c97": { + "Name": "blob_column", + "Type": { + "Name": "blob", + "Mods": [ + 65535 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c97" + }, + "c98": { + "Name": "mediumblob_column", + "Type": { + "Name": "mediumblob", + "Mods": [ + 16777215 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c98" + }, + "c99": { + "Name": "mediumtext_column", + "Type": { + "Name": "mediumtext", + "Mods": [ + 16777215 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c99" + } + }, + "PrimaryKeys": [ + { + "ColId": "c79", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t77" + }, + "t78": { + "Name": "AllDatatypeColumns2", + "Schema": "datatype_test", + "ColIds": [ + "c108", + "c109", + "c110", + "c111", + "c112", + "c113", + "c114", + "c115", + "c116", + "c117", + "c118", + "c119", + "c120", + "c121", + "c122", + "c123", + "c124", + "c125", + "c126", + "c127", + "c128", + "c129", + "c130", + "c131", + "c132", + "c133", + "c134", + "c135" + ], + "ColDefs": { + "c108": { + "Name": "varchar_column", + "Type": { + "Name": "varchar", + "Mods": [ + 20 + ], + "ArrayBounds": null + }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c108" + }, + "c109": { + "Name": "tinyint_column", + "Type": { + "Name": "tinyint", + "Mods": [ + 3 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c109" + }, + "c110": { + "Name": "text_column", + "Type": { + "Name": "text", + "Mods": [ + 65535 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c110" + }, + "c111": { + "Name": "date_column", + "Type": { + "Name": "date", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c111" + }, + "c112": { + "Name": "smallint_column", + "Type": { + "Name": "smallint", + "Mods": [ + 5 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c112" + }, + "c113": { + "Name": "mediumint_column", + "Type": { + "Name": "mediumint", + "Mods": [ + 7 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c113" + }, + "c114": { + "Name": "int_column", + "Type": { + "Name": "int", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c114" + }, + "c115": { + "Name": "bigint_column", + "Type": { + "Name": "bigint", + "Mods": [ + 19 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c115" + }, + "c116": { + "Name": "float_column", + "Type": { + "Name": "float", + "Mods": [ + 10, + 2 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c116" + }, + "c117": { + "Name": "double_column", + "Type": { + "Name": "double", + "Mods": [ + 22 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c117" + }, + "c118": { + "Name": "decimal_column", + "Type": { + "Name": "decimal", + "Mods": [ + 10, + 2 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c118" + }, + "c119": { + "Name": "datetime_column", + "Type": { + "Name": "datetime", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c119" + }, + "c120": { + "Name": "timestamp_column", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c120" + }, + "c121": { + "Name": "time_column", + "Type": { + "Name": "time", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c121" + }, + "c122": { + "Name": "year_column", + "Type": { + "Name": "year", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c122" + }, + "c123": { + "Name": "char_column", + "Type": { + "Name": "char", + "Mods": [ + 10 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c123" + }, + "c124": { + "Name": "tinyblob_column", + "Type": { + "Name": "tinyblob", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c124" + }, + "c125": { + "Name": "tinytext_column", + "Type": { + "Name": "tinytext", + "Mods": [ + 255 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c125" + }, + "c126": { + "Name": "blob_column", + "Type": { + "Name": "blob", + "Mods": [ + 65535 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c126" + }, + "c127": { + "Name": "mediumblob_column", + "Type": { + "Name": "mediumblob", + "Mods": [ + 16777215 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c127" + }, + "c128": { + "Name": "mediumtext_column", + "Type": { + "Name": "mediumtext", + "Mods": [ + 16777215 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c128" + }, + "c129": { + "Name": "longblob_column", + "Type": { + "Name": "longblob", + "Mods": [ + 4294967295 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c129" + }, + "c130": { + "Name": "longtext_column", + "Type": { + "Name": "longtext", + "Mods": [ + 4294967295 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c130" + }, + "c131": { + "Name": "enum_column", + "Type": { + "Name": "enum", + "Mods": [ + 1 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c131" + }, + "c132": { + "Name": "bool_column", + "Type": { + "Name": "tinyint", + "Mods": [ + 1 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c132" + }, + "c133": { + "Name": "binary_column", + "Type": { + "Name": "binary", + "Mods": [ + 20 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c133" + }, + "c134": { + "Name": "varbinary_column", + "Type": { + "Name": "varbinary", + "Mods": [ + 20 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c134" + }, + "c135": { + "Name": "bit_column", + "Type": { + "Name": "bit", + "Mods": [ + 7 + ], + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c135" + } + }, + "PrimaryKeys": [ + { + "ColId": "c108", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t78" + } + }, + "SchemaIssues": { + "t77": { + "ColumnLevelIssues": { + "c100": [], + "c101": [], + "c102": [], + "c103": [ + 14 + ], + "c104": [ + 14 + ], + "c105": [], + "c106": [], + "c107": [], + "c79": [], + "c80": [ + 14 + ], + "c81": [], + "c82": [ + 14 + ], + "c83": [ + 14 + ], + "c84": [ + 14 + ], + "c85": [ + 14 + ], + "c86": [ + 14 + ], + "c87": [ + 14 + ], + "c88": [ + 14 + ], + "c89": [ + 14 + ], + "c90": [ + 14 + ], + "c91": [ + 14 + ], + "c92": [ + 15 + ], + "c93": [ + 15 + ], + "c94": [], + "c95": [], + "c96": [], + "c97": [], + "c98": [], + "c99": [] + }, + "TableLevelIssues": null + }, + "t78": { + "ColumnLevelIssues": { + "c109": [ + 14 + ], + "c112": [ + 14 + ], + "c113": [ + 14 + ], + "c114": [ + 14 + ], + "c116": [ + 14 + ], + "c119": [ + 13 + ], + "c121": [ + 15 + ], + "c122": [ + 15 + ] + }, + "TableLevelIssues": null + } + }, + "Location": {}, + "TimezoneOffset": "+00:00", + "SpDialect": "google_standard_sql", + "UniquePKey": {}, + "Rules": [], + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false +} \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/spanner-schema.sql new file mode 100644 index 0000000000..2a252dc62e --- /dev/null +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/spanner-schema.sql @@ -0,0 +1,62 @@ +CREATE TABLE AllDatatypeColumns ( + varchar_column STRING(20) NOT NULL, + tinyint_column STRING(MAX), + text_column BYTES(MAX), + date_column STRING(MAX), + smallint_column STRING(MAX), + mediumint_column STRING(MAX), + int_column STRING(MAX), + bigint_column STRING(MAX), + float_column STRING(MAX), + double_column STRING(MAX), + decimal_column STRING(MAX), + datetime_column STRING(MAX), + timestamp_column STRING(MAX), + time_column STRING(MAX), + year_column STRING(MAX), + char_column BYTES(10), + tinyblob_column STRING(MAX), + tinytext_column BYTES(MAX), + blob_column STRING(MAX), + mediumblob_column STRING(MAX), + mediumtext_column BYTES(MAX), + longblob_column STRING(MAX), + longtext_column BYTES(MAX), + enum_column STRING(MAX), + bool_column INT64, + other_bool_column STRING(MAX), + binary_column STRING(MAX), + varbinary_column STRING(MAX), + bit_column STRING(MAX), +) PRIMARY KEY(varchar_column); + +CREATE TABLE AllDatatypeColumns2 ( + varchar_column STRING(20) NOT NULL, + tinyint_column INT64, + text_column STRING(MAX), + date_column DATE, + smallint_column INT64, + mediumint_column INT64, + int_column INT64, + bigint_column INT64, + float_column FLOAT64, + double_column FLOAT64, + decimal_column NUMERIC, + datetime_column TIMESTAMP, + timestamp_column TIMESTAMP, + time_column STRING(MAX), + year_column STRING(MAX), + char_column STRING(10), + tinyblob_column BYTES(MAX), + tinytext_column STRING(MAX), + blob_column BYTES(MAX), + mediumblob_column BYTES(MAX), + mediumtext_column STRING(MAX), + longblob_column BYTES(MAX), + longtext_column STRING(MAX), + enum_column STRING(MAX), + bool_column BOOL, + binary_column BYTES(MAX), + varbinary_column BYTES(MAX), + bit_column BYTES(MAX), +) PRIMARY KEY(varchar_column); \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-schema.sql b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-schema.sql index e4d1f0103a..e89cb8de72 100644 --- a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-schema.sql +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-schema.sql @@ -23,72 +23,6 @@ CREATE TABLE `Category` ( PRIMARY KEY (`category_id`) ); --- Database postgres_test in shreya-mysql -CREATE TABLE `AllDatatypeColumns` ( - `varchar_column` varchar(20) NOT NULL, -- To: varchar_column BYTES(20) - `tinyint_column` tinyint, -- To: tinyint_column STRING(MAX) - `text_column` text, -- To: text_column BYTES(MAX) - `date_column` date, -- To: date_column STRING(MAX) - `smallint_column` smallint, -- To: smallint_column STRING(MAX) - `mediumint_column` mediumint, -- To: mediumint_column STRING(MAX) - `int_column` int, -- To: int_column STRING(MAX) - `bigint_column` bigint, -- To: bigint_column STRING(MAX) - `float_column` float(10,2), -- To: float_column STRING(MAX) - `double_column` double, -- To: double_column STRING(MAX) - `decimal_column` decimal(10,2), -- To: decimal_column STRING(MAX) - `datetime_column` datetime, -- To: datetime_column STRING(MAX) - `timestamp_column` timestamp, -- To: timestamp_column STRING(MAX) - `time_column` time, -- To: time_column STRING(MAX) - `year_column` year, -- To: year_column STRING(MAX) - `char_column` char(10), -- To: char_column BYTES(10) - `tinyblob_column` tinyblob, -- To: tinyblob_column STRING(MAX) - `tinytext_column` tinytext, -- To: tinytext_column BYTES(MAX) - `blob_column` blob, -- To: blob_column STRING(MAX) - `mediumblob_column` mediumblob, -- To: mediumblob_column STRING(MAX) - `mediumtext_column` mediumtext, -- To: mediumtext_column BYTES(MAX) - `longblob_column` longblob, -- To: longblob_column STRING(MAX) - `longtext_column` longtext, -- To: longtext_column BYTES(MAX) - `enum_column` enum('1','2','3'), -- To: enum_column STRING(MAX) - `bool_column` tinyint(1), -- To: bool_column INT64 - `other_bool_column` tinyint(1), -- To: other_bool_column STRING(MAX) - `binary_column` binary(20), -- To: binary_column STRING(MAX) - `varbinary_column` varbinary(20), -- To: varbinary_column STRING(MAX) - `bit_column` bit(7), -- To: bit_column STRING(MAX) - PRIMARY KEY (`varchar_column`) -); - -CREATE TABLE `AllDatatypeColumns2` ( - `varchar_column` varchar(20) NOT NULL, -- To: varchar_column BYTES(20) - `tinyint_column` tinyint, -- To: tinyint_column INT64 - `text_column` text, -- To: text_column STRING(MAX) - `date_column` date, -- To: date_column DATE - `smallint_column` smallint, -- To: smallint_column INT64 - `mediumint_column` mediumint, -- To: mediumint_column INT64 - `int_column` int, -- To: int_column INT64 - `bigint_column` bigint, -- To: bigint_column INT64 - `float_column` float(10,2), -- To: float_column FLOAT64 - `double_column` double, -- To: double_column FLOAT64 - `decimal_column` decimal(10,2), -- To: decimal_column NUMERIC - `datetime_column` datetime, -- To: datetime_column TIMESTAMP - `timestamp_column` timestamp, -- To: timestamp_column TIMESTAMP - `time_column` time, -- To: time_column STRING(MAX) - `year_column` year, -- To: year_column STRING(MAX) - `char_column` char(10), -- To: char_column STRING(10) - `tinyblob_column` tinyblob, -- To: tinyblob_column BYTES(MAX) - `tinytext_column` tinytext, -- To: tinytext_column STRING(MAX) - `blob_column` blob, -- To: blob_column BYTES(MAX) - `mediumblob_column` mediumblob, -- To: mediumblob_column BYTES(MAX) - `mediumtext_column` mediumtext, -- To: mediumtext_column STRING(MAX) - `longblob_column` longblob, -- To: longblob_column BYTES(MAX) - `longtext_column` longtext, -- To: longtext_column STRING(MAX) - `enum_column` enum('1','2','3'), -- To: enum_column STRING(MAX) - `bool_column` tinyint(1), -- To: bool_column BOOL - `binary_column` binary(20), -- To: binary_column BYTES(MAX) - `varbinary_column` varbinary(20), -- To: varbinary_column BYTES(MAX) - `bit_column` bit(7), -- To: bit_column BYTES(MAX) - PRIMARY KEY (`varchar_column`) -); - CREATE TABLE `Authors` ( `author_id` int NOT NULL, `name` varchar(200), diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-session.json b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-session.json index 63f30b74a3..2dcf70719a 100644 --- a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-session.json +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/mysql-session.json @@ -2,22 +2,66 @@ "SessionName": "NewSession", "EditorName": "", "DatabaseType": "mysql", - "DatabaseName": "l1", + "DatabaseName": "simpleit", "Dialect": "google_standard_sql", "Notes": null, "Tags": null, "SpSchema": { - "t77": { - "Name": "Articles", + "t136": { + "Name": "Category", "ColIds": [ - "c83", - "c84", - "c85", - "c86" + "c142", + "c143" ], "ShardIdColumn": "", "ColDefs": { - "c83": { + "c142": { + "Name": "category_id", + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: category_id tinyint(3)", + "Id": "c142" + }, + "c143": { + "Name": "full_name", + "T": { + "Name": "STRING", + "Len": 25, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name varchar(25)", + "Id": "c143" + } + }, + "PrimaryKeys": [ + { + "ColId": "c142", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table Category", + "Id": "t136" + }, + "t137": { + "Name": "Movie", + "ColIds": [ + "c156", + "c157", + "c158", + "c159" + ], + "ShardIdColumn": "", + "ColDefs": { + "c156": { "Name": "id", "T": { "Name": "INT64", @@ -26,9 +70,9 @@ }, "NotNull": true, "Comment": "From: id int(10)", - "Id": "c83" + "Id": "c156" }, - "c84": { + "c157": { "Name": "name", "T": { "Name": "STRING", @@ -37,75 +81,99 @@ }, "NotNull": false, "Comment": "From: name varchar(200)", - "Id": "c84" + "Id": "c157" }, - "c85": { - "Name": "published_date", + "c158": { + "Name": "actor", "T": { - "Name": "DATE", + "Name": "NUMERIC", "Len": 0, "IsArray": false }, "NotNull": false, - "Comment": "From: published_date date", - "Id": "c85" + "Comment": "From: actor decimal(65,30)", + "Id": "c158" }, - "c86": { - "Name": "author_id", + "c159": { + "Name": "startTime", "T": { - "Name": "INT64", + "Name": "TIMESTAMP", "Len": 0, "IsArray": false }, "NotNull": false, - "Comment": "From: author_id int(10)", - "Id": "c86" + "Comment": "From: startTime timestamp", + "Id": "c159" } }, "PrimaryKeys": [ { - "ColId": "c83", - "Desc": false, - "Order": 2 - }, - { - "ColId": "c86", + "ColId": "c156", "Desc": false, "Order": 1 } ], - "ForeignKeys": [], - "Indexes": [ - { + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table Movie", + "Id": "t137" + }, + "t138": { + "Name": "Authors", + "ColIds": [ + "c147", + "c148" + ], + "ShardIdColumn": "", + "ColDefs": { + "c147": { "Name": "author_id", - "TableId": "t77", - "Unique": false, - "Keys": [ - { - "ColId": "c86", - "Desc": false, - "Order": 1 - } - ], - "Id": "i88", - "StoredColumnIds": null + "T": { + "Name": "INT64", + "Len": 0, + "IsArray": false + }, + "NotNull": true, + "Comment": "From: author_id int(10)", + "Id": "c147" + }, + "c148": { + "Name": "name", + "T": { + "Name": "STRING", + "Len": 200, + "IsArray": false + }, + "NotNull": false, + "Comment": "From: name varchar(200)", + "Id": "c148" + } + }, + "PrimaryKeys": [ + { + "ColId": "c147", + "Desc": false, + "Order": 1 } ], - "ParentId": "t81", - "Comment": "Spanner schema for source table Articles", - "Id": "t77" + "ForeignKeys": null, + "Indexes": null, + "ParentId": "", + "Comment": "Spanner schema for source table Authors", + "Id": "t138" }, - "t78": { - "Name": "Movie", + "t139": { + "Name": "Articles", "ColIds": [ - "c100", - "c101", - "c102", - "c103" + "c152", + "c153", + "c154", + "c155" ], "ShardIdColumn": "", "ColDefs": { - "c100": { + "c152": { "Name": "id", "T": { "Name": "INT64", @@ -114,9 +182,9 @@ }, "NotNull": true, "Comment": "From: id int(10)", - "Id": "c100" + "Id": "c152" }, - "c101": { + "c153": { "Name": "name", "T": { "Name": "STRING", @@ -125,54 +193,74 @@ }, "NotNull": false, "Comment": "From: name varchar(200)", - "Id": "c101" + "Id": "c153" }, - "c102": { - "Name": "actor", + "c154": { + "Name": "published_date", "T": { - "Name": "NUMERIC", + "Name": "DATE", "Len": 0, "IsArray": false }, "NotNull": false, - "Comment": "From: actor decimal(65,30)", - "Id": "c102" + "Comment": "From: published_date date", + "Id": "c154" }, - "c103": { - "Name": "startTime", + "c155": { + "Name": "author_id", "T": { - "Name": "TIMESTAMP", + "Name": "INT64", "Len": 0, "IsArray": false }, "NotNull": false, - "Comment": "From: startTime timestamp", - "Id": "c103" + "Comment": "From: author_id int(10)", + "Id": "c155" } }, "PrimaryKeys": [ { - "ColId": "c100", + "ColId": "c152", + "Desc": false, + "Order": 2 + }, + { + "ColId": "c155", "Desc": false, "Order": 1 } ], - "ForeignKeys": null, - "Indexes": null, - "ParentId": "", - "Comment": "Spanner schema for source table Movie", - "Id": "t78" + "ForeignKeys": [], + "Indexes": [ + { + "Name": "author_id", + "TableId": "t139", + "Unique": false, + "Keys": [ + { + "ColId": "c155", + "Desc": false, + "Order": 1 + } + ], + "Id": "i167", + "StoredColumnIds": null + } + ], + "ParentId": "t138", + "Comment": "Spanner schema for source table Articles", + "Id": "t139" }, - "t79": { + "t140": { "Name": "Books", "ColIds": [ - "c89", - "c90", - "c91" + "c149", + "c150", + "c151" ], "ShardIdColumn": "", "ColDefs": { - "c89": { + "c149": { "Name": "id", "T": { "Name": "INT64", @@ -181,9 +269,9 @@ }, "NotNull": true, "Comment": "From: id int(10)", - "Id": "c89" + "Id": "c149" }, - "c90": { + "c150": { "Name": "title", "T": { "Name": "STRING", @@ -192,9 +280,9 @@ }, "NotNull": false, "Comment": "From: title varchar(200)", - "Id": "c90" + "Id": "c150" }, - "c91": { + "c151": { "Name": "author_id", "T": { "Name": "INT64", @@ -203,12 +291,12 @@ }, "NotNull": false, "Comment": "From: author_id int(10)", - "Id": "c91" + "Id": "c151" } }, "PrimaryKeys": [ { - "ColId": "c89", + "ColId": "c149", "Desc": false, "Order": 1 } @@ -217,48 +305,48 @@ { "Name": "Books_ibfk_1", "ColIds": [ - "c91" + "c151" ], - "ReferTableId": "t81", + "ReferTableId": "t138", "ReferColumnIds": [ - "c98" + "c147" ], - "Id": "f87" + "Id": "f145" } ], "Indexes": [ { "Name": "author_id_6", - "TableId": "t79", + "TableId": "t140", "Unique": false, "Keys": [ { - "ColId": "c91", + "ColId": "c151", "Desc": false, "Order": 1 } ], - "Id": "i104", + "Id": "i166", "StoredColumnIds": null } ], "ParentId": "", "Comment": "Spanner schema for source table Books", - "Id": "t79" + "Id": "t140" }, - "t80": { + "t141": { "Name": "Users", "ColIds": [ - "c92", - "c93", - "c94", - "c95", - "c96", - "c97" + "c160", + "c161", + "c162", + "c163", + "c164", + "c165" ], "ShardIdColumn": "", "ColDefs": { - "c92": { + "c160": { "Name": "id", "T": { "Name": "INT64", @@ -267,9 +355,9 @@ }, "NotNull": true, "Comment": "From: id int(10)", - "Id": "c92" + "Id": "c160" }, - "c93": { + "c161": { "Name": "name", "T": { "Name": "STRING", @@ -278,9 +366,9 @@ }, "NotNull": false, "Comment": "From: name varchar(200)", - "Id": "c93" + "Id": "c161" }, - "c94": { + "c162": { "Name": "age_spanner", "T": { "Name": "INT64", @@ -289,9 +377,9 @@ }, "NotNull": false, "Comment": "From: age bigint(19)", - "Id": "c94" + "Id": "c162" }, - "c95": { + "c163": { "Name": "subscribed", "T": { "Name": "BOOL", @@ -300,9 +388,9 @@ }, "NotNull": false, "Comment": "From: subscribed bit(1)", - "Id": "c95" + "Id": "c163" }, - "c96": { + "c164": { "Name": "plan", "T": { "Name": "STRING", @@ -311,9 +399,9 @@ }, "NotNull": false, "Comment": "From: plan char(1)", - "Id": "c96" + "Id": "c164" }, - "c97": { + "c165": { "Name": "startDate", "T": { "Name": "DATE", @@ -322,12 +410,12 @@ }, "NotNull": false, "Comment": "From: startDate date", - "Id": "c97" + "Id": "c165" } }, "PrimaryKeys": [ { - "ColId": "c92", + "ColId": "c160", "Desc": false, "Order": 1 } @@ -336,66 +424,101 @@ "Indexes": null, "ParentId": "", "Comment": "Spanner schema for source table Users", - "Id": "t80" - }, - "t81": { - "Name": "Authors", + "Id": "t141" + } + }, + "SyntheticPKeys": {}, + "SrcSchema": { + "t136": { + "Name": "Category", + "Schema": "simpleit", "ColIds": [ - "c98", - "c99" + "c142", + "c143", + "c144" ], - "ShardIdColumn": "", "ColDefs": { - "c98": { - "Name": "author_id", - "T": { - "Name": "INT64", - "Len": 0, - "IsArray": false + "c142": { + "Name": "category_id", + "Type": { + "Name": "tinyint", + "Mods": [ + 3 + ], + "ArrayBounds": null }, "NotNull": true, - "Comment": "From: author_id int(10)", - "Id": "c98" + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c142" }, - "c99": { + "c143": { "Name": "name", - "T": { - "Name": "STRING", - "Len": 200, - "IsArray": false + "Type": { + "Name": "varchar", + "Mods": [ + 25 + ], + "ArrayBounds": null }, "NotNull": false, - "Comment": "From: name varchar(200)", - "Id": "c99" + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c143" + }, + "c144": { + "Name": "last_update", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c144" } }, "PrimaryKeys": [ { - "ColId": "c98", + "ColId": "c142", "Desc": false, "Order": 1 } ], "ForeignKeys": null, "Indexes": null, - "ParentId": "", - "Comment": "Spanner schema for source table Authors", - "Id": "t81" - } - }, - "SyntheticPKeys": {}, - "SrcSchema": { - "t77": { - "Name": "Articles", - "Schema": "l1", + "Id": "t136" + }, + "t137": { + "Name": "Movie", + "Schema": "simpleit", "ColIds": [ - "c83", - "c84", - "c85", - "c86" + "c156", + "c157", + "c158", + "c159" ], "ColDefs": { - "c83": { + "c156": { "Name": "id", "Type": { "Name": "int", @@ -413,9 +536,9 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c83" + "Id": "c156" }, - "c84": { + "c157": { "Name": "name", "Type": { "Name": "varchar", @@ -433,13 +556,16 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c84" + "Id": "c157" }, - "c85": { - "Name": "published_date", + "c158": { + "Name": "actor", "Type": { - "Name": "date", - "Mods": null, + "Name": "decimal", + "Mods": [ + 65, + 30 + ], "ArrayBounds": null }, "NotNull": false, @@ -451,9 +577,47 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c85" + "Id": "c158" }, - "c86": { + "c159": { + "Name": "startTime", + "Type": { + "Name": "timestamp", + "Mods": null, + "ArrayBounds": null + }, + "NotNull": false, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c159" + } + }, + "PrimaryKeys": [ + { + "ColId": "c156", + "Desc": false, + "Order": 1 + } + ], + "ForeignKeys": null, + "Indexes": null, + "Id": "t137" + }, + "t138": { + "Name": "Authors", + "Schema": "simpleit", + "ColIds": [ + "c147", + "c148" + ], + "ColDefs": { + "c147": { "Name": "author_id", "Type": { "Name": "int", @@ -462,6 +626,26 @@ ], "ArrayBounds": null }, + "NotNull": true, + "Ignored": { + "Check": false, + "Identity": false, + "Default": false, + "Exclusion": false, + "ForeignKey": false, + "AutoIncrement": false + }, + "Id": "c147" + }, + "c148": { + "Name": "name", + "Type": { + "Name": "varchar", + "Mods": [ + 200 + ], + "ArrayBounds": null + }, "NotNull": false, "Ignored": { "Check": false, @@ -471,59 +655,31 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c86" + "Id": "c148" } }, "PrimaryKeys": [ { - "ColId": "c83", + "ColId": "c147", "Desc": false, "Order": 1 } ], - "ForeignKeys": [ - { - "Name": "Articles_ibfk_1", - "ColIds": [ - "c86" - ], - "ReferTableId": "t81", - "ReferColumnIds": [ - "c98" - ], - "OnDelete": "", - "OnUpdate": "", - "Id": "f82" - } - ], - "Indexes": [ - { - "Name": "author_id", - "Unique": false, - "Keys": [ - { - "ColId": "c86", - "Desc": false, - "Order": 1 - } - ], - "Id": "i88", - "StoredColumnIds": null - } - ], - "Id": "t77" + "ForeignKeys": null, + "Indexes": null, + "Id": "t138" }, - "t78": { - "Name": "Movie", - "Schema": "l1", + "t139": { + "Name": "Articles", + "Schema": "simpleit", "ColIds": [ - "c100", - "c101", - "c102", - "c103" + "c152", + "c153", + "c154", + "c155" ], "ColDefs": { - "c100": { + "c152": { "Name": "id", "Type": { "Name": "int", @@ -541,9 +697,9 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c100" + "Id": "c152" }, - "c101": { + "c153": { "Name": "name", "Type": { "Name": "varchar", @@ -561,16 +717,13 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c101" + "Id": "c153" }, - "c102": { - "Name": "actor", + "c154": { + "Name": "published_date", "Type": { - "Name": "decimal", - "Mods": [ - 65, - 30 - ], + "Name": "date", + "Mods": null, "ArrayBounds": null }, "NotNull": false, @@ -582,13 +735,15 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c102" + "Id": "c154" }, - "c103": { - "Name": "startTime", + "c155": { + "Name": "author_id", "Type": { - "Name": "timestamp", - "Mods": null, + "Name": "int", + "Mods": [ + 10 + ], "ArrayBounds": null }, "NotNull": false, @@ -600,30 +755,58 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c103" + "Id": "c155" } }, "PrimaryKeys": [ { - "ColId": "c100", + "ColId": "c152", "Desc": false, "Order": 1 } ], - "ForeignKeys": null, - "Indexes": null, - "Id": "t78" + "ForeignKeys": [ + { + "Name": "Articles_ibfk_1", + "ColIds": [ + "c155" + ], + "ReferTableId": "t138", + "ReferColumnIds": [ + "c147" + ], + "OnDelete": "", + "OnUpdate": "", + "Id": "f146" + } + ], + "Indexes": [ + { + "Name": "author_id", + "Unique": false, + "Keys": [ + { + "ColId": "c155", + "Desc": false, + "Order": 1 + } + ], + "Id": "i167", + "StoredColumnIds": null + } + ], + "Id": "t139" }, - "t79": { + "t140": { "Name": "Books", - "Schema": "l1", + "Schema": "simpleit", "ColIds": [ - "c89", - "c90", - "c91" + "c149", + "c150", + "c151" ], "ColDefs": { - "c89": { + "c149": { "Name": "id", "Type": { "Name": "int", @@ -641,9 +824,9 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c89" + "Id": "c149" }, - "c90": { + "c150": { "Name": "title", "Type": { "Name": "varchar", @@ -661,9 +844,9 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c90" + "Id": "c150" }, - "c91": { + "c151": { "Name": "author_id", "Type": { "Name": "int", @@ -681,12 +864,12 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c91" + "Id": "c151" } }, "PrimaryKeys": [ { - "ColId": "c89", + "ColId": "c149", "Desc": false, "Order": 1 } @@ -695,47 +878,47 @@ { "Name": "Books_ibfk_1", "ColIds": [ - "c91" + "c151" ], - "ReferTableId": "t81", + "ReferTableId": "t138", "ReferColumnIds": [ - "c98" + "c147" ], "OnDelete": "", "OnUpdate": "", - "Id": "f87" + "Id": "f145" } ], "Indexes": [ { - "Name": "author_id", + "Name": "author_id_6", "Unique": false, "Keys": [ { - "ColId": "c91", + "ColId": "c151", "Desc": false, "Order": 1 } ], - "Id": "i104", + "Id": "i166", "StoredColumnIds": null } ], - "Id": "t79" + "Id": "t140" }, - "t80": { + "t141": { "Name": "Users", - "Schema": "l1", + "Schema": "simpleit", "ColIds": [ - "c92", - "c93", - "c94", - "c95", - "c96", - "c97" + "c160", + "c161", + "c162", + "c163", + "c164", + "c165" ], "ColDefs": { - "c92": { + "c160": { "Name": "id", "Type": { "Name": "int", @@ -753,9 +936,9 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c92" + "Id": "c160" }, - "c93": { + "c161": { "Name": "name", "Type": { "Name": "varchar", @@ -773,9 +956,9 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c93" + "Id": "c161" }, - "c94": { + "c162": { "Name": "age", "Type": { "Name": "bigint", @@ -793,9 +976,9 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c94" + "Id": "c162" }, - "c95": { + "c163": { "Name": "subscribed", "Type": { "Name": "bit", @@ -813,9 +996,9 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c95" + "Id": "c163" }, - "c96": { + "c164": { "Name": "plan", "Type": { "Name": "char", @@ -833,9 +1016,9 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c96" + "Id": "c164" }, - "c97": { + "c165": { "Name": "startDate", "Type": { "Name": "date", @@ -851,86 +1034,99 @@ "ForeignKey": false, "AutoIncrement": false }, - "Id": "c97" + "Id": "c165" } }, "PrimaryKeys": [ { - "ColId": "c92", + "ColId": "c160", "Desc": false, "Order": 1 } ], "ForeignKeys": null, "Indexes": null, - "Id": "t80" + "Id": "t141" + } + }, + "SchemaIssues": { + "t136": { + "ColumnLevelIssues": { + "c142": [ + 14 + ], + "c143": [] + }, + "TableLevelIssues": null }, - "t81": { - "Name": "Authors", - "Schema": "l1", - "ColIds": [ - "c98", - "c99" - ], - "ColDefs": { - "c98": { - "Name": "author_id", - "Type": { - "Name": "int", - "Mods": [ - 10 - ], - "ArrayBounds": null - }, - "NotNull": true, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c98" - }, - "c99": { - "Name": "name", - "Type": { - "Name": "varchar", - "Mods": [ - 200 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c99" - } + "t137": { + "ColumnLevelIssues": { + "c156": [ + 14 + ], + "c157": [], + "c158": [], + "c159": [] }, - "PrimaryKeys": [ - { - "ColId": "c98", - "Desc": false, - "Order": 1 - } - ], - "ForeignKeys": null, - "Indexes": null, - "Id": "t81" + "TableLevelIssues": null + }, + "t138": { + "ColumnLevelIssues": { + "c147": [ + 14 + ], + "c148": [] + }, + "TableLevelIssues": null + }, + "t139": { + "ColumnLevelIssues": { + "c152": [ + 14 + ], + "c153": [], + "c154": [], + "c155": [ + 14, + 19, + 19, + 19 + ] + }, + "TableLevelIssues": null + }, + "t140": { + "ColumnLevelIssues": { + "c149": [ + 14 + ], + "c151": [ + 14 + ] + }, + "TableLevelIssues": null + }, + "t141": { + "ColumnLevelIssues": { + "c160": [ + 14 + ], + "c161": [], + "c162": [], + "c163": [], + "c164": [], + "c165": [] + }, + "TableLevelIssues": null } }, - "SchemaIssues": {}, "Location": {}, "TimezoneOffset": "+00:00", "SpDialect": "google_standard_sql", "UniquePKey": {}, "Rules": [], - "IsSharded": false + "IsSharded": false, + "SpRegion": "", + "ResourceValidation": false, + "UI": false } \ No newline at end of file diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/spanner-schema.sql index 8d457ef2f4..da6459c2b7 100644 --- a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/spanner-schema.sql +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/spanner-schema.sql @@ -1,66 +1,3 @@ -CREATE TABLE AllDatatypeColumns ( - varchar_column STRING(20) NOT NULL, - tinyint_column STRING(MAX), - text_column BYTES(MAX), - date_column STRING(MAX), - smallint_column STRING(MAX), - mediumint_column STRING(MAX), - int_column STRING(MAX), - bigint_column STRING(MAX), - float_column STRING(MAX), - double_column STRING(MAX), - decimal_column STRING(MAX), - datetime_column STRING(MAX), - timestamp_column STRING(MAX), - time_column STRING(MAX), - year_column STRING(MAX), - char_column BYTES(10), - tinyblob_column STRING(MAX), - tinytext_column BYTES(MAX), - blob_column STRING(MAX), - mediumblob_column STRING(MAX), - mediumtext_column BYTES(MAX), - longblob_column STRING(MAX), - longtext_column BYTES(MAX), - enum_column STRING(MAX), - bool_column INT64, - other_bool_column STRING(MAX), - binary_column STRING(MAX), - varbinary_column STRING(MAX), - bit_column STRING(MAX), -) PRIMARY KEY(varchar_column); - -CREATE TABLE AllDatatypeColumns2 ( - varchar_column STRING(20) NOT NULL, - tinyint_column INT64, - text_column STRING(MAX), - date_column DATE, - smallint_column INT64, - mediumint_column INT64, - int_column INT64, - bigint_column INT64, - float_column FLOAT64, - double_column FLOAT64, - decimal_column NUMERIC, - datetime_column TIMESTAMP, - timestamp_column TIMESTAMP, - time_column STRING(MAX), - year_column STRING(MAX), - char_column STRING(10), - tinyblob_column BYTES(MAX), - tinytext_column STRING(MAX), - blob_column BYTES(MAX), - mediumblob_column BYTES(MAX), - mediumtext_column STRING(MAX), - longblob_column BYTES(MAX), - longtext_column STRING(MAX), - enum_column STRING(MAX), - bool_column BOOL, - binary_column BYTES(MAX), - varbinary_column BYTES(MAX), - bit_column BYTES(MAX), -) PRIMARY KEY(varchar_column); - CREATE TABLE Category ( category_id INT64 NOT NULL, full_name STRING(25), From 6f80f9d723f0b053c4b5471431762cdd90ce0cc0 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Thu, 21 Mar 2024 10:21:59 +0530 Subject: [PATCH 09/11] fixing checkstyle --- .../v2/templates/DataStreamToSpannerDatatypeIT.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerDatatypeIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerDatatypeIT.java index e715b95c7f..3a126b36d7 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerDatatypeIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerDatatypeIT.java @@ -21,7 +21,11 @@ import com.google.cloud.teleport.metadata.TemplateIntegrationTest; import java.io.IOException; import java.time.Duration; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import org.apache.beam.it.common.PipelineLauncher; import org.apache.beam.it.common.PipelineOperator; import org.apache.beam.it.common.utils.ResourceManagerUtils; From ca14baf9ed0e8e42ff8234188c81dad8dcd24e0e Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Thu, 21 Mar 2024 12:26:07 +0530 Subject: [PATCH 10/11] addressing comments --- .../DataStreamToSpannerDatatypeIT/spanner-schema.sql | 4 ++-- .../resources/DataStreamToSpannerSimpleIT/spanner-schema.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/spanner-schema.sql index 2a252dc62e..12dd89993f 100644 --- a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/spanner-schema.sql +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/spanner-schema.sql @@ -1,4 +1,4 @@ -CREATE TABLE AllDatatypeColumns ( +CREATE TABLE IF NOT EXISTS AllDatatypeColumns ( varchar_column STRING(20) NOT NULL, tinyint_column STRING(MAX), text_column BYTES(MAX), @@ -30,7 +30,7 @@ CREATE TABLE AllDatatypeColumns ( bit_column STRING(MAX), ) PRIMARY KEY(varchar_column); -CREATE TABLE AllDatatypeColumns2 ( +CREATE TABLE IF NOT EXISTS AllDatatypeColumns2 ( varchar_column STRING(20) NOT NULL, tinyint_column INT64, text_column STRING(MAX), diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/spanner-schema.sql index da6459c2b7..9336258954 100644 --- a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/spanner-schema.sql +++ b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerSimpleIT/spanner-schema.sql @@ -1,9 +1,9 @@ -CREATE TABLE Category ( +CREATE TABLE IF NOT EXISTS Category ( category_id INT64 NOT NULL, full_name STRING(25), ) PRIMARY KEY(category_id); -CREATE TABLE Movie ( +CREATE TABLE IF NOT EXISTS Movie ( id INT64 NOT NULL, name STRING(200), actor NUMERIC, From f8653efac3e9d0e9c2a837e278a9ede7562c98d3 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Tue, 2 Apr 2024 14:04:48 +0530 Subject: [PATCH 11/11] remove session file from datatype test case --- .../DataStreamToSpannerDatatypeIT.java | 5 +- .../templates/DataStreamToSpannerITBase.java | 12 +- .../mysql-session.json | 2059 ----------------- 3 files changed, 10 insertions(+), 2066 deletions(-) delete mode 100644 v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-session.json diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerDatatypeIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerDatatypeIT.java index 3a126b36d7..6c886aadb3 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerDatatypeIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerDatatypeIT.java @@ -50,9 +50,6 @@ public class DataStreamToSpannerDatatypeIT extends DataStreamToSpannerITBase { private static final Logger LOG = LoggerFactory.getLogger(DataStreamToSpannerDatatypeIT.class); - private static final String SESSION_FILE_RESOURCE = - "DataStreamToSpannerDatatypeIT/mysql-session.json"; - private static final String SPANNER_DDL_RESOURCE = "DataStreamToSpannerDatatypeIT/spanner-schema.sql"; @@ -83,7 +80,7 @@ public void setUp() throws IOException { jobInfo = launchDataflowJob( getClass().getSimpleName(), - SESSION_FILE_RESOURCE, + null, null, "DatatypeIT", spannerResourceManager, diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java index cf66b57b2a..8c8b6cb29b 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerITBase.java @@ -165,8 +165,11 @@ protected LaunchInfo launchDataflowJob( Map jobParameters) throws IOException { - gcsClient.uploadArtifact( - gcsPathPrefix + "/session.json", Resources.getResource(sessionFileResourceName).getPath()); + if (sessionFileResourceName != null) { + gcsClient.uploadArtifact( + gcsPathPrefix + "/session.json", + Resources.getResource(sessionFileResourceName).getPath()); + } if (transformationContextFileResourceName != null) { gcsClient.uploadArtifact( @@ -197,7 +200,6 @@ protected LaunchInfo launchDataflowJob( put("databaseId", spannerResourceManager.getDatabaseId()); put("projectId", PROJECT); put("deadLetterQueueDirectory", getGcsPath(gcsPathPrefix + "/dlq/")); - put("sessionFilePath", getGcsPath(gcsPathPrefix + "/session.json")); put("gcsPubSubSubscription", subscription.toString()); put("dlqGcsPubSubSubscription", dlqSubscription.toString()); put("datastreamSourceType", "mysql"); @@ -205,6 +207,10 @@ protected LaunchInfo launchDataflowJob( } }; + if (sessionFileResourceName != null) { + params.put("sessionFilePath", getGcsPath(gcsPathPrefix + "/session.json")); + } + if (transformationContextFileResourceName != null) { params.put( "transformationContextFilePath", diff --git a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-session.json b/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-session.json deleted file mode 100644 index 0c73f55b78..0000000000 --- a/v2/datastream-to-spanner/src/test/resources/DataStreamToSpannerDatatypeIT/mysql-session.json +++ /dev/null @@ -1,2059 +0,0 @@ -{ - "SessionName": "NewSession", - "EditorName": "", - "DatabaseType": "mysql", - "DatabaseName": "datatype_test", - "Dialect": "google_standard_sql", - "Notes": null, - "Tags": null, - "SpSchema": { - "t77": { - "Name": "AllDatatypeColumns", - "ColIds": [ - "c79", - "c80", - "c81", - "c82", - "c83", - "c84", - "c85", - "c86", - "c87", - "c88", - "c89", - "c90", - "c91", - "c92", - "c93", - "c94", - "c95", - "c96", - "c97", - "c98", - "c99", - "c100", - "c101", - "c102", - "c103", - "c104", - "c105", - "c106", - "c107" - ], - "ShardIdColumn": "", - "ColDefs": { - "c100": { - "Name": "longblob_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: longblob_column longblob(4294967295)", - "Id": "c100" - }, - "c101": { - "Name": "longtext_column", - "T": { - "Name": "BYTES", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: longtext_column longtext(4294967295)", - "Id": "c101" - }, - "c102": { - "Name": "enum_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: enum_column enum(1)", - "Id": "c102" - }, - "c103": { - "Name": "bool_column", - "T": { - "Name": "INT64", - "Len": 0, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: bool_column tinyint(1)", - "Id": "c103" - }, - "c104": { - "Name": "other_bool_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: other_bool_column tinyint(1)", - "Id": "c104" - }, - "c105": { - "Name": "binary_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: binary_column binary(20)", - "Id": "c105" - }, - "c106": { - "Name": "varbinary_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: varbinary_column varbinary(20)", - "Id": "c106" - }, - "c107": { - "Name": "bit_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: bit_column bit(7)", - "Id": "c107" - }, - "c79": { - "Name": "varchar_column", - "T": { - "Name": "BYTES", - "Len": 20, - "IsArray": false - }, - "NotNull": true, - "Comment": "From: varchar_column varchar(20)", - "Id": "c79" - }, - "c80": { - "Name": "tinyint_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: tinyint_column tinyint(3)", - "Id": "c80" - }, - "c81": { - "Name": "text_column", - "T": { - "Name": "BYTES", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: text_column text(65535)", - "Id": "c81" - }, - "c82": { - "Name": "date_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: date_column date", - "Id": "c82" - }, - "c83": { - "Name": "smallint_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: smallint_column smallint(5)", - "Id": "c83" - }, - "c84": { - "Name": "mediumint_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: mediumint_column mediumint(7)", - "Id": "c84" - }, - "c85": { - "Name": "int_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: int_column int(10)", - "Id": "c85" - }, - "c86": { - "Name": "bigint_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: bigint_column bigint(19)", - "Id": "c86" - }, - "c87": { - "Name": "float_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: float_column float(10,2)", - "Id": "c87" - }, - "c88": { - "Name": "double_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: double_column double(22)", - "Id": "c88" - }, - "c89": { - "Name": "decimal_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: decimal_column decimal(10,2)", - "Id": "c89" - }, - "c90": { - "Name": "datetime_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: datetime_column datetime", - "Id": "c90" - }, - "c91": { - "Name": "timestamp_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: timestamp_column timestamp", - "Id": "c91" - }, - "c92": { - "Name": "time_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: time_column time", - "Id": "c92" - }, - "c93": { - "Name": "year_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: year_column year", - "Id": "c93" - }, - "c94": { - "Name": "char_column", - "T": { - "Name": "BYTES", - "Len": 10, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: char_column char(10)", - "Id": "c94" - }, - "c95": { - "Name": "tinyblob_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: tinyblob_column tinyblob(255)", - "Id": "c95" - }, - "c96": { - "Name": "tinytext_column", - "T": { - "Name": "BYTES", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: tinytext_column tinytext(255)", - "Id": "c96" - }, - "c97": { - "Name": "blob_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: blob_column blob(65535)", - "Id": "c97" - }, - "c98": { - "Name": "mediumblob_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: mediumblob_column mediumblob(16777215)", - "Id": "c98" - }, - "c99": { - "Name": "mediumtext_column", - "T": { - "Name": "BYTES", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: mediumtext_column mediumtext(16777215)", - "Id": "c99" - } - }, - "PrimaryKeys": [ - { - "ColId": "c79", - "Desc": false, - "Order": 1 - } - ], - "ForeignKeys": null, - "Indexes": null, - "ParentId": "", - "Comment": "Spanner schema for source table AllDatatypeColumns", - "Id": "t77" - }, - "t78": { - "Name": "AllDatatypeColumns2", - "ColIds": [ - "c108", - "c109", - "c110", - "c111", - "c112", - "c113", - "c114", - "c115", - "c116", - "c117", - "c118", - "c119", - "c120", - "c121", - "c122", - "c123", - "c124", - "c125", - "c126", - "c127", - "c128", - "c129", - "c130", - "c131", - "c132", - "c133", - "c134", - "c135" - ], - "ShardIdColumn": "", - "ColDefs": { - "c108": { - "Name": "varchar_column", - "T": { - "Name": "STRING", - "Len": 20, - "IsArray": false - }, - "NotNull": true, - "Comment": "From: varchar_column varchar(20)", - "Id": "c108" - }, - "c109": { - "Name": "tinyint_column", - "T": { - "Name": "INT64", - "Len": 0, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: tinyint_column tinyint(3)", - "Id": "c109" - }, - "c110": { - "Name": "text_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: text_column text(65535)", - "Id": "c110" - }, - "c111": { - "Name": "date_column", - "T": { - "Name": "DATE", - "Len": 0, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: date_column date", - "Id": "c111" - }, - "c112": { - "Name": "smallint_column", - "T": { - "Name": "INT64", - "Len": 0, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: smallint_column smallint(5)", - "Id": "c112" - }, - "c113": { - "Name": "mediumint_column", - "T": { - "Name": "INT64", - "Len": 0, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: mediumint_column mediumint(7)", - "Id": "c113" - }, - "c114": { - "Name": "int_column", - "T": { - "Name": "INT64", - "Len": 0, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: int_column int(10)", - "Id": "c114" - }, - "c115": { - "Name": "bigint_column", - "T": { - "Name": "INT64", - "Len": 0, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: bigint_column bigint(19)", - "Id": "c115" - }, - "c116": { - "Name": "float_column", - "T": { - "Name": "FLOAT64", - "Len": 0, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: float_column float(10,2)", - "Id": "c116" - }, - "c117": { - "Name": "double_column", - "T": { - "Name": "FLOAT64", - "Len": 0, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: double_column double(22)", - "Id": "c117" - }, - "c118": { - "Name": "decimal_column", - "T": { - "Name": "NUMERIC", - "Len": 0, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: decimal_column decimal(10,2)", - "Id": "c118" - }, - "c119": { - "Name": "datetime_column", - "T": { - "Name": "TIMESTAMP", - "Len": 0, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: datetime_column datetime", - "Id": "c119" - }, - "c120": { - "Name": "timestamp_column", - "T": { - "Name": "TIMESTAMP", - "Len": 0, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: timestamp_column timestamp", - "Id": "c120" - }, - "c121": { - "Name": "time_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: time_column time", - "Id": "c121" - }, - "c122": { - "Name": "year_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: year_column year", - "Id": "c122" - }, - "c123": { - "Name": "char_column", - "T": { - "Name": "STRING", - "Len": 10, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: char_column char(10)", - "Id": "c123" - }, - "c124": { - "Name": "tinyblob_column", - "T": { - "Name": "BYTES", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: tinyblob_column tinyblob(255)", - "Id": "c124" - }, - "c125": { - "Name": "tinytext_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: tinytext_column tinytext(255)", - "Id": "c125" - }, - "c126": { - "Name": "blob_column", - "T": { - "Name": "BYTES", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: blob_column blob(65535)", - "Id": "c126" - }, - "c127": { - "Name": "mediumblob_column", - "T": { - "Name": "BYTES", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: mediumblob_column mediumblob(16777215)", - "Id": "c127" - }, - "c128": { - "Name": "mediumtext_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: mediumtext_column mediumtext(16777215)", - "Id": "c128" - }, - "c129": { - "Name": "longblob_column", - "T": { - "Name": "BYTES", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: longblob_column longblob(4294967295)", - "Id": "c129" - }, - "c130": { - "Name": "longtext_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: longtext_column longtext(4294967295)", - "Id": "c130" - }, - "c131": { - "Name": "enum_column", - "T": { - "Name": "STRING", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: enum_column enum(1)", - "Id": "c131" - }, - "c132": { - "Name": "bool_column", - "T": { - "Name": "BOOL", - "Len": 0, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: bool_column tinyint(1)", - "Id": "c132" - }, - "c133": { - "Name": "binary_column", - "T": { - "Name": "BYTES", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: binary_column binary(20)", - "Id": "c133" - }, - "c134": { - "Name": "varbinary_column", - "T": { - "Name": "BYTES", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: varbinary_column varbinary(20)", - "Id": "c134" - }, - "c135": { - "Name": "bit_column", - "T": { - "Name": "BYTES", - "Len": 9223372036854775807, - "IsArray": false - }, - "NotNull": false, - "Comment": "From: bit_column bit(7)", - "Id": "c135" - } - }, - "PrimaryKeys": [ - { - "ColId": "c108", - "Desc": false, - "Order": 1 - } - ], - "ForeignKeys": null, - "Indexes": null, - "ParentId": "", - "Comment": "Spanner schema for source table AllDatatypeColumns2", - "Id": "t78" - } - }, - "SyntheticPKeys": {}, - "SrcSchema": { - "t77": { - "Name": "AllDatatypeColumns", - "Schema": "datatype_test", - "ColIds": [ - "c79", - "c80", - "c81", - "c82", - "c83", - "c84", - "c85", - "c86", - "c87", - "c88", - "c89", - "c90", - "c91", - "c92", - "c93", - "c94", - "c95", - "c96", - "c97", - "c98", - "c99", - "c100", - "c101", - "c102", - "c103", - "c104", - "c105", - "c106", - "c107" - ], - "ColDefs": { - "c100": { - "Name": "longblob_column", - "Type": { - "Name": "longblob", - "Mods": [ - 4294967295 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c100" - }, - "c101": { - "Name": "longtext_column", - "Type": { - "Name": "longtext", - "Mods": [ - 4294967295 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c101" - }, - "c102": { - "Name": "enum_column", - "Type": { - "Name": "enum", - "Mods": [ - 1 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c102" - }, - "c103": { - "Name": "bool_column", - "Type": { - "Name": "tinyint", - "Mods": [ - 1 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c103" - }, - "c104": { - "Name": "other_bool_column", - "Type": { - "Name": "tinyint", - "Mods": [ - 1 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c104" - }, - "c105": { - "Name": "binary_column", - "Type": { - "Name": "binary", - "Mods": [ - 20 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c105" - }, - "c106": { - "Name": "varbinary_column", - "Type": { - "Name": "varbinary", - "Mods": [ - 20 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c106" - }, - "c107": { - "Name": "bit_column", - "Type": { - "Name": "bit", - "Mods": [ - 7 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c107" - }, - "c79": { - "Name": "varchar_column", - "Type": { - "Name": "varchar", - "Mods": [ - 20 - ], - "ArrayBounds": null - }, - "NotNull": true, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c79" - }, - "c80": { - "Name": "tinyint_column", - "Type": { - "Name": "tinyint", - "Mods": [ - 3 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c80" - }, - "c81": { - "Name": "text_column", - "Type": { - "Name": "text", - "Mods": [ - 65535 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c81" - }, - "c82": { - "Name": "date_column", - "Type": { - "Name": "date", - "Mods": null, - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c82" - }, - "c83": { - "Name": "smallint_column", - "Type": { - "Name": "smallint", - "Mods": [ - 5 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c83" - }, - "c84": { - "Name": "mediumint_column", - "Type": { - "Name": "mediumint", - "Mods": [ - 7 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c84" - }, - "c85": { - "Name": "int_column", - "Type": { - "Name": "int", - "Mods": [ - 10 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c85" - }, - "c86": { - "Name": "bigint_column", - "Type": { - "Name": "bigint", - "Mods": [ - 19 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c86" - }, - "c87": { - "Name": "float_column", - "Type": { - "Name": "float", - "Mods": [ - 10, - 2 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c87" - }, - "c88": { - "Name": "double_column", - "Type": { - "Name": "double", - "Mods": [ - 22 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c88" - }, - "c89": { - "Name": "decimal_column", - "Type": { - "Name": "decimal", - "Mods": [ - 10, - 2 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c89" - }, - "c90": { - "Name": "datetime_column", - "Type": { - "Name": "datetime", - "Mods": null, - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c90" - }, - "c91": { - "Name": "timestamp_column", - "Type": { - "Name": "timestamp", - "Mods": null, - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c91" - }, - "c92": { - "Name": "time_column", - "Type": { - "Name": "time", - "Mods": null, - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c92" - }, - "c93": { - "Name": "year_column", - "Type": { - "Name": "year", - "Mods": null, - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c93" - }, - "c94": { - "Name": "char_column", - "Type": { - "Name": "char", - "Mods": [ - 10 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c94" - }, - "c95": { - "Name": "tinyblob_column", - "Type": { - "Name": "tinyblob", - "Mods": [ - 255 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c95" - }, - "c96": { - "Name": "tinytext_column", - "Type": { - "Name": "tinytext", - "Mods": [ - 255 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c96" - }, - "c97": { - "Name": "blob_column", - "Type": { - "Name": "blob", - "Mods": [ - 65535 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c97" - }, - "c98": { - "Name": "mediumblob_column", - "Type": { - "Name": "mediumblob", - "Mods": [ - 16777215 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c98" - }, - "c99": { - "Name": "mediumtext_column", - "Type": { - "Name": "mediumtext", - "Mods": [ - 16777215 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c99" - } - }, - "PrimaryKeys": [ - { - "ColId": "c79", - "Desc": false, - "Order": 1 - } - ], - "ForeignKeys": null, - "Indexes": null, - "Id": "t77" - }, - "t78": { - "Name": "AllDatatypeColumns2", - "Schema": "datatype_test", - "ColIds": [ - "c108", - "c109", - "c110", - "c111", - "c112", - "c113", - "c114", - "c115", - "c116", - "c117", - "c118", - "c119", - "c120", - "c121", - "c122", - "c123", - "c124", - "c125", - "c126", - "c127", - "c128", - "c129", - "c130", - "c131", - "c132", - "c133", - "c134", - "c135" - ], - "ColDefs": { - "c108": { - "Name": "varchar_column", - "Type": { - "Name": "varchar", - "Mods": [ - 20 - ], - "ArrayBounds": null - }, - "NotNull": true, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c108" - }, - "c109": { - "Name": "tinyint_column", - "Type": { - "Name": "tinyint", - "Mods": [ - 3 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c109" - }, - "c110": { - "Name": "text_column", - "Type": { - "Name": "text", - "Mods": [ - 65535 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c110" - }, - "c111": { - "Name": "date_column", - "Type": { - "Name": "date", - "Mods": null, - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c111" - }, - "c112": { - "Name": "smallint_column", - "Type": { - "Name": "smallint", - "Mods": [ - 5 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c112" - }, - "c113": { - "Name": "mediumint_column", - "Type": { - "Name": "mediumint", - "Mods": [ - 7 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c113" - }, - "c114": { - "Name": "int_column", - "Type": { - "Name": "int", - "Mods": [ - 10 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c114" - }, - "c115": { - "Name": "bigint_column", - "Type": { - "Name": "bigint", - "Mods": [ - 19 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c115" - }, - "c116": { - "Name": "float_column", - "Type": { - "Name": "float", - "Mods": [ - 10, - 2 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c116" - }, - "c117": { - "Name": "double_column", - "Type": { - "Name": "double", - "Mods": [ - 22 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c117" - }, - "c118": { - "Name": "decimal_column", - "Type": { - "Name": "decimal", - "Mods": [ - 10, - 2 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c118" - }, - "c119": { - "Name": "datetime_column", - "Type": { - "Name": "datetime", - "Mods": null, - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c119" - }, - "c120": { - "Name": "timestamp_column", - "Type": { - "Name": "timestamp", - "Mods": null, - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c120" - }, - "c121": { - "Name": "time_column", - "Type": { - "Name": "time", - "Mods": null, - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c121" - }, - "c122": { - "Name": "year_column", - "Type": { - "Name": "year", - "Mods": null, - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c122" - }, - "c123": { - "Name": "char_column", - "Type": { - "Name": "char", - "Mods": [ - 10 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c123" - }, - "c124": { - "Name": "tinyblob_column", - "Type": { - "Name": "tinyblob", - "Mods": [ - 255 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c124" - }, - "c125": { - "Name": "tinytext_column", - "Type": { - "Name": "tinytext", - "Mods": [ - 255 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c125" - }, - "c126": { - "Name": "blob_column", - "Type": { - "Name": "blob", - "Mods": [ - 65535 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c126" - }, - "c127": { - "Name": "mediumblob_column", - "Type": { - "Name": "mediumblob", - "Mods": [ - 16777215 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c127" - }, - "c128": { - "Name": "mediumtext_column", - "Type": { - "Name": "mediumtext", - "Mods": [ - 16777215 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c128" - }, - "c129": { - "Name": "longblob_column", - "Type": { - "Name": "longblob", - "Mods": [ - 4294967295 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c129" - }, - "c130": { - "Name": "longtext_column", - "Type": { - "Name": "longtext", - "Mods": [ - 4294967295 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c130" - }, - "c131": { - "Name": "enum_column", - "Type": { - "Name": "enum", - "Mods": [ - 1 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c131" - }, - "c132": { - "Name": "bool_column", - "Type": { - "Name": "tinyint", - "Mods": [ - 1 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c132" - }, - "c133": { - "Name": "binary_column", - "Type": { - "Name": "binary", - "Mods": [ - 20 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c133" - }, - "c134": { - "Name": "varbinary_column", - "Type": { - "Name": "varbinary", - "Mods": [ - 20 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c134" - }, - "c135": { - "Name": "bit_column", - "Type": { - "Name": "bit", - "Mods": [ - 7 - ], - "ArrayBounds": null - }, - "NotNull": false, - "Ignored": { - "Check": false, - "Identity": false, - "Default": false, - "Exclusion": false, - "ForeignKey": false, - "AutoIncrement": false - }, - "Id": "c135" - } - }, - "PrimaryKeys": [ - { - "ColId": "c108", - "Desc": false, - "Order": 1 - } - ], - "ForeignKeys": null, - "Indexes": null, - "Id": "t78" - } - }, - "SchemaIssues": { - "t77": { - "ColumnLevelIssues": { - "c100": [], - "c101": [], - "c102": [], - "c103": [ - 14 - ], - "c104": [ - 14 - ], - "c105": [], - "c106": [], - "c107": [], - "c79": [], - "c80": [ - 14 - ], - "c81": [], - "c82": [ - 14 - ], - "c83": [ - 14 - ], - "c84": [ - 14 - ], - "c85": [ - 14 - ], - "c86": [ - 14 - ], - "c87": [ - 14 - ], - "c88": [ - 14 - ], - "c89": [ - 14 - ], - "c90": [ - 14 - ], - "c91": [ - 14 - ], - "c92": [ - 15 - ], - "c93": [ - 15 - ], - "c94": [], - "c95": [], - "c96": [], - "c97": [], - "c98": [], - "c99": [] - }, - "TableLevelIssues": null - }, - "t78": { - "ColumnLevelIssues": { - "c109": [ - 14 - ], - "c112": [ - 14 - ], - "c113": [ - 14 - ], - "c114": [ - 14 - ], - "c116": [ - 14 - ], - "c119": [ - 13 - ], - "c121": [ - 15 - ], - "c122": [ - 15 - ] - }, - "TableLevelIssues": null - } - }, - "Location": {}, - "TimezoneOffset": "+00:00", - "SpDialect": "google_standard_sql", - "UniquePKey": {}, - "Rules": [], - "IsSharded": false, - "SpRegion": "", - "ResourceValidation": false, - "UI": false -} \ No newline at end of file