Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added a test to detect COPY ON ERROR clause in the migration assessment test #2157

Merged
merged 2 commits into from
Jan 7, 2025
Merged

Added a test to detect COPY ON ERROR clause in the migration assessment test #2157

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d7859a5
Added a test to deteck COPY ON ERROR clause in the migration assessme…
ShivanshGahlot Jan 7, 2025
207f19f
Added a test to deteck COPY ON ERROR clause in the migration assessme…
ShivanshGahlot Jan 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 29 additions & 8 deletions migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,15 @@
},
{
"ObjectType": "SEQUENCE",
"TotalCount": 43,
"TotalCount": 42,
"InvalidCount": 0,
"ObjectNames": "public.employeesforview_id_seq, schema2.employeesforview_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq"
"ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.bigint_multirange_table_id_seq, public.date_multirange_table_id_seq, public.employees2_id_seq, public.employees_employee_id_seq, public.employeesforview_id_seq, public.ext_test_id_seq, public.int_multirange_table_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.numeric_multirange_table_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.timestamp_multirange_table_id_seq, public.timestamptz_multirange_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.bigint_multirange_table_id_seq, schema2.date_multirange_table_id_seq, schema2.employees2_id_seq, schema2.employeesforview_id_seq, schema2.ext_test_id_seq, schema2.int_multirange_table_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.numeric_multirange_table_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.timestamp_multirange_table_id_seq, schema2.timestamptz_multirange_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq"
},
{
"ObjectType": "TABLE",
"TotalCount": 82,
"TotalCount": 83,
"InvalidCount": 36,
"ObjectNames": "public.employeesforview, schema2.employeesforview, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2"
"ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employeescopyfromwhere, public.employeescopyonerror, public.employeesforview, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.employeesforview, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2"
},
{
"ObjectType": "INDEX",
Expand Down Expand Up @@ -172,7 +172,8 @@
"public.library_nested",
"public.orders_lateral",
"public.employees",
"public.employees3",
"public.employeescopyfromwhere",
"public.employeescopyonerror",
"public.bigint_multirange_table",
"public.date_multirange_table",
"public.int_multirange_table",
Expand All @@ -187,7 +188,7 @@
"schema2.timestamptz_multirange_table",
"public.employeesforview"
],
"ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 88 objects (80 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.",
"ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 89 objects (81 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.",
"ShardedTables": [
"public.combined_tbl",
"public.citext_type",
Expand Down Expand Up @@ -2111,7 +2112,7 @@
},
{
"SchemaName": "public",
"ObjectName": "employees3",
"ObjectName": "employeescopyfromwhere",
"RowCount": 2,
"ColumnCount": 3,
"Reads": 0,
Expand Down Expand Up @@ -2304,6 +2305,20 @@
"ObjectType": "",
"ParentTableName": null,
"SizeInBytes": 0
},
{
"SchemaName": "public",
"ObjectName": "employeescopyonerror",
"RowCount": 3,
"ColumnCount": 3,
"Reads": 0,
"Writes": 3,
"ReadsPerSecond": 0,
"WritesPerSecond": 0,
"IsIndex": false,
"ObjectType": "",
"ParentTableName": null,
"SizeInBytes": 8192
}

],
Expand Down Expand Up @@ -2507,7 +2522,13 @@
},
{
"ConstructTypeName": "COPY FROM ... WHERE",
"Query": "COPY employees3 (id, name, age)\nFROM STDIN WITH (FORMAT csv)\nWHERE age \u003e 30",
"Query": "COPY employeesCopyFromWhere (id, name, age)\nFROM STDIN WITH (FORMAT csv)\nWHERE age \u003e 30",
"DocsLink": "",
"MinimumVersionsFixedIn": null
},
{
"ConstructTypeName": "COPY ... ON_ERROR",
"Query": "COPY employeesCopyOnError (id, name, age)\nFROM STDIN WITH (FORMAT csv, ON_ERROR IGNORE )",
"DocsLink": "",
"MinimumVersionsFixedIn": null
}
Expand Down
27 changes: 16 additions & 11 deletions migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,28 +142,33 @@ SELECT lo_create('32142');

-- Unsupported COPY constructs

CREATE TABLE IF NOT EXISTS employees3 (
id SERIAL PRIMARY KEY,
CREATE TABLE IF NOT EXISTS employeesCopyFromWhere (
id INT PRIMARY KEY,
name TEXT NOT NULL,
age INT NOT NULL
);


-- COPY FROM with WHERE clause
COPY employees3 (id, name, age)
COPY employeesCopyFromWhere (id, name, age)
FROM STDIN WITH (FORMAT csv)
WHERE age > 30;
1,John Smith,25
2,Jane Doe,34
3,Bob Johnson,31
\.

-- This can be uncommented when we start using PG 17 or later in the tests
-- -- COPY with ON_ERROR clause
-- COPY employees (id, name, age)
-- FROM STDIN WITH (FORMAT csv, ON_ERROR IGNORE );
-- 4,Adam Smith,22
-- 5,John Doe,34
-- 6,Ron Johnson,31
-- \.
CREATE TABLE IF NOT EXISTS employeesCopyOnError (
id INT PRIMARY KEY,
name TEXT NOT NULL,
age INT NOT NULL
);

-- COPY with ON_ERROR clause
COPY employeesCopyOnError (id, name, age)
FROM STDIN WITH (FORMAT csv, ON_ERROR IGNORE );
4,Adam Smith,22
5,John Doe,34
6,Ron Johnson,31
\.