From 0d1263adca5dc1e324b13793f31c508a891e671d Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Fri, 27 Dec 2024 07:55:40 +0000 Subject: [PATCH 01/16] Reporting unique nulls not distinct in analyze schema and assessment report --- yb-voyager/cmd/assessMigrationCommand.go | 1 + yb-voyager/src/query/queryissue/constants.go | 3 + yb-voyager/src/query/queryissue/detectors.go | 133 ++++++++++++++++++ yb-voyager/src/query/queryissue/issues_ddl.go | 13 ++ .../query/queryissue/parser_issue_detector.go | 1 + .../queryissue/parser_issue_detector_test.go | 13 ++ .../src/query/queryparser/helpers_protomsg.go | 21 +++ 7 files changed, 185 insertions(+) diff --git a/yb-voyager/cmd/assessMigrationCommand.go b/yb-voyager/cmd/assessMigrationCommand.go index 927aba2b6..2774b98b4 100644 --- a/yb-voyager/cmd/assessMigrationCommand.go +++ b/yb-voyager/cmd/assessMigrationCommand.go @@ -1068,6 +1068,7 @@ func fetchUnsupportedPGFeaturesFromSchemaReport(schemaAnalysisReport utils.Schem unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_CONSTRUCTOR_FUNCTION_NAME, "", queryissue.JSON_CONSTRUCTOR_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.AGGREGATION_FUNCTIONS_NAME, "", queryissue.AGGREGATE_FUNCTION, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.SECURITY_INVOKER_VIEWS_NAME, "", queryissue.SECURITY_INVOKER_VIEWS, schemaAnalysisReport, false, "")) + unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.UNIQUE_NULLS_NOT_DISTINCT_NAME, "", queryissue.UNIQUE_NULLS_NOT_DISTINCT, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSONB_SUBSCRIPTING_NAME, "", queryissue.JSONB_SUBSCRIPTING, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME, "", queryissue.FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE, schemaAnalysisReport, false, "")) unsupportedFeatures = append(unsupportedFeatures, getUnsupportedFeaturesFromSchemaAnalysisReport(queryissue.JSON_TYPE_PREDICATE_NAME, "", queryissue.JSON_TYPE_PREDICATE, schemaAnalysisReport, false, "")) diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index edb8edf27..2db0cab2e 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -81,6 +81,9 @@ const ( FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE = "FOREIGN_KEY_REFERENCED_PARTITIONED_TABLE" FOREIGN_KEY_REFERENCES_PARTITIONED_TABLE_NAME = "Foreign key constraint references partitioned table" + + UNIQUE_NULLS_NOT_DISTINCT = "UNIQUE_NULLS_NOT_DISTINCT" + UNIQUE_NULLS_NOT_DISTINCT_NAME = "Unique Nulls Not Distinct" ) // Object types diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index 1bc45b099..0b11427b0 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -453,6 +453,139 @@ func (d *JsonQueryFunctionDetector) GetIssues() []QueryIssue { return issues } +type UniqueNullsNotDistinctDetector struct { + query string + detected bool +} + +func NewUniqueNullsNotDistinctDetector(query string) *UniqueNullsNotDistinctDetector { + return &UniqueNullsNotDistinctDetector{ + query: query, + } +} + +// { +// "parseTreeVersion": 170000, +// "statements": { +// "statement": { +// "createStatement": { +// "relation": { +// "schemaName": "public", +// "relationName": "products", +// "inherits": true, +// "persistence": "p", // permanent table +// "location": 13 +// }, +// "tableElements": [ +// { +// "columnDefinition": { +// "columnName": "id", +// "typeName": { +// "names": [ +// { "string": { "value": "serial" } } +// ], +// "typeModifier": -1, +// "location": 38 +// }, +// "isLocal": true, +// "constraints": [ +// { +// "constraintType": "PRIMARY_KEY", +// "location": 45 +// } +// ], +// "location": 35 +// } +// }, +// { +// "columnDefinition": { +// "columnName": "product_name", +// "typeName": { +// "names": [ +// { "string": { "value": "pg_catalog" } }, +// { "string": { "value": "varchar" } } +// ], +// "typeModifiers": [ +// { +// "constant": { +// "integerValue": 100, +// "location": 83 +// } +// } +// ], +// "typeModifier": -1, +// "location": 75 +// }, +// "isLocal": true, +// "location": 62 +// } +// }, +// { +// "columnDefinition": { +// "columnName": "serial_number", +// "typeName": { +// "names": [ +// { "string": { "value": "text" } } +// ], +// "typeModifier": -1, +// "location": 107 +// }, +// "isLocal": true, +// "location": 93 +// } +// } +// ], +// "tableConstraints": [ +// { +// "constraintType": "UNIQUE", +// "nullsNotDistinct": true, +// "keys": [ +// { "string": { "value": "product_name" } }, +// { "string": { "value": "serial_number" } } +// ], +// "location": 117 +// } +// ], +// "onCommit": "ONCOMMIT_NOOP" +// } +// } +// }, +// "statementLength": 174 +// } + +// Detect checks if a unique constraint is defined which has nulls not distinct +func (d *UniqueNullsNotDistinctDetector) Detect(msg protoreflect.Message) error { + // If message is of type PG_QUERY_TABLECONSTRAINT_NODE + if !(queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_TABLECONSTRAINT_NODE) { + return nil + } + + // Fetch contype as an enum number + constraintType := queryparser.GetEnumNumField(msg, "contype") + constraintTypeStr := queryparser.GetResolvedEnumName(msg, "contype", constraintType) + + // Check if the constraint is of type CONSTR_UNIQUE + if constraintTypeStr != "CONSTR_UNIQUE" { + return nil + } + + // Check if the constraint has nulls not distinct + nullsNotDistinct := queryparser.GetBoolField(msg, "nulls_not_distinct") + if nullsNotDistinct { + d.detected = true + } + + return nil +} + +func (d *UniqueNullsNotDistinctDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.detected { + issues = append(issues, NewUniqueNullsNotDistinctIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } + return issues +} + type JsonPredicateExprDetector struct { query string detected bool diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 7a06bcaa0..09110c170 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -515,3 +515,16 @@ func NewForeignKeyReferencesPartitionedTableIssue(objectType string, objectName } return newQueryIssue(foreignKeyReferencesPartitionedTableIssue, objectType, objectName, SqlStatement, details) } + +var uniqueNullsNotDistinctIssue = issue.Issue{ + Type: UNIQUE_NULLS_NOT_DISTINCT, + TypeName: "UNIQUE NULLS NOT DISTINCT", + TypeDescription: "", + Suggestion: "", + GH: "", + DocsLink: "", +} + +func NewUniqueNullsNotDistinctIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(uniqueNullsNotDistinctIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index defde4987..e452f34a3 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -394,6 +394,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewJsonConstructorFuncDetector(query), NewJsonQueryFunctionDetector(query), NewJsonbSubscriptingDetector(query, p.jsonbColumns, p.getJsonbReturnTypeFunctions()), + NewUniqueNullsNotDistinctDetector(query), NewJsonPredicateExprDetector(query), } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index e13597625..c8901276c 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -163,6 +163,13 @@ CHECK (xpath_exists('/invoice/customer', data));` WITH (security_invoker = true) AS SELECT employee_id, first_name FROM public.employees;` + stmt21 = `CREATE TABLE public.products ( + id SERIAL PRIMARY KEY, + product_name VARCHAR(100), + serial_number TEXT, + UNIQUE NULLS NOT DISTINCT (product_name, serial_number) + );` + stmt22 = `ALTER TABLE public.products ADD CONSTRAINT unique_product_name UNIQUE NULLS NOT DISTINCT (product_name);` ) func modifiedIssuesforPLPGSQL(issues []QueryIssue, objType string, objName string) []QueryIssue { @@ -286,6 +293,12 @@ func TestDDLIssues(t *testing.T) { stmt20: []QueryIssue{ NewSecurityInvokerViewIssue("VIEW", "public.view_explicit_security_invoker", stmt20), }, + stmt21: []QueryIssue{ + NewUniqueNullsNotDistinctIssue("TABLE", "public.products", stmt21), + }, + stmt22: []QueryIssue{ + NewUniqueNullsNotDistinctIssue("TABLE", "public.products", stmt22), + }, } for _, stmt := range requiredDDLs { err := parserIssueDetector.ParseRequiredDDLs(stmt) diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index 73d76b7e1..aadfdfa20 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -395,6 +395,27 @@ func GetEnumNumField(msg protoreflect.Message, fieldName string) protoreflect.En return 0 } +// GetResolvedEnumName resolves the enum value to its symbolic name. +func GetResolvedEnumName(msg protoreflect.Message, fieldName string, enumNum protoreflect.EnumNumber) string { + fieldDescriptor := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) + if fieldDescriptor == nil { + return "UNKNOWN" + } + + enumDescriptor := fieldDescriptor.Enum() + if enumDescriptor == nil { + return "UNKNOWN" + } + + // Resolve the enum value to its symbolic name + resolvedName := enumDescriptor.Values().ByNumber(protoreflect.EnumNumber(enumNum)).Name() + + if resolvedName == "" { + return "UNKNOWN" + } + return string(resolvedName) +} + // GetSchemaAndObjectName extracts the schema and object name from a list. func GetSchemaAndObjectName(nameList protoreflect.List) (string, string) { var schemaName, objectName string From c85fd79a561a20d9890527cf382a9c39ad5c2fe5 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Fri, 27 Dec 2024 08:20:26 +0000 Subject: [PATCH 02/16] Added end to end test in analyze schema test --- .../dummy-export-dir/schema/tables/table.sql | 30 +++++++++++++++++++ .../tests/analyze-schema/expected_issues.json | 30 +++++++++++++++++++ migtests/tests/analyze-schema/summary.json | 7 ++--- 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 9ea26b115..3f2801fa6 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -429,3 +429,33 @@ CREATE TABLE timestamptz_multirange_table ( global_event_times tstzmultirange ); +-- Testing tables with unique nulls not distinct constraints + +-- Control case +CREATE TABLE users_unique_nulls_distinct ( + id SERIAL PRIMARY KEY, + email TEXT, + UNIQUE (email) +); + +CREATE TABLE users_unique_nulls_not_distinct ( + id SERIAL PRIMARY KEY, + email TEXT, + UNIQUE NULLS NOT DISTINCT (email) +); + +CREATE TABLE sales_unique_nulls_not_distinct ( + store_id INT, + product_id INT, + sale_date DATE, + UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date) +); + +CREATE TABLE sales_unique_nulls_not_distinct_alter ( + store_id INT, + product_id INT, + sale_date DATE +); + +ALTER TABLE sales_unique_nulls_not_distinct_alter + ADD CONSTRAINT sales_unique_nulls_not_distinct_alter_unique UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date); diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 6f5af66fb..b1c52a9e6 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -2060,5 +2060,35 @@ "Suggestion": "Multirange data type is not yet supported in YugabyteDB, no workaround available currently", "GH": "", "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "users_unique_nulls_not_distinct", + "Reason": "UNIQUE NULLS NOT DISTINCT", + "SqlStatement": "CREATE TABLE users_unique_nulls_not_distinct (\n id SERIAL PRIMARY KEY,\n email TEXT,\n UNIQUE NULLS NOT DISTINCT (email)\n);", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "sales_unique_nulls_not_distinct", + "Reason": "UNIQUE NULLS NOT DISTINCT", + "SqlStatement": "CREATE TABLE sales_unique_nulls_not_distinct (\n store_id INT,\n product_id INT,\n sale_date DATE,\n UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date)\n);", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "TABLE", + "ObjectName": "sales_unique_nulls_not_distinct_alter", + "Reason": "UNIQUE NULLS NOT DISTINCT", + "SqlStatement": "ALTER TABLE sales_unique_nulls_not_distinct_alter\n\tADD CONSTRAINT sales_unique_nulls_not_distinct_alter_unique UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date);", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null } ] diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 8cc96548f..6c437949b 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -26,10 +26,9 @@ }, { "ObjectType": "TABLE", - "TotalCount": 59, - "InvalidCount": 50, - "ObjectNames": "public.json_data, employees, image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table" }, - + "TotalCount": 61, + "InvalidCount": 52, + "ObjectNames": "image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table, users_unique_nulls_not_distinct, sales_unique_nulls_not_distinct, users_unique_nulls_distinct, sales_unique_nulls_not_distinct_alter" }, { "ObjectType": "INDEX", "TotalCount": 43, From a3e4ca8f59072631597d44601b1aef3cbc0afa27 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Fri, 27 Dec 2024 09:36:41 +0000 Subject: [PATCH 03/16] Added end to end test in migration assessment test --- .../expectedAssessmentReport.json | 274 +++++++++++++++++- .../pg_assessment_report.sql | 32 ++ .../queryissue/parser_issue_detector_test.go | 2 +- 3 files changed, 301 insertions(+), 7 deletions(-) diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index c341d8b26..54f6e735a 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -44,9 +44,9 @@ }, { "ObjectType": "TABLE", - "TotalCount": 83, - "InvalidCount": 36, - "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employeescopyfromwhere, public.employeescopyonerror, public.employeesforview, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.employeesforview, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" + "TotalCount": 88, + "InvalidCount": 41, + "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.sales_unique_nulls_not_distinct, public.sales_unique_nulls_not_distinct_alter, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.users_unique_nulls_distinct, public.users_unique_nulls_not_distinct, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.sales_unique_nulls_not_distinct, schema2.sales_unique_nulls_not_distinct_alter, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.users_unique_nulls_distinct, schema2.users_unique_nulls_not_distinct, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { "ObjectType": "INDEX", @@ -186,9 +186,17 @@ "schema2.numeric_multirange_table", "schema2.timestamp_multirange_table", "schema2.timestamptz_multirange_table", - "public.employeesforview" + "public.employeesforview", + "schema2.users_unique_nulls_distinct", + "schema2.users_unique_nulls_not_distinct", + "schema2.sales_unique_nulls_not_distinct", + "schema2.sales_unique_nulls_not_distinct_alter", + "public.users_unique_nulls_distinct", + "public.users_unique_nulls_not_distinct", + "public.sales_unique_nulls_not_distinct", + "public.sales_unique_nulls_not_distinct_alter" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 89 objects (81 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 102 objects (86 tables/materialized views and 16 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -674,7 +682,37 @@ "Objects": [ { "ObjectName": "public.view_explicit_security_invoker", - "SqlStatement": "CREATE VIEW public.view_explicit_security_invoker WITH (security_invoker='true') AS\n SELECT employee_id,\n first_name\n FROM public.employees;" + "SqlStatement": "CREATE VIEW public.view_explicit_security_invoker WITH (security_invoker='true') AS\n SELECT employees.employee_id,\n employees.first_name\n FROM public.employees;" + } + ], + "MinimumVersionsFixedIn": null + }, + { + "FeatureName": "Unique Nulls Not Distinct", + "Objects": [ + { + "ObjectName": "public.sales_unique_nulls_not_distinct", + "SqlStatement": "ALTER TABLE ONLY public.sales_unique_nulls_not_distinct\n ADD CONSTRAINT sales_unique_nulls_not_distin_store_id_product_id_sale_date_key UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date);" + }, + { + "ObjectName": "public.sales_unique_nulls_not_distinct_alter", + "SqlStatement": "ALTER TABLE ONLY public.sales_unique_nulls_not_distinct_alter\n ADD CONSTRAINT sales_unique_nulls_not_distinct_alter_unique UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date);" + }, + { + "ObjectName": "public.users_unique_nulls_not_distinct", + "SqlStatement": "ALTER TABLE ONLY public.users_unique_nulls_not_distinct\n ADD CONSTRAINT users_unique_nulls_not_distinct_email_key UNIQUE NULLS NOT DISTINCT (email);" + }, + { + "ObjectName": "schema2.sales_unique_nulls_not_distinct", + "SqlStatement": "ALTER TABLE ONLY schema2.sales_unique_nulls_not_distinct\n ADD CONSTRAINT sales_unique_nulls_not_distin_store_id_product_id_sale_date_key UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date);" + }, + { + "ObjectName": "schema2.sales_unique_nulls_not_distinct_alter", + "SqlStatement": "ALTER TABLE ONLY schema2.sales_unique_nulls_not_distinct_alter\n ADD CONSTRAINT sales_unique_nulls_not_distinct_alter_unique UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date);" + }, + { + "ObjectName": "schema2.users_unique_nulls_not_distinct", + "SqlStatement": "ALTER TABLE ONLY schema2.users_unique_nulls_not_distinct\n ADD CONSTRAINT users_unique_nulls_not_distinct_email_key UNIQUE NULLS NOT DISTINCT (email);" } ], "MinimumVersionsFixedIn": null @@ -682,6 +720,230 @@ ], "UnsupportedFeaturesDesc": "Features of the source database that are not supported on the target YugabyteDB.", "TableIndexStats": [ + { + "SchemaName": "schema2", + "ObjectName": "sales_unique_nulls_not_distinct_alter_unique", + "RowCount": null, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "schema2.sales_unique_nulls_not_distinct_alter", + "SizeInBytes": 8192 + }, + { + "SchemaName": "schema2", + "ObjectName": "sales_unique_nulls_not_distin_store_id_product_id_sale_date_key", + "RowCount": null, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "schema2.sales_unique_nulls_not_distinct", + "SizeInBytes": 8192 + }, + { + "SchemaName": "schema2", + "ObjectName": "users_unique_nulls_not_distinct_email_key", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "schema2.users_unique_nulls_not_distinct", + "SizeInBytes": 8192 + }, + { + "SchemaName": "schema2", + "ObjectName": "users_unique_nulls_distinct_email_key", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "schema2.users_unique_nulls_distinct", + "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "sales_unique_nulls_not_distinct_alter_unique", + "RowCount": null, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "public.sales_unique_nulls_not_distinct_alter", + "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "sales_unique_nulls_not_distin_store_id_product_id_sale_date_key", + "RowCount": null, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "public.sales_unique_nulls_not_distinct", + "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "users_unique_nulls_not_distinct_email_key", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "public.users_unique_nulls_not_distinct", + "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "users_unique_nulls_distinct_email_key", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "public.users_unique_nulls_distinct", + "SizeInBytes": 8192 + }, + { + "SchemaName": "schema2", + "ObjectName": "users_unique_nulls_not_distinct", + "RowCount": 0, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "users_unique_nulls_distinct", + "RowCount": 0, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "sales_unique_nulls_not_distinct", + "RowCount": 0, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "sales_unique_nulls_not_distinct_alter", + "RowCount": 0, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "users_unique_nulls_not_distinct", + "RowCount": 0, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "users_unique_nulls_distinct", + "RowCount": 0, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "sales_unique_nulls_not_distinct_alter", + "RowCount": 0, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "public", + "ObjectName": "sales_unique_nulls_not_distinct", + "RowCount": 0, + "ColumnCount": 3, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, { "SchemaName": "public", "ObjectName": "Recipients", diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index 72e5ea1e7..f5ef11c5e 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -430,3 +430,35 @@ WITH (security_invoker = true) AS SELECT employee_id, first_name FROM public.employees; +-- Testing tables with unique nulls not distinct constraints + +-- Control case +CREATE TABLE users_unique_nulls_distinct ( + id INTEGER PRIMARY KEY, + email TEXT, + UNIQUE (email) +); + +CREATE TABLE users_unique_nulls_not_distinct ( + id INTEGER PRIMARY KEY, + email TEXT, + UNIQUE NULLS NOT DISTINCT (email) +); + +CREATE TABLE sales_unique_nulls_not_distinct ( + store_id INT, + product_id INT, + sale_date DATE, + UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date) +); + +CREATE TABLE sales_unique_nulls_not_distinct_alter ( + store_id INT, + product_id INT, + sale_date DATE +); + +ALTER TABLE sales_unique_nulls_not_distinct_alter + ADD CONSTRAINT sales_unique_nulls_not_distinct_alter_unique UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date); + + diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index c8901276c..c0eaff60a 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -164,7 +164,7 @@ CHECK (xpath_exists('/invoice/customer', data));` SELECT employee_id, first_name FROM public.employees;` stmt21 = `CREATE TABLE public.products ( - id SERIAL PRIMARY KEY, + id INTEGER PRIMARY KEY, product_name VARCHAR(100), serial_number TEXT, UNIQUE NULLS NOT DISTINCT (product_name, serial_number) From 2da9364273cc3cc47941d33fd91b7ae89285760a Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Fri, 27 Dec 2024 09:47:33 +0000 Subject: [PATCH 04/16] Added test to issues_ddl_test.go --- .../src/query/queryissue/issues_ddl_test.go | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/yb-voyager/src/query/queryissue/issues_ddl_test.go b/yb-voyager/src/query/queryissue/issues_ddl_test.go index 55642bd8e..53498dffa 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl_test.go +++ b/yb-voyager/src/query/queryissue/issues_ddl_test.go @@ -287,6 +287,23 @@ func testForeignKeyReferencesPartitionedTableIssue(t *testing.T) { assertErrorCorrectlyThrownForIssueForYBVersion(t, err, `cannot reference partitioned table "abc1"`, foreignKeyReferencesPartitionedTableIssue) } +func testUniqueNullsNotDistinctIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + _, err = conn.Exec(ctx, ` + CREATE TABLE public.products ( + id INTEGER PRIMARY KEY, + product_name VARCHAR(100), + serial_number TEXT, + UNIQUE NULLS NOT DISTINCT (product_name, serial_number) + );`) + + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "syntax error", uniqueNullsNotDistinctIssue) +} + func TestDDLIssuesInYBVersion(t *testing.T) { var err error ybVersion := os.Getenv("YB_VERSION") @@ -348,4 +365,7 @@ func TestDDLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "foreign key referenced partitioned table", ybVersion), testForeignKeyReferencesPartitionedTableIssue) assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "unique nulls not distinct", ybVersion), testUniqueNullsNotDistinctIssue) + assert.True(t, success) } From d5e0bf6ce148981eede297cab720cd3dc3c6e659 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Fri, 27 Dec 2024 09:53:14 +0000 Subject: [PATCH 05/16] Removed comment --- yb-voyager/src/query/queryissue/detectors.go | 89 -------------------- 1 file changed, 89 deletions(-) diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index 0b11427b0..35f9e2684 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -464,95 +464,6 @@ func NewUniqueNullsNotDistinctDetector(query string) *UniqueNullsNotDistinctDete } } -// { -// "parseTreeVersion": 170000, -// "statements": { -// "statement": { -// "createStatement": { -// "relation": { -// "schemaName": "public", -// "relationName": "products", -// "inherits": true, -// "persistence": "p", // permanent table -// "location": 13 -// }, -// "tableElements": [ -// { -// "columnDefinition": { -// "columnName": "id", -// "typeName": { -// "names": [ -// { "string": { "value": "serial" } } -// ], -// "typeModifier": -1, -// "location": 38 -// }, -// "isLocal": true, -// "constraints": [ -// { -// "constraintType": "PRIMARY_KEY", -// "location": 45 -// } -// ], -// "location": 35 -// } -// }, -// { -// "columnDefinition": { -// "columnName": "product_name", -// "typeName": { -// "names": [ -// { "string": { "value": "pg_catalog" } }, -// { "string": { "value": "varchar" } } -// ], -// "typeModifiers": [ -// { -// "constant": { -// "integerValue": 100, -// "location": 83 -// } -// } -// ], -// "typeModifier": -1, -// "location": 75 -// }, -// "isLocal": true, -// "location": 62 -// } -// }, -// { -// "columnDefinition": { -// "columnName": "serial_number", -// "typeName": { -// "names": [ -// { "string": { "value": "text" } } -// ], -// "typeModifier": -1, -// "location": 107 -// }, -// "isLocal": true, -// "location": 93 -// } -// } -// ], -// "tableConstraints": [ -// { -// "constraintType": "UNIQUE", -// "nullsNotDistinct": true, -// "keys": [ -// { "string": { "value": "product_name" } }, -// { "string": { "value": "serial_number" } } -// ], -// "location": 117 -// } -// ], -// "onCommit": "ONCOMMIT_NOOP" -// } -// } -// }, -// "statementLength": 174 -// } - // Detect checks if a unique constraint is defined which has nulls not distinct func (d *UniqueNullsNotDistinctDetector) Detect(msg protoreflect.Message) error { // If message is of type PG_QUERY_TABLECONSTRAINT_NODE From db4322c79588ec0e8da583b78c81fe66fe4075db Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Mon, 6 Jan 2025 13:12:43 +0000 Subject: [PATCH 06/16] Made suggested changes --- migtests/tests/analyze-schema/summary.json | 4 +-- .../expectedAssessmentReport.json | 26 +++++++++++----- .../pg_assessment_report.sql | 10 +++++++ yb-voyager/src/query/queryissue/detectors.go | 30 +++++++++---------- .../queryissue/parser_issue_detector_test.go | 6 +++- .../src/query/queryparser/helpers_protomsg.go | 21 ------------- .../src/query/queryparser/traversal_proto.go | 1 + 7 files changed, 50 insertions(+), 48 deletions(-) diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index 6c437949b..e8eb69241 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -26,9 +26,9 @@ }, { "ObjectType": "TABLE", - "TotalCount": 61, + "TotalCount": 62, "InvalidCount": 52, - "ObjectNames": "image, public.xml_data_example, combined_tbl1, test_arr_enum, public.locations, test_udt, combined_tbl, public.ts_query_table, public.documents, public.citext_type, public.inet_type, public.test_jsonb, test_xml_type, test_xid_type, public.range_columns_partition_test_copy, anydata_test, uritype_test, public.foreign_def_test, test_4, enum_example.bugs, table_abc, anydataset_test, unique_def_test1, test_2, table_1, public.range_columns_partition_test, table_xyz, public.users, test_3, test_5, test_7, foreign_def_test2, unique_def_test, sales_data, table_test, test_interval, test_non_pk_multi_column_list, test_9, test_8, order_details, public.employees4, anytype_test, public.meeting, test_table_in_type_file, sales, test_1, \"Test\", foreign_def_test1, salaries2, test_6, public.pr, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table, users_unique_nulls_not_distinct, sales_unique_nulls_not_distinct, users_unique_nulls_distinct, sales_unique_nulls_not_distinct_alter" }, + "ObjectNames": "test_table_in_type_file, sales_data, salaries2, sales, test_1, test_2, test_non_pk_multi_column_list, test_3, test_4, test_5, test_6, test_7, test_8, test_9, order_details, public.employees4, enum_example.bugs, table_xyz, table_abc, table_1, table_test, test_interval, public.range_columns_partition_test, public.range_columns_partition_test_copy, anydata_test, anydataset_test, anytype_test, uritype_test, \"Test\", public.meeting, public.pr, public.foreign_def_test, public.users, foreign_def_test1, foreign_def_test2, unique_def_test, unique_def_test1, test_xml_type, test_xid_type, public.test_jsonb, public.inet_type, public.citext_type, public.documents, public.ts_query_table, combined_tbl, combined_tbl1, test_udt, test_arr_enum, public.locations, public.xml_data_example, image, employees, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table, users_unique_nulls_distinct, users_unique_nulls_not_distinct, sales_unique_nulls_not_distinct, sales_unique_nulls_not_distinct_alter" }, { "ObjectType": "INDEX", "TotalCount": 43, diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 54f6e735a..fcf036e9e 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -44,15 +44,15 @@ }, { "ObjectType": "TABLE", - "TotalCount": 88, + "TotalCount": 92, "InvalidCount": 41, - "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.sales_unique_nulls_not_distinct, public.sales_unique_nulls_not_distinct_alter, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.users_unique_nulls_distinct, public.users_unique_nulls_not_distinct, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.sales_unique_nulls_not_distinct, schema2.sales_unique_nulls_not_distinct_alter, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.users_unique_nulls_distinct, schema2.users_unique_nulls_not_distinct, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" + "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.employeesforview, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.sales_unique_nulls_not_distinct, public.sales_unique_nulls_not_distinct_alter, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.users_unique_nulls_distinct, public.users_unique_nulls_not_distinct, public.users_unique_nulls_not_distinct_index, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.employeesforview, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.sales_unique_nulls_not_distinct, schema2.sales_unique_nulls_not_distinct_alter, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.users_unique_nulls_distinct, schema2.users_unique_nulls_not_distinct, schema2.users_unique_nulls_not_distinct_index, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { "ObjectType": "INDEX", - "TotalCount": 26, - "InvalidCount": 22, - "ObjectNames": "idx8 ON public.combined_tbl, idx9 ON public.combined_tbl, idx1 ON public.combined_tbl, idx2 ON public.combined_tbl, idx3 ON public.combined_tbl, idx4 ON public.combined_tbl, idx5 ON public.combined_tbl, idx6 ON public.combined_tbl, idx7 ON public.combined_tbl, idx_array ON public.documents, idx_box_data ON public.mixed_data_types_table1, idx_box_data_brin ON public.mixed_data_types_table1, idx_citext ON public.citext_type, idx_citext1 ON public.citext_type, idx_citext2 ON public.citext_type, idx_inet ON public.inet_type, idx_inet1 ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_point_data ON public.mixed_data_types_table1, idx_valid ON public.test_jsonb, tsquery_idx ON public.ts_query_table, tsvector_idx ON public.documents, idx_box_data ON schema2.mixed_data_types_table1, idx_box_data_spgist ON schema2.mixed_data_types_table1, idx_point_data ON schema2.mixed_data_types_table1" + "TotalCount": 28, + "InvalidCount": 24, + "ObjectNames": "idx1 ON public.combined_tbl, idx2 ON public.combined_tbl, idx3 ON public.combined_tbl, idx4 ON public.combined_tbl, idx5 ON public.combined_tbl, idx6 ON public.combined_tbl, idx7 ON public.combined_tbl, idx8 ON public.combined_tbl, idx9 ON public.combined_tbl, idx_array ON public.documents, idx_box_data ON public.mixed_data_types_table1, idx_box_data ON schema2.mixed_data_types_table1, idx_box_data_brin ON public.mixed_data_types_table1, idx_box_data_spgist ON schema2.mixed_data_types_table1, idx_citext ON public.citext_type, idx_citext1 ON public.citext_type, idx_citext2 ON public.citext_type, idx_inet ON public.inet_type, idx_inet1 ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_point_data ON public.mixed_data_types_table1, idx_point_data ON schema2.mixed_data_types_table1, idx_valid ON public.test_jsonb, tsquery_idx ON public.ts_query_table, tsvector_idx ON public.documents, users_unique_nulls_not_distinct_index_email ON public.users_unique_nulls_not_distinct_index, users_unique_nulls_not_distinct_index_email ON schema2.users_unique_nulls_not_distinct_index" }, { "ObjectType": "FUNCTION", @@ -75,7 +75,7 @@ "ObjectType": "VIEW", "TotalCount": 10, "InvalidCount": 6, - "ObjectNames": "public.ordersentry_view, public.sales_employees, schema2.sales_employees, test_views.v1, test_views.v2, test_views.v3, test_views.v4, public.view_explicit_security_invoker, schema2.top_employees_view, public.top_employees_view" + "ObjectNames": "public.ordersentry_view, public.sales_employees, public.top_employees_view, public.view_explicit_security_invoker, schema2.sales_employees, schema2.top_employees_view, test_views.v1, test_views.v2, test_views.v3, test_views.v4" }, { "ObjectType": "TRIGGER", @@ -190,13 +190,15 @@ "schema2.users_unique_nulls_distinct", "schema2.users_unique_nulls_not_distinct", "schema2.sales_unique_nulls_not_distinct", + "public.users_unique_nulls_not_distinct_index", + "schema2.users_unique_nulls_not_distinct_index", "schema2.sales_unique_nulls_not_distinct_alter", "public.users_unique_nulls_distinct", "public.users_unique_nulls_not_distinct", "public.sales_unique_nulls_not_distinct", "public.sales_unique_nulls_not_distinct_alter" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 102 objects (86 tables/materialized views and 16 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 108 objects (90 tables/materialized views and 18 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -682,7 +684,7 @@ "Objects": [ { "ObjectName": "public.view_explicit_security_invoker", - "SqlStatement": "CREATE VIEW public.view_explicit_security_invoker WITH (security_invoker='true') AS\n SELECT employees.employee_id,\n employees.first_name\n FROM public.employees;" + "SqlStatement": "CREATE VIEW public.view_explicit_security_invoker WITH (security_invoker='true') AS\n SELECT employee_id,\n first_name\n FROM public.employees;" } ], "MinimumVersionsFixedIn": null @@ -690,6 +692,14 @@ { "FeatureName": "Unique Nulls Not Distinct", "Objects": [ + { + "ObjectName": "users_unique_nulls_not_distinct_index_email ON public.users_unique_nulls_not_distinct_index", + "SqlStatement": "CREATE UNIQUE INDEX users_unique_nulls_not_distinct_index_email ON public.users_unique_nulls_not_distinct_index USING btree (email) NULLS NOT DISTINCT;" + }, + { + "ObjectName": "users_unique_nulls_not_distinct_index_email ON schema2.users_unique_nulls_not_distinct_index", + "SqlStatement": "CREATE UNIQUE INDEX users_unique_nulls_not_distinct_index_email ON schema2.users_unique_nulls_not_distinct_index USING btree (email) NULLS NOT DISTINCT;" + }, { "ObjectName": "public.sales_unique_nulls_not_distinct", "SqlStatement": "ALTER TABLE ONLY public.sales_unique_nulls_not_distinct\n ADD CONSTRAINT sales_unique_nulls_not_distin_store_id_product_id_sale_date_key UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date);" diff --git a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql index f5ef11c5e..e8882d3a9 100644 --- a/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql +++ b/migtests/tests/pg/assessment-report-test/pg_assessment_report.sql @@ -461,4 +461,14 @@ CREATE TABLE sales_unique_nulls_not_distinct_alter ( ALTER TABLE sales_unique_nulls_not_distinct_alter ADD CONSTRAINT sales_unique_nulls_not_distinct_alter_unique UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date); +-- Create a unique index on a column with NULLs with the NULLS NOT DISTINCT option +CREATE TABLE users_unique_nulls_not_distinct_index ( + id INTEGER PRIMARY KEY, + email TEXT +); + +CREATE UNIQUE INDEX users_unique_nulls_not_distinct_index_email + ON users_unique_nulls_not_distinct_index (email) + NULLS NOT DISTINCT; + diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index 35f9e2684..e5cf0a8da 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -19,7 +19,9 @@ import ( "slices" mapset "github.com/deckarep/golang-set/v2" + pg_query "github.com/pganalyze/pg_query_go/v6" log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" @@ -466,24 +468,20 @@ func NewUniqueNullsNotDistinctDetector(query string) *UniqueNullsNotDistinctDete // Detect checks if a unique constraint is defined which has nulls not distinct func (d *UniqueNullsNotDistinctDetector) Detect(msg protoreflect.Message) error { - // If message is of type PG_QUERY_TABLECONSTRAINT_NODE - if !(queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_TABLECONSTRAINT_NODE) { - return nil - } - - // Fetch contype as an enum number - constraintType := queryparser.GetEnumNumField(msg, "contype") - constraintTypeStr := queryparser.GetResolvedEnumName(msg, "contype", constraintType) + if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_INDEX_STMT_NODE { + proto := msg.Interface().(proto.Message) + indexStmt := proto.(*pg_query.IndexStmt) - // Check if the constraint is of type CONSTR_UNIQUE - if constraintTypeStr != "CONSTR_UNIQUE" { - return nil - } + if indexStmt.Unique && indexStmt.NullsNotDistinct { + d.detected = true + } + } else if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_TABLECONSTRAINT_NODE { + proto := msg.Interface().(proto.Message) + constraintNode := proto.(*pg_query.Constraint) - // Check if the constraint has nulls not distinct - nullsNotDistinct := queryparser.GetBoolField(msg, "nulls_not_distinct") - if nullsNotDistinct { - d.detected = true + if constraintNode.Contype == pg_query.ConstrType_CONSTR_UNIQUE && constraintNode.NullsNotDistinct { + d.detected = true + } } return nil diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index c0eaff60a..97e364f92 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -170,6 +170,7 @@ CHECK (xpath_exists('/invoice/customer', data));` UNIQUE NULLS NOT DISTINCT (product_name, serial_number) );` stmt22 = `ALTER TABLE public.products ADD CONSTRAINT unique_product_name UNIQUE NULLS NOT DISTINCT (product_name);` + stmt23 = `CREATE UNIQUE INDEX unique_email_idx ON users (email) NULLS NOT DISTINCT;` ) func modifiedIssuesforPLPGSQL(issues []QueryIssue, objType string, objName string) []QueryIssue { @@ -299,6 +300,9 @@ func TestDDLIssues(t *testing.T) { stmt22: []QueryIssue{ NewUniqueNullsNotDistinctIssue("TABLE", "public.products", stmt22), }, + stmt23: []QueryIssue{ + NewUniqueNullsNotDistinctIssue("INDEX", "unique_email_idx ON users", stmt23), + }, } for _, stmt := range requiredDDLs { err := parserIssueDetector.ParseRequiredDDLs(stmt) @@ -313,7 +317,7 @@ func TestDDLIssues(t *testing.T) { found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { return cmp.Equal(expectedIssue, queryIssue) }) - assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) + assert.True(t, found, "Expected issue not found: %v in statement: %s. \nFound: %v", expectedIssue, stmt, issues) } } } diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index aadfdfa20..73d76b7e1 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -395,27 +395,6 @@ func GetEnumNumField(msg protoreflect.Message, fieldName string) protoreflect.En return 0 } -// GetResolvedEnumName resolves the enum value to its symbolic name. -func GetResolvedEnumName(msg protoreflect.Message, fieldName string, enumNum protoreflect.EnumNumber) string { - fieldDescriptor := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) - if fieldDescriptor == nil { - return "UNKNOWN" - } - - enumDescriptor := fieldDescriptor.Enum() - if enumDescriptor == nil { - return "UNKNOWN" - } - - // Resolve the enum value to its symbolic name - resolvedName := enumDescriptor.Values().ByNumber(protoreflect.EnumNumber(enumNum)).Name() - - if resolvedName == "" { - return "UNKNOWN" - } - return string(resolvedName) -} - // GetSchemaAndObjectName extracts the schema and object name from a list. func GetSchemaAndObjectName(nameList protoreflect.List) (string, string) { var schemaName, objectName string diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index 9177afc10..26038fc09 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -54,6 +54,7 @@ const ( PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" PG_QUERY_COPYSTSMT_NODE = "pg_query.CopyStmt" PG_QUERY_CONSTRAINT_NODE = "pg_query.Constraint" + PG_QUERY_INDEX_STMT_NODE = "pg_query.IndexStmt" ) // function type for processing nodes during traversal From 29983198e004fa3fee36edfe4d449a1df8e71366 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Mon, 6 Jan 2025 13:30:32 +0000 Subject: [PATCH 07/16] Made changes to fix merge conflicts --- migtests/scripts/postgresql/env.sh | 4 ++-- migtests/tests/analyze-schema/summary.json | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/migtests/scripts/postgresql/env.sh b/migtests/scripts/postgresql/env.sh index fcc466b82..4068315a4 100644 --- a/migtests/scripts/postgresql/env.sh +++ b/migtests/scripts/postgresql/env.sh @@ -1,6 +1,6 @@ export SOURCE_DB_HOST=${SOURCE_DB_HOST:-"127.0.0.1"} export SOURCE_DB_PORT=${SOURCE_DB_PORT:-5432} -export SOURCE_DB_USER=${SOURCE_DB_USER:-"ybvoyager"} -export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'Test@123#$%^&*()!'} +export SOURCE_DB_USER=${SOURCE_DB_USER:-"postgres"} +export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'postgres'} export SOURCE_DB_ADMIN_USER=${SOURCE_DB_ADMIN_USER:-"postgres"} export SOURCE_DB_ADMIN_PASSWORD=${SOURCE_DB_ADMIN_PASSWORD:-"postgres"} diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index e8eb69241..e496be8fb 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -26,9 +26,9 @@ }, { "ObjectType": "TABLE", - "TotalCount": 62, - "InvalidCount": 52, - "ObjectNames": "test_table_in_type_file, sales_data, salaries2, sales, test_1, test_2, test_non_pk_multi_column_list, test_3, test_4, test_5, test_6, test_7, test_8, test_9, order_details, public.employees4, enum_example.bugs, table_xyz, table_abc, table_1, table_test, test_interval, public.range_columns_partition_test, public.range_columns_partition_test_copy, anydata_test, anydataset_test, anytype_test, uritype_test, \"Test\", public.meeting, public.pr, public.foreign_def_test, public.users, foreign_def_test1, foreign_def_test2, unique_def_test, unique_def_test1, test_xml_type, test_xid_type, public.test_jsonb, public.inet_type, public.citext_type, public.documents, public.ts_query_table, combined_tbl, combined_tbl1, test_udt, test_arr_enum, public.locations, public.xml_data_example, image, employees, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table, users_unique_nulls_distinct, users_unique_nulls_not_distinct, sales_unique_nulls_not_distinct, sales_unique_nulls_not_distinct_alter" }, + "TotalCount": 63, + "InvalidCount": 53, + "ObjectNames": "test_table_in_type_file, sales_data, salaries2, sales, test_1, test_2, test_non_pk_multi_column_list, test_3, test_4, test_5, test_6, test_7, test_8, test_9, order_details, public.employees4, enum_example.bugs, table_xyz, table_abc, table_1, table_test, test_interval, public.range_columns_partition_test, public.range_columns_partition_test_copy, anydata_test, anydataset_test, anytype_test, uritype_test, \"Test\", public.meeting, public.pr, public.foreign_def_test, public.users, foreign_def_test1, foreign_def_test2, unique_def_test, unique_def_test1, test_xml_type, test_xid_type, public.test_jsonb, public.inet_type, public.citext_type, public.documents, public.ts_query_table, combined_tbl, combined_tbl1, test_udt, test_arr_enum, public.locations, public.xml_data_example, image, public.json_data, employees, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table, users_unique_nulls_distinct, users_unique_nulls_not_distinct, sales_unique_nulls_not_distinct, sales_unique_nulls_not_distinct_alter" }, { "ObjectType": "INDEX", "TotalCount": 43, From bda4dfcc0253a64db86a9187430d1d966b1daa28 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Tue, 7 Jan 2025 06:03:38 +0000 Subject: [PATCH 08/16] Made changes to fix merge conflicts --- migtests/scripts/postgresql/env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migtests/scripts/postgresql/env.sh b/migtests/scripts/postgresql/env.sh index 4068315a4..fcc466b82 100644 --- a/migtests/scripts/postgresql/env.sh +++ b/migtests/scripts/postgresql/env.sh @@ -1,6 +1,6 @@ export SOURCE_DB_HOST=${SOURCE_DB_HOST:-"127.0.0.1"} export SOURCE_DB_PORT=${SOURCE_DB_PORT:-5432} -export SOURCE_DB_USER=${SOURCE_DB_USER:-"postgres"} -export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'postgres'} +export SOURCE_DB_USER=${SOURCE_DB_USER:-"ybvoyager"} +export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'Test@123#$%^&*()!'} export SOURCE_DB_ADMIN_USER=${SOURCE_DB_ADMIN_USER:-"postgres"} export SOURCE_DB_ADMIN_PASSWORD=${SOURCE_DB_ADMIN_PASSWORD:-"postgres"} From 52738505c6006dd8490c60db9ea0d749081de9ab Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Tue, 7 Jan 2025 06:10:31 +0000 Subject: [PATCH 09/16] Added test case to analyze schema test --- .../dummy-export-dir/schema/tables/INDEXES_table.sql | 7 ++++++- .../dummy-export-dir/schema/tables/table.sql | 6 ++++++ migtests/tests/analyze-schema/expected_issues.json | 10 ++++++++++ migtests/tests/analyze-schema/summary.json | 10 +++++----- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/INDEXES_table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/INDEXES_table.sql index efc811b0f..3457b1c67 100644 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/INDEXES_table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/INDEXES_table.sql @@ -103,4 +103,9 @@ CREATE INDEX idx_udt1 on test_udt(home_address1); CREATE INDEX idx_enum on test_udt(some_field); -CREATE INDEX "idx&_enum2" on test_udt((some_field::non_public.enum_test)); \ No newline at end of file +CREATE INDEX "idx&_enum2" on test_udt((some_field::non_public.enum_test)); + +-- Create a unique index on a column with NULLs with the NULLS NOT DISTINCT option +CREATE UNIQUE INDEX users_unique_nulls_not_distinct_index_email + ON users_unique_nulls_not_distinct_index (email) + NULLS NOT DISTINCT; diff --git a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql index 3f2801fa6..48908efa2 100755 --- a/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql +++ b/migtests/tests/analyze-schema/dummy-export-dir/schema/tables/table.sql @@ -459,3 +459,9 @@ CREATE TABLE sales_unique_nulls_not_distinct_alter ( ALTER TABLE sales_unique_nulls_not_distinct_alter ADD CONSTRAINT sales_unique_nulls_not_distinct_alter_unique UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date); + +-- Create a unique index on a column with NULLs with the NULLS NOT DISTINCT option +CREATE TABLE users_unique_nulls_not_distinct_index ( + id INTEGER PRIMARY KEY, + email TEXT +); diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index b1c52a9e6..428c7663d 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -2090,5 +2090,15 @@ "Suggestion": "", "GH": "", "MinimumVersionsFixedIn": null + }, + { + "IssueType": "unsupported_features", + "ObjectType": "INDEX", + "ObjectName": "users_unique_nulls_not_distinct_index_email ON users_unique_nulls_not_distinct_index", + "Reason": "UNIQUE NULLS NOT DISTINCT", + "SqlStatement": "CREATE UNIQUE INDEX users_unique_nulls_not_distinct_index_email\n ON users_unique_nulls_not_distinct_index (email)\n NULLS NOT DISTINCT;", + "Suggestion": "", + "GH": "", + "MinimumVersionsFixedIn": null } ] diff --git a/migtests/tests/analyze-schema/summary.json b/migtests/tests/analyze-schema/summary.json index e496be8fb..2052fbd69 100644 --- a/migtests/tests/analyze-schema/summary.json +++ b/migtests/tests/analyze-schema/summary.json @@ -26,14 +26,14 @@ }, { "ObjectType": "TABLE", - "TotalCount": 63, + "TotalCount": 64, "InvalidCount": 53, - "ObjectNames": "test_table_in_type_file, sales_data, salaries2, sales, test_1, test_2, test_non_pk_multi_column_list, test_3, test_4, test_5, test_6, test_7, test_8, test_9, order_details, public.employees4, enum_example.bugs, table_xyz, table_abc, table_1, table_test, test_interval, public.range_columns_partition_test, public.range_columns_partition_test_copy, anydata_test, anydataset_test, anytype_test, uritype_test, \"Test\", public.meeting, public.pr, public.foreign_def_test, public.users, foreign_def_test1, foreign_def_test2, unique_def_test, unique_def_test1, test_xml_type, test_xid_type, public.test_jsonb, public.inet_type, public.citext_type, public.documents, public.ts_query_table, combined_tbl, combined_tbl1, test_udt, test_arr_enum, public.locations, public.xml_data_example, image, public.json_data, employees, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table, users_unique_nulls_distinct, users_unique_nulls_not_distinct, sales_unique_nulls_not_distinct, sales_unique_nulls_not_distinct_alter" }, + "ObjectNames": "test_table_in_type_file, sales_data, salaries2, sales, test_1, test_2, test_non_pk_multi_column_list, test_3, test_4, test_5, test_6, test_7, test_8, test_9, order_details, public.employees4, enum_example.bugs, table_xyz, table_abc, table_1, table_test, test_interval, public.range_columns_partition_test, public.range_columns_partition_test_copy, anydata_test, anydataset_test, anytype_test, uritype_test, \"Test\", public.meeting, public.pr, public.foreign_def_test, public.users, foreign_def_test1, foreign_def_test2, unique_def_test, unique_def_test1, test_xml_type, test_xid_type, public.test_jsonb, public.inet_type, public.citext_type, public.documents, public.ts_query_table, combined_tbl, combined_tbl1, test_udt, test_arr_enum, public.locations, public.xml_data_example, image, public.json_data, employees, bigint_multirange_table, date_multirange_table, int_multirange_table, numeric_multirange_table, timestamp_multirange_table, timestamptz_multirange_table, users_unique_nulls_distinct, users_unique_nulls_not_distinct, sales_unique_nulls_not_distinct, sales_unique_nulls_not_distinct_alter, users_unique_nulls_not_distinct_index" }, { "ObjectType": "INDEX", - "TotalCount": 43, - "InvalidCount": 39, - "ObjectNames": "idx1 ON combined_tbl1, idx2 ON combined_tbl1, idx3 ON combined_tbl1, idx4 ON combined_tbl1, idx5 ON combined_tbl1, idx6 ON combined_tbl1, idx7 ON combined_tbl1, idx8 ON combined_tbl1, film_fulltext_idx ON public.film, idx_actor_last_name ON public.actor, idx_name1 ON table_name, idx_name2 ON table_name, idx_name3 ON schema_name.table_name, idx_fileinfo_name_splitted ON public.fileinfo, abc ON public.example, abc ON schema2.example, tsvector_idx ON public.documents, tsquery_idx ON public.ts_query_table, idx_citext ON public.citext_type, idx_inet ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_valid ON public.test_jsonb, idx_array ON public.documents, idx1 ON combined_tbl, idx2 ON combined_tbl, idx3 ON combined_tbl, idx4 ON combined_tbl, idx5 ON combined_tbl, idx6 ON combined_tbl, idx7 ON combined_tbl, idx8 ON combined_tbl, idx9 ON combined_tbl, idx10 ON combined_tbl, idx11 ON combined_tbl, idx12 ON combined_tbl, idx13 ON combined_tbl, idx14 ON combined_tbl, idx15 ON combined_tbl, idx_udt ON test_udt, idx_udt1 ON test_udt, idx_enum ON test_udt, \"idx\u0026_enum2\" ON test_udt", + "TotalCount": 44, + "InvalidCount": 40, + "ObjectNames": "film_fulltext_idx ON public.film, idx_actor_last_name ON public.actor, idx_name1 ON table_name, idx_name2 ON table_name, idx_name3 ON schema_name.table_name, idx_fileinfo_name_splitted ON public.fileinfo, abc ON public.example, abc ON schema2.example, tsvector_idx ON public.documents, tsquery_idx ON public.ts_query_table, idx_citext ON public.citext_type, idx_inet ON public.inet_type, idx_json ON public.test_jsonb, idx_json2 ON public.test_jsonb, idx_valid ON public.test_jsonb, idx_array ON public.documents, idx1 ON combined_tbl, idx2 ON combined_tbl, idx3 ON combined_tbl, idx4 ON combined_tbl, idx5 ON combined_tbl, idx6 ON combined_tbl, idx7 ON combined_tbl, idx8 ON combined_tbl, idx9 ON combined_tbl, idx10 ON combined_tbl, idx11 ON combined_tbl, idx12 ON combined_tbl, idx13 ON combined_tbl, idx14 ON combined_tbl, idx15 ON combined_tbl, idx1 ON combined_tbl1, idx2 ON combined_tbl1, idx3 ON combined_tbl1, idx4 ON combined_tbl1, idx5 ON combined_tbl1, idx6 ON combined_tbl1, idx7 ON combined_tbl1, idx8 ON combined_tbl1, idx_udt ON test_udt, idx_udt1 ON test_udt, idx_enum ON test_udt, \"idx\u0026_enum2\" ON test_udt, users_unique_nulls_not_distinct_index_email ON users_unique_nulls_not_distinct_index", "Details": "There are some GIN indexes present in the schema, but GIN indexes are partially supported in YugabyteDB as mentioned in (https://github.com/yugabyte/yugabyte-db/issues/7850) so take a look and modify them if not supported." }, { From 0d14beed0bb5715a94e7a30e60104ee08576901e Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Tue, 7 Jan 2025 11:51:32 +0000 Subject: [PATCH 10/16] Made the suggested changes --- yb-voyager/src/query/queryissue/detectors.go | 17 +++++++++++++---- .../src/query/queryparser/helpers_protomsg.go | 12 ++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index e5cf0a8da..5c29aa0eb 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -16,6 +16,7 @@ limitations under the License. package queryissue import ( + "fmt" "slices" mapset "github.com/deckarep/golang-set/v2" @@ -469,15 +470,23 @@ func NewUniqueNullsNotDistinctDetector(query string) *UniqueNullsNotDistinctDete // Detect checks if a unique constraint is defined which has nulls not distinct func (d *UniqueNullsNotDistinctDetector) Detect(msg protoreflect.Message) error { if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_INDEX_STMT_NODE { - proto := msg.Interface().(proto.Message) - indexStmt := proto.(*pg_query.IndexStmt) + indexStmt, err := queryparser.ProtoAsIndexStmt(msg) + if err != nil { + return err + } if indexStmt.Unique && indexStmt.NullsNotDistinct { d.detected = true } } else if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_TABLECONSTRAINT_NODE { - proto := msg.Interface().(proto.Message) - constraintNode := proto.(*pg_query.Constraint) + proto, ok := msg.Interface().(proto.Message) + if !ok { + return fmt.Errorf("failed to cast msg to proto.Message") + } + constraintNode, ok := proto.(*pg_query.Constraint) + if !ok { + return fmt.Errorf("failed to cast msg to %s", queryparser.PG_QUERY_TABLECONSTRAINT_NODE) + } if constraintNode.Contype == pg_query.ConstrType_CONSTR_UNIQUE && constraintNode.NullsNotDistinct { d.detected = true diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index 73d76b7e1..125b06f37 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -420,6 +420,18 @@ func ProtoAsSelectStmt(msg protoreflect.Message) (*pg_query.SelectStmt, error) { return selectStmtNode, nil } +func ProtoAsIndexStmt(msg protoreflect.Message) (*pg_query.IndexStmt, error) { + protoMsg, ok := msg.Interface().(proto.Message) + if !ok { + return nil, fmt.Errorf("failed to cast msg to proto.Message") + } + indexStmtNode, ok := protoMsg.(*pg_query.IndexStmt) + if !ok { + return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_INDEX_STMT_NODE) + } + return indexStmtNode, nil +} + /* Example: options:{def_elem:{defname:"security_invoker" arg:{string:{sval:"true"}} defaction:DEFELEM_UNSPEC location:32}} From d4824272ee63ecf9890a4d84ca5207095266661a Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Tue, 7 Jan 2025 14:33:25 +0000 Subject: [PATCH 11/16] Made suggested changes --- yb-voyager/src/query/queryissue/detectors.go | 15 ++++----------- .../src/query/queryparser/helpers_protomsg.go | 12 ++++++++++++ 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index 5c29aa0eb..428ea9741 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -16,13 +16,10 @@ limitations under the License. package queryissue import ( - "fmt" "slices" mapset "github.com/deckarep/golang-set/v2" - pg_query "github.com/pganalyze/pg_query_go/v6" log "github.com/sirupsen/logrus" - "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protoreflect" "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" @@ -479,16 +476,12 @@ func (d *UniqueNullsNotDistinctDetector) Detect(msg protoreflect.Message) error d.detected = true } } else if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_TABLECONSTRAINT_NODE { - proto, ok := msg.Interface().(proto.Message) - if !ok { - return fmt.Errorf("failed to cast msg to proto.Message") - } - constraintNode, ok := proto.(*pg_query.Constraint) - if !ok { - return fmt.Errorf("failed to cast msg to %s", queryparser.PG_QUERY_TABLECONSTRAINT_NODE) + constraintNode, err := queryparser.ProtoAsTableConstraint(msg) + if err != nil { + return err } - if constraintNode.Contype == pg_query.ConstrType_CONSTR_UNIQUE && constraintNode.NullsNotDistinct { + if constraintNode.Contype == queryparser.UNIQUE_CONSTR_TYPE && constraintNode.NullsNotDistinct { d.detected = true } } diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index 125b06f37..b4d375e6b 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -432,6 +432,18 @@ func ProtoAsIndexStmt(msg protoreflect.Message) (*pg_query.IndexStmt, error) { return indexStmtNode, nil } +func ProtoAsTableConstraint(msg protoreflect.Message) (*pg_query.Constraint, error) { + proto, ok := msg.Interface().(proto.Message) + if !ok { + return nil, fmt.Errorf("failed to cast msg to proto.Message") + } + constraintNode, ok := proto.(*pg_query.Constraint) + if !ok { + return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_TABLECONSTRAINT_NODE) + } + return constraintNode, nil +} + /* Example: options:{def_elem:{defname:"security_invoker" arg:{string:{sval:"true"}} defaction:DEFELEM_UNSPEC location:32}} From 7e8fa0079ad5451f18f2cfd34a91fb912384602b Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Thu, 9 Jan 2025 05:51:43 +0000 Subject: [PATCH 12/16] Made some suggested changes --- yb-voyager/src/query/queryissue/detectors.go | 4 ++-- yb-voyager/src/query/queryparser/helpers_protomsg.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index 428ea9741..1ba0a5336 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -466,7 +466,7 @@ func NewUniqueNullsNotDistinctDetector(query string) *UniqueNullsNotDistinctDete // Detect checks if a unique constraint is defined which has nulls not distinct func (d *UniqueNullsNotDistinctDetector) Detect(msg protoreflect.Message) error { - if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_INDEX_STMT_NODE { + if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_INDEXSTMT_NODE { indexStmt, err := queryparser.ProtoAsIndexStmt(msg) if err != nil { return err @@ -475,7 +475,7 @@ func (d *UniqueNullsNotDistinctDetector) Detect(msg protoreflect.Message) error if indexStmt.Unique && indexStmt.NullsNotDistinct { d.detected = true } - } else if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_TABLECONSTRAINT_NODE { + } else if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_CONSTRAINT_NODE { constraintNode, err := queryparser.ProtoAsTableConstraint(msg) if err != nil { return err diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index b4d375e6b..7889fb952 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -427,7 +427,7 @@ func ProtoAsIndexStmt(msg protoreflect.Message) (*pg_query.IndexStmt, error) { } indexStmtNode, ok := protoMsg.(*pg_query.IndexStmt) if !ok { - return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_INDEX_STMT_NODE) + return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_INDEXSTMT_NODE) } return indexStmtNode, nil } @@ -439,7 +439,7 @@ func ProtoAsTableConstraint(msg protoreflect.Message) (*pg_query.Constraint, err } constraintNode, ok := proto.(*pg_query.Constraint) if !ok { - return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_TABLECONSTRAINT_NODE) + return nil, fmt.Errorf("failed to cast msg to %s", PG_QUERY_CONSTRAINT_NODE) } return constraintNode, nil } From 2e0ee2e4c9355f04d6ca4c3d07de7e1ea0162015 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Thu, 9 Jan 2025 07:52:24 +0000 Subject: [PATCH 13/16] Fixing issues after rebase --- yb-voyager/src/query/queryissue/issues_ddl.go | 12 ++++++------ yb-voyager/src/query/queryparser/traversal_proto.go | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/yb-voyager/src/query/queryissue/issues_ddl.go b/yb-voyager/src/query/queryissue/issues_ddl.go index 09110c170..c6eb5f395 100644 --- a/yb-voyager/src/query/queryissue/issues_ddl.go +++ b/yb-voyager/src/query/queryissue/issues_ddl.go @@ -517,12 +517,12 @@ func NewForeignKeyReferencesPartitionedTableIssue(objectType string, objectName } var uniqueNullsNotDistinctIssue = issue.Issue{ - Type: UNIQUE_NULLS_NOT_DISTINCT, - TypeName: "UNIQUE NULLS NOT DISTINCT", - TypeDescription: "", - Suggestion: "", - GH: "", - DocsLink: "", + Type: UNIQUE_NULLS_NOT_DISTINCT, + Name: UNIQUE_NULLS_NOT_DISTINCT_NAME, + Impact: constants.IMPACT_LEVEL_1, + Suggestion: "", + GH: "", + DocsLink: "", } func NewUniqueNullsNotDistinctIssue(objectType string, objectName string, sqlStatement string) QueryIssue { diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index 26038fc09..9bf2b2716 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -54,7 +54,7 @@ const ( PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" PG_QUERY_COPYSTSMT_NODE = "pg_query.CopyStmt" PG_QUERY_CONSTRAINT_NODE = "pg_query.Constraint" - PG_QUERY_INDEX_STMT_NODE = "pg_query.IndexStmt" + PG_QUERY_INDEXSTMT_NODE = "pg_query.IndexStmt" ) // function type for processing nodes during traversal From 36c1014f2286f7023d6982dab40f8d6b7fe36a41 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Thu, 9 Jan 2025 08:01:45 +0000 Subject: [PATCH 14/16] Made changes to analyze schema test --- migtests/tests/analyze-schema/expected_issues.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/migtests/tests/analyze-schema/expected_issues.json b/migtests/tests/analyze-schema/expected_issues.json index 428c7663d..8baa84606 100644 --- a/migtests/tests/analyze-schema/expected_issues.json +++ b/migtests/tests/analyze-schema/expected_issues.json @@ -2065,7 +2065,7 @@ "IssueType": "unsupported_features", "ObjectType": "TABLE", "ObjectName": "users_unique_nulls_not_distinct", - "Reason": "UNIQUE NULLS NOT DISTINCT", + "Reason": "Unique Nulls Not Distinct", "SqlStatement": "CREATE TABLE users_unique_nulls_not_distinct (\n id SERIAL PRIMARY KEY,\n email TEXT,\n UNIQUE NULLS NOT DISTINCT (email)\n);", "Suggestion": "", "GH": "", @@ -2075,7 +2075,7 @@ "IssueType": "unsupported_features", "ObjectType": "TABLE", "ObjectName": "sales_unique_nulls_not_distinct", - "Reason": "UNIQUE NULLS NOT DISTINCT", + "Reason": "Unique Nulls Not Distinct", "SqlStatement": "CREATE TABLE sales_unique_nulls_not_distinct (\n store_id INT,\n product_id INT,\n sale_date DATE,\n UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date)\n);", "Suggestion": "", "GH": "", @@ -2085,7 +2085,7 @@ "IssueType": "unsupported_features", "ObjectType": "TABLE", "ObjectName": "sales_unique_nulls_not_distinct_alter", - "Reason": "UNIQUE NULLS NOT DISTINCT", + "Reason": "Unique Nulls Not Distinct", "SqlStatement": "ALTER TABLE sales_unique_nulls_not_distinct_alter\n\tADD CONSTRAINT sales_unique_nulls_not_distinct_alter_unique UNIQUE NULLS NOT DISTINCT (store_id, product_id, sale_date);", "Suggestion": "", "GH": "", @@ -2095,7 +2095,7 @@ "IssueType": "unsupported_features", "ObjectType": "INDEX", "ObjectName": "users_unique_nulls_not_distinct_index_email ON users_unique_nulls_not_distinct_index", - "Reason": "UNIQUE NULLS NOT DISTINCT", + "Reason": "Unique Nulls Not Distinct", "SqlStatement": "CREATE UNIQUE INDEX users_unique_nulls_not_distinct_index_email\n ON users_unique_nulls_not_distinct_index (email)\n NULLS NOT DISTINCT;", "Suggestion": "", "GH": "", From 3b9ccf79165ee808c4c28f5fcfd31ea103c65868 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Thu, 9 Jan 2025 08:07:57 +0000 Subject: [PATCH 15/16] Fixed assessment test --- migtests/scripts/postgresql/env.sh | 4 +- .../expectedAssessmentReport.json | 64 +++++++++++++++++-- 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/migtests/scripts/postgresql/env.sh b/migtests/scripts/postgresql/env.sh index fcc466b82..4068315a4 100644 --- a/migtests/scripts/postgresql/env.sh +++ b/migtests/scripts/postgresql/env.sh @@ -1,6 +1,6 @@ export SOURCE_DB_HOST=${SOURCE_DB_HOST:-"127.0.0.1"} export SOURCE_DB_PORT=${SOURCE_DB_PORT:-5432} -export SOURCE_DB_USER=${SOURCE_DB_USER:-"ybvoyager"} -export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'Test@123#$%^&*()!'} +export SOURCE_DB_USER=${SOURCE_DB_USER:-"postgres"} +export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'postgres'} export SOURCE_DB_ADMIN_USER=${SOURCE_DB_ADMIN_USER:-"postgres"} export SOURCE_DB_ADMIN_PASSWORD=${SOURCE_DB_ADMIN_PASSWORD:-"postgres"} diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index fcf036e9e..9ded5a5ed 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -44,9 +44,9 @@ }, { "ObjectType": "TABLE", - "TotalCount": 92, - "InvalidCount": 41, - "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employees3, public.employeesforview, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.sales_unique_nulls_not_distinct, public.sales_unique_nulls_not_distinct_alter, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.users_unique_nulls_distinct, public.users_unique_nulls_not_distinct, public.users_unique_nulls_not_distinct_index, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.employeesforview, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.sales_unique_nulls_not_distinct, schema2.sales_unique_nulls_not_distinct_alter, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.users_unique_nulls_distinct, schema2.users_unique_nulls_not_distinct, schema2.users_unique_nulls_not_distinct_index, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" + "TotalCount": 93, + "InvalidCount": 42, + "ObjectNames": "public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.bigint_multirange_table, public.boston, public.c, public.child_table, public.citext_type, public.combined_tbl, public.date_multirange_table, public.documents, public.employees, public.employees2, public.employeescopyfromwhere, public.employeescopyonerror, public.employeesforview, public.ext_test, public.foo, public.inet_type, public.int_multirange_table, public.library_nested, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.numeric_multirange_table, public.orders, public.orders2, public.orders_lateral, public.ordersentry, public.parent_table, public.products, public.sales_region, public.sales_unique_nulls_not_distinct, public.sales_unique_nulls_not_distinct_alter, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.timestamp_multirange_table, public.timestamptz_multirange_table, public.ts_query_table, public.tt, public.users_unique_nulls_distinct, public.users_unique_nulls_not_distinct, public.users_unique_nulls_not_distinct_index, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.bigint_multirange_table, schema2.boston, schema2.c, schema2.child_table, schema2.date_multirange_table, schema2.employees2, schema2.employeesforview, schema2.ext_test, schema2.foo, schema2.int_multirange_table, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.numeric_multirange_table, schema2.orders, schema2.orders2, schema2.parent_table, schema2.products, schema2.sales_region, schema2.sales_unique_nulls_not_distinct, schema2.sales_unique_nulls_not_distinct_alter, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.timestamp_multirange_table, schema2.timestamptz_multirange_table, schema2.tt, schema2.users_unique_nulls_distinct, schema2.users_unique_nulls_not_distinct, schema2.users_unique_nulls_not_distinct_index, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2" }, { "ObjectType": "INDEX", @@ -198,7 +198,7 @@ "public.sales_unique_nulls_not_distinct", "public.sales_unique_nulls_not_distinct_alter" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 108 objects (90 tables/materialized views and 18 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 109 objects (91 tables/materialized views and 18 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -2591,6 +2591,62 @@ "ObjectType": "", "ParentTableName": null, "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "users_unique_nulls_not_distinct_index_email", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "public.users_unique_nulls_not_distinct_index", + "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "users_unique_nulls_not_distinct_index", + "RowCount": 0, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "users_unique_nulls_not_distinct_index", + "RowCount": 0, + "ColumnCount": 2, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 0 + }, + { + "SchemaName": "schema2", + "ObjectName": "users_unique_nulls_not_distinct_index_email", + "RowCount": null, + "ColumnCount": 1, + "Reads": 0, + "Writes": 0, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": true, + "ObjectType": "", + "ParentTableName": "schema2.users_unique_nulls_not_distinct_index", + "SizeInBytes": 8192 } ], From f4e546fc0a7f6a91e3cd2be943a3ccd4df8a8d04 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Thu, 9 Jan 2025 08:08:30 +0000 Subject: [PATCH 16/16] Fixed assessment test --- migtests/scripts/postgresql/env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migtests/scripts/postgresql/env.sh b/migtests/scripts/postgresql/env.sh index 4068315a4..fcc466b82 100644 --- a/migtests/scripts/postgresql/env.sh +++ b/migtests/scripts/postgresql/env.sh @@ -1,6 +1,6 @@ export SOURCE_DB_HOST=${SOURCE_DB_HOST:-"127.0.0.1"} export SOURCE_DB_PORT=${SOURCE_DB_PORT:-5432} -export SOURCE_DB_USER=${SOURCE_DB_USER:-"postgres"} -export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'postgres'} +export SOURCE_DB_USER=${SOURCE_DB_USER:-"ybvoyager"} +export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'Test@123#$%^&*()!'} export SOURCE_DB_ADMIN_USER=${SOURCE_DB_ADMIN_USER:-"postgres"} export SOURCE_DB_ADMIN_PASSWORD=${SOURCE_DB_ADMIN_PASSWORD:-"postgres"}