From c7e19305c4226e5e154948cc211bc6970f79967c Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Thu, 19 Dec 2024 07:21:14 +0000 Subject: [PATCH 01/12] Added a new detector for catching unsupported COPY command structure --- yb-voyager/src/query/queryissue/detectors.go | 76 +++++++++++++++---- .../src/query/queryissue/detectors_test.go | 47 ++++++++++++ yb-voyager/src/query/queryissue/issues_dml.go | 26 +++++++ .../query/queryissue/parser_issue_detector.go | 1 + .../src/query/queryparser/traversal_proto.go | 1 + 5 files changed, 138 insertions(+), 13 deletions(-) diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index 8a57c581d..f8b944397 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -204,6 +204,56 @@ func (d *RangeTableFuncDetector) GetIssues() []QueryIssue { return issues } +type CopyCommandUnsupportedConstructsDetector struct{} + +func NewCopyCommandUnsupportedConstructsDetector() *CopyCommandUnsupportedConstructsDetector { + return &CopyCommandUnsupportedConstructsDetector{} +} + +// Detect if COPY command uses unsupported syntax i.e. COPY FROM ... WHERE and COPY... ON_ERROR +func (d *CopyCommandUnsupportedConstructsDetector) Detect(msg protoreflect.Message) ([]string, error) { + unsupportedConstructs := []string{} + + // Check if the message is a COPY statement + if msg.Descriptor().FullName() != queryparser.PG_QUERY_COPYSTSMT_NODE { + return unsupportedConstructs, nil // Not a COPY statement, nothing to detect + } + + // Check for COPY FROM ... WHERE clause + isFromField := msg.Descriptor().Fields().ByName("is_from") + whereField := msg.Descriptor().Fields().ByName("where_clause") + if isFromField != nil && msg.Has(isFromField) { + isFrom := msg.Get(isFromField).Bool() + if isFrom && whereField != nil && msg.Has(whereField) { + unsupportedConstructs = append(unsupportedConstructs, COPY_FROM_WHERE_NAME) + } + } + + // Check for COPY ... ON_ERROR clause + optionsField := msg.Descriptor().Fields().ByName("options") + if optionsField != nil && msg.Has(optionsField) { + optionsList := msg.Get(optionsField).List() + for i := 0; i < optionsList.Len(); i++ { + option := optionsList.Get(i).Message() + + // Check for nested def_elem field + defElemField := option.Descriptor().Fields().ByName("def_elem") + if defElemField != nil && option.Has(defElemField) { + defElem := option.Get(defElemField).Message() + defNameField := defElem.Descriptor().Fields().ByName("defname") + if defNameField != nil && defElem.Has(defNameField) { + defName := defElem.Get(defNameField).String() + if defName == "on_error" { + unsupportedConstructs = append(unsupportedConstructs, COPY_ON_ERROR_NAME) + } + } + } + } + } + + return unsupportedConstructs, nil +} + type JsonConstructorFuncDetector struct { query string unsupportedJsonConstructorFunctionsDetected mapset.Set[string] @@ -253,19 +303,19 @@ func NewJsonQueryFunctionDetector(query string) *JsonQueryFunctionDetector { func (j *JsonQueryFunctionDetector) Detect(msg protoreflect.Message) error { if queryparser.GetMsgFullName(msg) == queryparser.PG_QUERY_JSON_TABLE_NODE { /* - SELECT * FROM json_table( - '[{"a":10,"b":20},{"a":30,"b":40}]'::jsonb, - '$[*]' - COLUMNS ( - column_a int4 path '$.a', - column_b int4 path '$.b' - ) - ); - stmts:{stmt:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{a_star:{}} location:530}} location:530}} - from_clause:{json_table:{context_item:{raw_expr:{type_cast:{arg:{a_const:{sval:{sval:"[{\"a\":10,\"b\":20},{\"a\":30,\"b\":40}]"} - location:553}} type_name:{names:{string:{sval:"jsonb"}} ..... name_location:-1 location:601} - columns:{json_table_column:{coltype:JTC_REGULAR name:"column_a" type_name:{names:{string:{sval:"int4"}} typemod:-1 location:639} - pathspec:{string:{a_const:{sval:{sval:"$.a"} location:649}} name_location:-1 location:649} ... + SELECT * FROM json_table( + '[{"a":10,"b":20},{"a":30,"b":40}]'::jsonb, + '$[*]' + COLUMNS ( + column_a int4 path '$.a', + column_b int4 path '$.b' + ) + ); + stmts:{stmt:{select_stmt:{target_list:{res_target:{val:{column_ref:{fields:{a_star:{}} location:530}} location:530}} + from_clause:{json_table:{context_item:{raw_expr:{type_cast:{arg:{a_const:{sval:{sval:"[{\"a\":10,\"b\":20},{\"a\":30,\"b\":40}]"} + location:553}} type_name:{names:{string:{sval:"jsonb"}} ..... name_location:-1 location:601} + columns:{json_table_column:{coltype:JTC_REGULAR name:"column_a" type_name:{names:{string:{sval:"int4"}} typemod:-1 location:639} + pathspec:{string:{a_const:{sval:{sval:"$.a"} location:649}} name_location:-1 location:649} ... */ j.unsupportedJsonQueryFunctionsDetected.Add(JSON_TABLE) return nil diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index 266f053ba..ce87f4fb8 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -27,6 +27,53 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" ) +func TestCopyCommandUnsupportedConstructsDetector(t *testing.T) { + copyCommandSqlsMap := map[string][]string{ + // Valid COPY commands without WHERE or ON_ERROR + `COPY my_table FROM '/path/to/data.csv' WITH (FORMAT csv);`: {}, + `COPY my_table FROM '/path/to/data.csv' WITH (FORMAT text);`: {}, + `COPY my_table FROM '/path/to/data.csv';`: {}, + `COPY my_table FROM '/path/to/data.csv' WITH (DELIMITER ',');`: {}, + `COPY my_table(col1, col2) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true);`: {}, + + // COPY commands with WHERE clause + `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`: {COPY_FROM_WHERE_NAME}, + `COPY my_table(col1, col2) FROM '/path/to/data.csv' WHERE col2 = 'test';`: {COPY_FROM_WHERE_NAME}, + `COPY my_table FROM '/path/to/data.csv' WHERE TRUE;`: {COPY_FROM_WHERE_NAME}, + + // COPY commands with ON_ERROR clause + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`: {COPY_ON_ERROR_NAME}, + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP);`: {COPY_ON_ERROR_NAME}, + + // COPY commands with both ON_ERROR and WHERE clause + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE) WHERE age > 18;`: {COPY_FROM_WHERE_NAME, COPY_ON_ERROR_NAME}, + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP) WHERE name = 'Alice';`: {COPY_FROM_WHERE_NAME, COPY_ON_ERROR_NAME}, + } + + detector := NewCopyCommandUnsupportedConstructsDetector() + for sql, expectedConstructs := range copyCommandSqlsMap { + parseResult, err := queryparser.Parse(sql) + assert.NoError(t, err, "Failed to parse SQL: %s", sql) + + visited := make(map[protoreflect.Message]bool) + unsupportedConstructs := []string{} + + processor := func(msg protoreflect.Message) error { + constructs, err := detector.Detect(msg) + if err != nil { + return err + } + unsupportedConstructs = append(unsupportedConstructs, constructs...) + return nil + } + + parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) + err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) + assert.NoError(t, err) + assert.ElementsMatch(t, expectedConstructs, unsupportedConstructs, "Unsupported Constructs not detected in SQL: %s", sql) + } +} + func getDetectorIssues(t *testing.T, detector UnsupportedConstructDetector, sql string) []QueryIssue { parseResult, err := queryparser.Parse(sql) assert.NoError(t, err, "Failed to parse SQL: %s", sql) diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 43dc8ab07..2a0257f6a 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -141,3 +141,29 @@ func NewLOFuntionsIssue(objectType string, objectName string, sqlStatement strin } return newQueryIssue(loFunctionsIssue, objectType, objectName, sqlStatement, details) } + +var copyFromWhereIssue = issue.Issue{ + Type: COPY_FROM_WHERE, + TypeName: "COPY FROM ... WHERE", + TypeDescription: "", + Suggestion: "", + GH: "", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#copy-from-where-is-not-yet-supported", +} + +func NewCopyFromWhereIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(copyFromWhereIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} + +var copyOnErrorIssue = issue.Issue{ + Type: COPY_ON_ERROR, + TypeName: "COPY ... ON_ERROR", + TypeDescription: "", + Suggestion: "", + GH: "", + DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#copy-on-error-is-not-yet-supported", +} + +func NewCopyOnErrorIssue(objectType string, objectName string, sqlStatement string) QueryIssue { + return newQueryIssue(copyOnErrorIssue, objectType, objectName, sqlStatement, map[string]interface{}{}) +} diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index 8b5f028b9..d601745de 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -375,6 +375,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewColumnRefDetector(query), NewXmlExprDetector(query), NewRangeTableFuncDetector(query), + NewCopyCommandUnsupportedConstructsDetector(), NewJsonConstructorFuncDetector(query), NewJsonQueryFunctionDetector(query), } diff --git a/yb-voyager/src/query/queryparser/traversal_proto.go b/yb-voyager/src/query/queryparser/traversal_proto.go index c988ee48e..e742a8172 100644 --- a/yb-voyager/src/query/queryparser/traversal_proto.go +++ b/yb-voyager/src/query/queryparser/traversal_proto.go @@ -49,6 +49,7 @@ const ( PG_QUERY_JSON_OBJECT_CONSTRUCTOR_NODE = "pg_query.JsonObjectConstructor" PG_QUERY_JSON_TABLE_NODE = "pg_query.JsonTable" PG_QUERY_VIEWSTMT_NODE = "pg_query.ViewStmt" + PG_QUERY_COPYSTSMT_NODE = "pg_query.CopyStmt" ) // function type for processing nodes during traversal From 038fd11abb6ee29ff64938279e890b747c8b3e3b Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Thu, 19 Dec 2024 12:46:36 +0000 Subject: [PATCH 02/12] Fixed errors after rebase --- yb-voyager/src/query/queryissue/constants.go | 2 + yb-voyager/src/query/queryissue/detectors.go | 41 ++++++++++++++----- .../src/query/queryissue/detectors_test.go | 33 +++++++++------ .../query/queryissue/parser_issue_detector.go | 2 +- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/yb-voyager/src/query/queryissue/constants.go b/yb-voyager/src/query/queryissue/constants.go index f0f86239a..8d319777e 100644 --- a/yb-voyager/src/query/queryissue/constants.go +++ b/yb-voyager/src/query/queryissue/constants.go @@ -70,6 +70,8 @@ const ( XML_FUNCTIONS_NAME = "XML Functions" REGEX_FUNCTIONS = "REGEX_FUNCTIONS" + COPY_FROM_WHERE = "COPY FROM ... WHERE" + COPY_ON_ERROR = "COPY ... ON_ERROR" ) // Object types diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index f8b944397..e681eb911 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -16,6 +16,8 @@ limitations under the License. package queryissue import ( + "fmt" + mapset "github.com/deckarep/golang-set/v2" log "github.com/sirupsen/logrus" "google.golang.org/protobuf/reflect/protoreflect" @@ -204,28 +206,35 @@ func (d *RangeTableFuncDetector) GetIssues() []QueryIssue { return issues } -type CopyCommandUnsupportedConstructsDetector struct{} +type CopyCommandUnsupportedConstructsDetector struct { + query string + copyFromWhereConstructDetected bool + copyOnErrorConstructDetected bool +} -func NewCopyCommandUnsupportedConstructsDetector() *CopyCommandUnsupportedConstructsDetector { - return &CopyCommandUnsupportedConstructsDetector{} +func NewCopyCommandUnsupportedConstructsDetector(query string) *CopyCommandUnsupportedConstructsDetector { + return &CopyCommandUnsupportedConstructsDetector{ + query: query, + } } // Detect if COPY command uses unsupported syntax i.e. COPY FROM ... WHERE and COPY... ON_ERROR -func (d *CopyCommandUnsupportedConstructsDetector) Detect(msg protoreflect.Message) ([]string, error) { - unsupportedConstructs := []string{} - +func (d *CopyCommandUnsupportedConstructsDetector) Detect(msg protoreflect.Message) error { // Check if the message is a COPY statement if msg.Descriptor().FullName() != queryparser.PG_QUERY_COPYSTSMT_NODE { - return unsupportedConstructs, nil // Not a COPY statement, nothing to detect + fmt.Println("Not a COPY statement") + return nil // Not a COPY statement, nothing to detect } + fmt.Println("Copy command detected: ", msg) + // Check for COPY FROM ... WHERE clause isFromField := msg.Descriptor().Fields().ByName("is_from") whereField := msg.Descriptor().Fields().ByName("where_clause") if isFromField != nil && msg.Has(isFromField) { isFrom := msg.Get(isFromField).Bool() if isFrom && whereField != nil && msg.Has(whereField) { - unsupportedConstructs = append(unsupportedConstructs, COPY_FROM_WHERE_NAME) + d.copyFromWhereConstructDetected = true } } @@ -244,14 +253,26 @@ func (d *CopyCommandUnsupportedConstructsDetector) Detect(msg protoreflect.Messa if defNameField != nil && defElem.Has(defNameField) { defName := defElem.Get(defNameField).String() if defName == "on_error" { - unsupportedConstructs = append(unsupportedConstructs, COPY_ON_ERROR_NAME) + d.copyOnErrorConstructDetected = true + break } } } } } - return unsupportedConstructs, nil + return nil +} + +func (d *CopyCommandUnsupportedConstructsDetector) GetIssues() []QueryIssue { + var issues []QueryIssue + if d.copyFromWhereConstructDetected { + issues = append(issues, NewCopyFromWhereIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } + if d.copyOnErrorConstructDetected { + issues = append(issues, NewCopyOnErrorIssue(DML_QUERY_OBJECT_TYPE, "", d.query)) + } + return issues } type JsonConstructorFuncDetector struct { diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index ce87f4fb8..30ea3b67f 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -37,40 +37,47 @@ func TestCopyCommandUnsupportedConstructsDetector(t *testing.T) { `COPY my_table(col1, col2) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true);`: {}, // COPY commands with WHERE clause - `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`: {COPY_FROM_WHERE_NAME}, - `COPY my_table(col1, col2) FROM '/path/to/data.csv' WHERE col2 = 'test';`: {COPY_FROM_WHERE_NAME}, - `COPY my_table FROM '/path/to/data.csv' WHERE TRUE;`: {COPY_FROM_WHERE_NAME}, + `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`: {COPY_FROM_WHERE}, + `COPY my_table(col1, col2) FROM '/path/to/data.csv' WHERE col2 = 'test';`: {COPY_FROM_WHERE}, + `COPY my_table FROM '/path/to/data.csv' WHERE TRUE;`: {COPY_FROM_WHERE}, // COPY commands with ON_ERROR clause - `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`: {COPY_ON_ERROR_NAME}, - `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP);`: {COPY_ON_ERROR_NAME}, + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`: {COPY_ON_ERROR}, + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP);`: {COPY_ON_ERROR}, // COPY commands with both ON_ERROR and WHERE clause - `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE) WHERE age > 18;`: {COPY_FROM_WHERE_NAME, COPY_ON_ERROR_NAME}, - `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP) WHERE name = 'Alice';`: {COPY_FROM_WHERE_NAME, COPY_ON_ERROR_NAME}, + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE) WHERE age > 18;`: {COPY_FROM_WHERE, COPY_ON_ERROR}, + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP) WHERE name = 'Alice';`: {COPY_FROM_WHERE, COPY_ON_ERROR}, } - detector := NewCopyCommandUnsupportedConstructsDetector() - for sql, expectedConstructs := range copyCommandSqlsMap { + detectConstructs := func(sql string) []QueryIssue { + detector := NewCopyCommandUnsupportedConstructsDetector(sql) parseResult, err := queryparser.Parse(sql) assert.NoError(t, err, "Failed to parse SQL: %s", sql) visited := make(map[protoreflect.Message]bool) - unsupportedConstructs := []string{} processor := func(msg protoreflect.Message) error { - constructs, err := detector.Detect(msg) + err := detector.Detect(msg) if err != nil { return err } - unsupportedConstructs = append(unsupportedConstructs, constructs...) return nil } parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) assert.NoError(t, err) - assert.ElementsMatch(t, expectedConstructs, unsupportedConstructs, "Unsupported Constructs not detected in SQL: %s", sql) + + return detector.GetIssues() + } + + for sql, expectedIssues := range copyCommandSqlsMap { + issues := detectConstructs(sql) + assert.Equal(t, len(expectedIssues), len(issues), "Expected %d issues for SQL: %s", len(expectedIssues), sql) + for i, issue := range issues { + assert.Equal(t, expectedIssues[i], issue.Type, "Expected issue %s for SQL: %s", expectedIssues[i], sql) + } } } diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index d601745de..29660b2d0 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -375,7 +375,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) NewColumnRefDetector(query), NewXmlExprDetector(query), NewRangeTableFuncDetector(query), - NewCopyCommandUnsupportedConstructsDetector(), + NewCopyCommandUnsupportedConstructsDetector(query), NewJsonConstructorFuncDetector(query), NewJsonQueryFunctionDetector(query), } From 1069f601c13dd8fb6c269f8b2a672a595d74c58d Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Fri, 20 Dec 2024 05:01:45 +0000 Subject: [PATCH 03/12] Fixed errors after rebase --- yb-voyager/src/query/queryissue/detectors.go | 5 ----- yb-voyager/src/query/queryissue/parser_issue_detector.go | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index e681eb911..862929595 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -16,8 +16,6 @@ limitations under the License. package queryissue import ( - "fmt" - mapset "github.com/deckarep/golang-set/v2" log "github.com/sirupsen/logrus" "google.golang.org/protobuf/reflect/protoreflect" @@ -222,12 +220,9 @@ func NewCopyCommandUnsupportedConstructsDetector(query string) *CopyCommandUnsup func (d *CopyCommandUnsupportedConstructsDetector) Detect(msg protoreflect.Message) error { // Check if the message is a COPY statement if msg.Descriptor().FullName() != queryparser.PG_QUERY_COPYSTSMT_NODE { - fmt.Println("Not a COPY statement") return nil // Not a COPY statement, nothing to detect } - fmt.Println("Copy command detected: ", msg) - // Check for COPY FROM ... WHERE clause isFromField := msg.Descriptor().Fields().ByName("is_from") whereField := msg.Descriptor().Fields().ByName("where_clause") diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector.go b/yb-voyager/src/query/queryissue/parser_issue_detector.go index 29660b2d0..9c32fca53 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector.go @@ -415,7 +415,7 @@ func (p *ParserIssueDetector) genericIssues(query string) ([]QueryIssue, error) xmlIssueAdded = true } } - result = append(result, issues...) + result = append(result, issue) } } From e3879535d56454120dba9695aa69bc1eb8fc7c04 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Fri, 20 Dec 2024 09:50:51 +0000 Subject: [PATCH 04/12] Added a few more unit tests --- yb-voyager/src/query/queryissue/issues_dml.go | 4 +- .../src/query/queryissue/issues_dml_test.go | 28 ++++++++++++ .../queryissue/parser_issue_detector_test.go | 44 ++++++++++++++++--- 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/yb-voyager/src/query/queryissue/issues_dml.go b/yb-voyager/src/query/queryissue/issues_dml.go index 2a0257f6a..e6915177c 100644 --- a/yb-voyager/src/query/queryissue/issues_dml.go +++ b/yb-voyager/src/query/queryissue/issues_dml.go @@ -148,7 +148,7 @@ var copyFromWhereIssue = issue.Issue{ TypeDescription: "", Suggestion: "", GH: "", - DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#copy-from-where-is-not-yet-supported", + DocsLink: "", } func NewCopyFromWhereIssue(objectType string, objectName string, sqlStatement string) QueryIssue { @@ -161,7 +161,7 @@ var copyOnErrorIssue = issue.Issue{ TypeDescription: "", Suggestion: "", GH: "", - DocsLink: "https://docs.yugabyte.com/preview/yugabyte-voyager/known-issues/postgresql/#copy-on-error-is-not-yet-supported", + DocsLink: "", } func NewCopyOnErrorIssue(objectType string, objectName string, sqlStatement string) QueryIssue { diff --git a/yb-voyager/src/query/queryissue/issues_dml_test.go b/yb-voyager/src/query/queryissue/issues_dml_test.go index 5b0cd0151..d24d4f194 100644 --- a/yb-voyager/src/query/queryissue/issues_dml_test.go +++ b/yb-voyager/src/query/queryissue/issues_dml_test.go @@ -58,6 +58,29 @@ func testRegexFunctionsIssue(t *testing.T) { } } +func testCopyOnErrorIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + // In case the COPY ... ON_ERROR construct gets supported in the future, this test will fail with a different error message-something related to the data.csv file not being found. + _, err = conn.Exec(ctx, `COPY pg_largeobject (loid, pageno, data) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ERROR: option \"on_error\" not recognized (SQLSTATE 42601)", copyOnErrorIssue) + +} + +func testCopyFromWhereIssue(t *testing.T) { + ctx := context.Background() + conn, err := getConn() + assert.NoError(t, err) + + defer conn.Close(context.Background()) + // In case the COPY FROM ... WHERE construct gets supported in the future, this test will fail with a different error message-something related to the data.csv file not being found. + _, err = conn.Exec(ctx, `COPY pg_largeobject (loid, pageno, data) FROM '/path/to/data.csv' WHERE loid = 1 WITH (FORMAT csv, HEADER true);`) + assertErrorCorrectlyThrownForIssueForYBVersion(t, err, "ERROR: syntax error at or near \"WHERE\" (SQLSTATE 42601)", copyFromWhereIssue) +} + func testJsonConstructorFunctions(t *testing.T) { ctx := context.Background() conn, err := getConn() @@ -145,6 +168,11 @@ func TestDMLIssuesInYBVersion(t *testing.T) { success = t.Run(fmt.Sprintf("%s-%s", "regex functions", ybVersion), testRegexFunctionsIssue) assert.True(t, success) + success = t.Run(fmt.Sprintf("%s-%s", "copy on error", ybVersion), testCopyOnErrorIssue) + assert.True(t, success) + + success = t.Run(fmt.Sprintf("%s-%s", "copy from where", ybVersion), testCopyFromWhereIssue) + assert.True(t, success) success = t.Run(fmt.Sprintf("%s-%s", "json constructor functions", ybVersion), testJsonConstructorFunctions) assert.True(t, success) diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 1450f4edd..41cd6ed14 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -449,11 +449,11 @@ $$ LANGUAGE plpgsql; NewLOFuntionsIssue("TRIGGER", "t_raster ON image", sqls[5], []string{"lo_manage"}), }, } - expectedSQLsWithIssues[sqls[0]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[0]], "FUNCTION", "manage_large_object") - expectedSQLsWithIssues[sqls[1]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[1]], "FUNCTION", "import_file_to_table") - expectedSQLsWithIssues[sqls[2]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[2]], "FUNCTION", "export_large_object") - expectedSQLsWithIssues[sqls[3]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[3]], "PROCEDURE", "read_large_object") - expectedSQLsWithIssues[sqls[4]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[4]], "FUNCTION", "write_to_large_object") + expectedSQLsWithIssues[sqls[0]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[0]], "FUNCTION", "manage_large_object") + expectedSQLsWithIssues[sqls[1]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[1]], "FUNCTION", "import_file_to_table") + expectedSQLsWithIssues[sqls[2]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[2]], "FUNCTION", "export_large_object") + expectedSQLsWithIssues[sqls[3]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[3]], "PROCEDURE", "read_large_object") + expectedSQLsWithIssues[sqls[4]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[4]], "FUNCTION", "write_to_large_object") parserIssueDetector := NewParserIssueDetector() @@ -649,3 +649,37 @@ func TestRegexFunctionsIssue(t *testing.T) { } } + +func TestCopyUnsupportedConstructIssuesDetected(t *testing.T) { + expectedIssues := map[string][]QueryIssue{ + `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`)}, + `COPY my_table(col1, col2) FROM '/path/to/data.csv' WHERE col2 = 'test';`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table(col1, col2) FROM '/path/to/data.csv' WHERE col2 = 'test';`)}, + `COPY my_table FROM '/path/to/data.csv' WHERE TRUE;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table FROM '/path/to/data.csv' WHERE TRUE;`)}, + + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`: {NewCopyOnErrorIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`)}, + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP);`: {NewCopyOnErrorIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP);`)}, + + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE) WHERE age > 18;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE) WHERE age > 18;`), NewCopyOnErrorIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE) WHERE age > 18;`)}, + `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP) WHERE name = 'Alice';`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP) WHERE name = 'Alice';`), NewCopyOnErrorIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP) WHERE name = 'Alice';`)}, + + `COPY my_table FROM '/path/to/data.csv' WITH (FORMAT csv);`: {}, + `COPY my_table FROM '/path/to/data.csv' WITH (FORMAT text);`: {}, + `COPY my_table FROM '/path/to/data.csv';`: {}, + `COPY my_table FROM '/path/to/data.csv' WITH (DELIMITER ',');`: {}, + `COPY my_table(col1, col2) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true);`: {}, + } + + parserIssueDetector := NewParserIssueDetector() + + for stmt, expectedIssues := range expectedIssues { + issues, err := parserIssueDetector.getDMLIssues(stmt) + fatalIfError(t, err) + assert.Equal(t, len(expectedIssues), len(issues)) + for _, expectedIssue := range expectedIssues { + found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { + return cmp.Equal(expectedIssue, queryIssue) + }) + assert.True(t, found) + } + } +} From f70de4d404fae60877d496b9ae21f8efb0b4b0d6 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Tue, 24 Dec 2024 11:25:23 +0000 Subject: [PATCH 05/12] Added the suggested changes --- yb-voyager/src/query/queryissue/detectors.go | 39 ++++++------------- .../src/query/queryparser/helpers_protomsg.go | 10 ++++- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/yb-voyager/src/query/queryissue/detectors.go b/yb-voyager/src/query/queryissue/detectors.go index 862929595..e6ec4c4cc 100644 --- a/yb-voyager/src/query/queryissue/detectors.go +++ b/yb-voyager/src/query/queryissue/detectors.go @@ -16,6 +16,8 @@ limitations under the License. package queryissue import ( + "slices" + mapset "github.com/deckarep/golang-set/v2" log "github.com/sirupsen/logrus" "google.golang.org/protobuf/reflect/protoreflect" @@ -224,36 +226,19 @@ func (d *CopyCommandUnsupportedConstructsDetector) Detect(msg protoreflect.Messa } // Check for COPY FROM ... WHERE clause - isFromField := msg.Descriptor().Fields().ByName("is_from") - whereField := msg.Descriptor().Fields().ByName("where_clause") - if isFromField != nil && msg.Has(isFromField) { - isFrom := msg.Get(isFromField).Bool() - if isFrom && whereField != nil && msg.Has(whereField) { - d.copyFromWhereConstructDetected = true - } + fromField := queryparser.GetBoolField(msg, "is_from") + whereField := queryparser.GetMessageField(msg, "where_clause") + if fromField && whereField != nil { + d.copyFromWhereConstructDetected = true } // Check for COPY ... ON_ERROR clause - optionsField := msg.Descriptor().Fields().ByName("options") - if optionsField != nil && msg.Has(optionsField) { - optionsList := msg.Get(optionsField).List() - for i := 0; i < optionsList.Len(); i++ { - option := optionsList.Get(i).Message() - - // Check for nested def_elem field - defElemField := option.Descriptor().Fields().ByName("def_elem") - if defElemField != nil && option.Has(defElemField) { - defElem := option.Get(defElemField).Message() - defNameField := defElem.Descriptor().Fields().ByName("defname") - if defNameField != nil && defElem.Has(defNameField) { - defName := defElem.Get(defNameField).String() - if defName == "on_error" { - d.copyOnErrorConstructDetected = true - break - } - } - } - } + defNames, err := queryparser.TraverseAndExtractDefNamesFromDefElem(msg) + if err != nil { + log.Errorf("error extracting defnames from COPY statement: %v", err) + } + if slices.Contains(defNames, "on_error") { + d.copyOnErrorConstructDetected = true } return nil diff --git a/yb-voyager/src/query/queryparser/helpers_protomsg.go b/yb-voyager/src/query/queryparser/helpers_protomsg.go index e0de00604..8fe62bc82 100644 --- a/yb-voyager/src/query/queryparser/helpers_protomsg.go +++ b/yb-voyager/src/query/queryparser/helpers_protomsg.go @@ -366,6 +366,14 @@ func GetMessageField(msg protoreflect.Message, fieldName string) protoreflect.Me return nil } +func GetBoolField(msg protoreflect.Message, fieldName string) bool { + field := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) + if field != nil && msg.Has(field) { + return msg.Get(field).Bool() + } + return false +} + // GetListField retrieves a list field from a message. func GetListField(msg protoreflect.Message, fieldName string) protoreflect.List { field := msg.Descriptor().Fields().ByName(protoreflect.Name(fieldName)) @@ -403,12 +411,10 @@ func GetSchemaAndObjectName(nameList protoreflect.List) (string, string) { Example: options:{def_elem:{defname:"security_invoker" arg:{string:{sval:"true"}} defaction:DEFELEM_UNSPEC location:32}} options:{def_elem:{defname:"security_barrier" arg:{string:{sval:"false"}} defaction:DEFELEM_UNSPEC location:57}} - Extract all defnames from the def_eleme node */ func TraverseAndExtractDefNamesFromDefElem(msg protoreflect.Message) ([]string, error) { var defNames []string - collectorFunc := func(msg protoreflect.Message) error { if GetMsgFullName(msg) != PG_QUERY_DEFELEM_NODE { return nil From be6ca71fcdd539f4d7152b0ee79985a8850cb11c Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Tue, 24 Dec 2024 12:14:28 +0000 Subject: [PATCH 06/12] Added COPY FROM ... WHERE to end to end assessment test, COPY ... ON_ERROR is still commented --- migtests/scripts/postgresql/env.sh | 6 ++-- .../expectedAssessmentReport.json | 24 +++++++++++++-- .../unsupported_query_constructs.sql | 30 ++++++++++++++++++- .../src/query/queryissue/detectors_test.go | 3 ++ .../queryissue/parser_issue_detector_test.go | 7 ++++- 5 files changed, 63 insertions(+), 7 deletions(-) diff --git a/migtests/scripts/postgresql/env.sh b/migtests/scripts/postgresql/env.sh index ec1f12358..4068315a4 100644 --- a/migtests/scripts/postgresql/env.sh +++ b/migtests/scripts/postgresql/env.sh @@ -1,6 +1,6 @@ export SOURCE_DB_HOST=${SOURCE_DB_HOST:-"127.0.0.1"} export SOURCE_DB_PORT=${SOURCE_DB_PORT:-5432} -export SOURCE_DB_USER=${SOURCE_DB_USER:-"ybvoyager"} -export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'Test@123#$%^&*()!'} +export SOURCE_DB_USER=${SOURCE_DB_USER:-"postgres"} +export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'postgres'} export SOURCE_DB_ADMIN_USER=${SOURCE_DB_ADMIN_USER:-"postgres"} -export SOURCE_DB_ADMIN_PASSWORD=${SOURCE_DB_ADMIN_PASSWORD:-"secret"} +export SOURCE_DB_ADMIN_PASSWORD=${SOURCE_DB_ADMIN_PASSWORD:-"postgres"} diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 1057ad9fd..8c1938792 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -40,7 +40,7 @@ "ObjectType": "SEQUENCE", "TotalCount": 28, "InvalidCount": 0, - "ObjectNames": "public.ordersentry_order_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees_employee_id_seq, public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" + "ObjectNames": "public.employees_id_seq, public.ordersentry_order_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" }, { "ObjectType": "TABLE", @@ -1983,6 +1983,20 @@ "ObjectType": "", "ParentTableName": "schema2.mixed_data_types_table1", "SizeInBytes": 8192 + }, + { + "SchemaName": "public", + "ObjectName": "employees", + "RowCount": 2, + "ColumnCount": 3, + "Reads": 0, + "Writes": 2, + "ReadsPerSecond": 0, + "WritesPerSecond": 0, + "IsIndex": false, + "ObjectType": "", + "ParentTableName": null, + "SizeInBytes": 8192 } ], "Notes": [ @@ -2182,6 +2196,12 @@ "Query": "SELECT lo_create($1)", "DocsLink": "", "MinimumVersionsFixedIn": null + }, + { + "ConstructTypeName": "COPY FROM ... WHERE", + "Query": "COPY employees (id, name, age)\nFROM STDIN WITH (FORMAT csv)\nWHERE age \u003e 30", + "DocsLink": "", + "MinimumVersionsFixedIn": null } ], "UnsupportedPlPgSqlObjects": [ @@ -2231,4 +2251,4 @@ "MinimumVersionsFixedIn": null } ] -} \ No newline at end of file +} diff --git a/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql index cc7251039..c14492920 100644 --- a/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql +++ b/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql @@ -138,4 +138,32 @@ FROM ) AS items; -SELECT lo_create('32142'); \ No newline at end of file +SELECT lo_create('32142'); + +-- Unsupported COPY constructs + +CREATE TABLE IF NOT EXISTS employees ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + age INT NOT NULL +); + + +-- COPY FROM with WHERE clause +COPY employees (id, name, age) +FROM STDIN WITH (FORMAT csv) +WHERE age > 30; +1,John Smith,25 +2,Jane Doe,34 +3,Bob Johnson,31 +\. + +-- This can be uncommented when we start using PG 17 or later in the tests +-- -- COPY with ON_ERROR clause +-- COPY employees (id, name, age) +-- FROM STDIN WITH (FORMAT csv, ON_ERROR IGNORE ); +-- 4,Adam Smith,22 +-- 5,John Doe,34 +-- 6,Ron Johnson,31 +-- \. + diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index 30ea3b67f..502babf06 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -40,6 +40,9 @@ func TestCopyCommandUnsupportedConstructsDetector(t *testing.T) { `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`: {COPY_FROM_WHERE}, `COPY my_table(col1, col2) FROM '/path/to/data.csv' WHERE col2 = 'test';`: {COPY_FROM_WHERE}, `COPY my_table FROM '/path/to/data.csv' WHERE TRUE;`: {COPY_FROM_WHERE}, + `COPY employees (id, name, age) + FROM STDIN WITH (FORMAT csv) + WHERE age > 30;`: {COPY_FROM_WHERE}, // COPY commands with ON_ERROR clause `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`: {COPY_ON_ERROR}, diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index 41cd6ed14..aab82be6d 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -655,6 +655,11 @@ func TestCopyUnsupportedConstructIssuesDetected(t *testing.T) { `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`)}, `COPY my_table(col1, col2) FROM '/path/to/data.csv' WHERE col2 = 'test';`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table(col1, col2) FROM '/path/to/data.csv' WHERE col2 = 'test';`)}, `COPY my_table FROM '/path/to/data.csv' WHERE TRUE;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY my_table FROM '/path/to/data.csv' WHERE TRUE;`)}, + `COPY employees (id, name, age) + FROM STDIN WITH (FORMAT csv) + WHERE age > 30;`: {NewCopyFromWhereIssue("DML_QUERY", "", `COPY employees (id, name, age) + FROM STDIN WITH (FORMAT csv) + WHERE age > 30;`)}, `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`: {NewCopyOnErrorIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`)}, `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP);`: {NewCopyOnErrorIssue("DML_QUERY", "", `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP);`)}, @@ -679,7 +684,7 @@ func TestCopyUnsupportedConstructIssuesDetected(t *testing.T) { found := slices.ContainsFunc(issues, func(queryIssue QueryIssue) bool { return cmp.Equal(expectedIssue, queryIssue) }) - assert.True(t, found) + assert.True(t, found, "Expected issue not found: %v in statement: %s", expectedIssue, stmt) } } } From e687c7d7cfa720f5857bf1bfaa2b53e5dae98e2a Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Tue, 24 Dec 2024 12:44:10 +0000 Subject: [PATCH 07/12] Fixed issues after rebase --- .../expectedAssessmentReport.json | 15 ++++++++------- .../unsupported_query_constructs.sql | 4 ++-- .../queryissue/parser_issue_detector_test.go | 10 +++++----- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 8c1938792..1c93d7c69 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -40,13 +40,13 @@ "ObjectType": "SEQUENCE", "TotalCount": 28, "InvalidCount": 0, - "ObjectNames": "public.employees_id_seq, public.ordersentry_order_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" + "ObjectNames": "public.employees_id_seq, public.ordersentry_order_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq, public.employees_employee_id_seq" }, { "ObjectType": "TABLE", - "TotalCount": 67, + "TotalCount": 68, "InvalidCount": 23, - "ObjectNames": "public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2, public.employees" + "ObjectNames": "public.employees3, public.ordersentry, public.library_nested, public.orders_lateral, public.\"Case_Sensitive_Columns\", public.\"Mixed_Case_Table_Name_Test\", public.\"Recipients\", public.\"WITH\", public.audit, public.sales_region, public.boston, public.c, public.parent_table, public.child_table, public.citext_type, public.combined_tbl, public.documents, public.employees2, public.ext_test, public.foo, public.inet_type, public.london, public.mixed_data_types_table1, public.mixed_data_types_table2, public.orders, public.orders2, public.products, public.session_log, public.session_log1, public.session_log2, public.sydney, public.test_exclude_basic, public.test_jsonb, public.test_xml_type, public.ts_query_table, public.tt, public.with_example1, public.with_example2, schema2.\"Case_Sensitive_Columns\", schema2.\"Mixed_Case_Table_Name_Test\", schema2.\"Recipients\", schema2.\"WITH\", schema2.audit, schema2.sales_region, schema2.boston, schema2.c, schema2.parent_table, schema2.child_table, schema2.employees2, schema2.ext_test, schema2.foo, schema2.london, schema2.mixed_data_types_table1, schema2.mixed_data_types_table2, schema2.orders, schema2.orders2, schema2.products, schema2.session_log, schema2.session_log1, schema2.session_log2, schema2.sydney, schema2.test_xml_type, schema2.tt, schema2.with_example1, schema2.with_example2, test_views.view_table1, test_views.view_table2, public.employees" }, { "ObjectType": "INDEX", @@ -170,9 +170,10 @@ "test_views.view_table1", "public.library_nested", "public.orders_lateral", - "public.employees" + "public.employees", + "public.employees3" ], - "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 73 objects (65 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", + "ColocatedReasoning": "Recommended instance type with 4 vCPU and 16 GiB memory could fit 74 objects (66 tables/materialized views and 8 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec as colocated. Rest 28 objects (5 tables/materialized views and 23 explicit/implicit indexes) with 0.00 MB size and throughput requirement of 0 reads/sec and 0 writes/sec need to be migrated as range partitioned tables. Non leaf partition tables/indexes and unsupported tables/indexes were not considered.", "ShardedTables": [ "public.combined_tbl", "public.citext_type", @@ -1986,7 +1987,7 @@ }, { "SchemaName": "public", - "ObjectName": "employees", + "ObjectName": "employees3", "RowCount": 2, "ColumnCount": 3, "Reads": 0, @@ -2199,7 +2200,7 @@ }, { "ConstructTypeName": "COPY FROM ... WHERE", - "Query": "COPY employees (id, name, age)\nFROM STDIN WITH (FORMAT csv)\nWHERE age \u003e 30", + "Query": "COPY employees3 (id, name, age)\nFROM STDIN WITH (FORMAT csv)\nWHERE age \u003e 30", "DocsLink": "", "MinimumVersionsFixedIn": null } diff --git a/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql b/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql index c14492920..284c8f7d0 100644 --- a/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql +++ b/migtests/tests/pg/assessment-report-test/unsupported_query_constructs.sql @@ -142,7 +142,7 @@ SELECT lo_create('32142'); -- Unsupported COPY constructs -CREATE TABLE IF NOT EXISTS employees ( +CREATE TABLE IF NOT EXISTS employees3 ( id SERIAL PRIMARY KEY, name TEXT NOT NULL, age INT NOT NULL @@ -150,7 +150,7 @@ CREATE TABLE IF NOT EXISTS employees ( -- COPY FROM with WHERE clause -COPY employees (id, name, age) +COPY employees3 (id, name, age) FROM STDIN WITH (FORMAT csv) WHERE age > 30; 1,John Smith,25 diff --git a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go index aab82be6d..0d6e9d23f 100644 --- a/yb-voyager/src/query/queryissue/parser_issue_detector_test.go +++ b/yb-voyager/src/query/queryissue/parser_issue_detector_test.go @@ -449,11 +449,11 @@ $$ LANGUAGE plpgsql; NewLOFuntionsIssue("TRIGGER", "t_raster ON image", sqls[5], []string{"lo_manage"}), }, } - expectedSQLsWithIssues[sqls[0]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[0]], "FUNCTION", "manage_large_object") - expectedSQLsWithIssues[sqls[1]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[1]], "FUNCTION", "import_file_to_table") - expectedSQLsWithIssues[sqls[2]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[2]], "FUNCTION", "export_large_object") - expectedSQLsWithIssues[sqls[3]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[3]], "PROCEDURE", "read_large_object") - expectedSQLsWithIssues[sqls[4]] = modifyiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[4]], "FUNCTION", "write_to_large_object") + expectedSQLsWithIssues[sqls[0]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[0]], "FUNCTION", "manage_large_object") + expectedSQLsWithIssues[sqls[1]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[1]], "FUNCTION", "import_file_to_table") + expectedSQLsWithIssues[sqls[2]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[2]], "FUNCTION", "export_large_object") + expectedSQLsWithIssues[sqls[3]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[3]], "PROCEDURE", "read_large_object") + expectedSQLsWithIssues[sqls[4]] = modifiedIssuesforPLPGSQL(expectedSQLsWithIssues[sqls[4]], "FUNCTION", "write_to_large_object") parserIssueDetector := NewParserIssueDetector() From a417d932e7783a289b725cac597d6553a0b29ece Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Tue, 24 Dec 2024 12:54:07 +0000 Subject: [PATCH 08/12] Fixed issues after rebase --- migtests/scripts/postgresql/env.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/migtests/scripts/postgresql/env.sh b/migtests/scripts/postgresql/env.sh index 4068315a4..ec1f12358 100644 --- a/migtests/scripts/postgresql/env.sh +++ b/migtests/scripts/postgresql/env.sh @@ -1,6 +1,6 @@ export SOURCE_DB_HOST=${SOURCE_DB_HOST:-"127.0.0.1"} export SOURCE_DB_PORT=${SOURCE_DB_PORT:-5432} -export SOURCE_DB_USER=${SOURCE_DB_USER:-"postgres"} -export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'postgres'} +export SOURCE_DB_USER=${SOURCE_DB_USER:-"ybvoyager"} +export SOURCE_DB_PASSWORD=${SOURCE_DB_PASSWORD:-'Test@123#$%^&*()!'} export SOURCE_DB_ADMIN_USER=${SOURCE_DB_ADMIN_USER:-"postgres"} -export SOURCE_DB_ADMIN_PASSWORD=${SOURCE_DB_ADMIN_PASSWORD:-"postgres"} +export SOURCE_DB_ADMIN_PASSWORD=${SOURCE_DB_ADMIN_PASSWORD:-"secret"} From 3a6915f3341d3008c0d041bfda1f4d0fbab300e5 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Tue, 24 Dec 2024 13:12:18 +0000 Subject: [PATCH 09/12] Fixed issues after rebase --- .../pg/assessment-report-test/expectedAssessmentReport.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 1c93d7c69..2122d5d99 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -38,9 +38,9 @@ }, { "ObjectType": "SEQUENCE", - "TotalCount": 28, + "TotalCount": 29, "InvalidCount": 0, - "ObjectNames": "public.employees_id_seq, public.ordersentry_order_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq, public.employees_employee_id_seq" + "ObjectNames": "public.employees3_id_seq, public.employees_id_seq, public.ordersentry_order_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq, public.employees_employee_id_seq" }, { "ObjectType": "TABLE", From 5eace0834b4924576dd07e03973a1c124840e401 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Tue, 24 Dec 2024 14:06:11 +0000 Subject: [PATCH 10/12] Fixes after rebase --- .../pg/assessment-report-test/expectedAssessmentReport.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index 2122d5d99..c907107ca 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -40,7 +40,7 @@ "ObjectType": "SEQUENCE", "TotalCount": 29, "InvalidCount": 0, - "ObjectNames": "public.employees3_id_seq, public.employees_id_seq, public.ordersentry_order_id_seq, public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq, public.employees_employee_id_seq" + "ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.employees_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" }, { "ObjectType": "TABLE", From 06ab5858c56922f2f0bc84cddfaff5c7afd147b5 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Thu, 26 Dec 2024 08:50:58 +0000 Subject: [PATCH 11/12] Fixing tests --- .../pg/assessment-report-test/expectedAssessmentReport.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json index c907107ca..5a861d641 100644 --- a/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json +++ b/migtests/tests/pg/assessment-report-test/expectedAssessmentReport.json @@ -40,7 +40,7 @@ "ObjectType": "SEQUENCE", "TotalCount": 29, "InvalidCount": 0, - "ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.employees_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" + "ObjectNames": "public.\"Case_Sensitive_Columns_id_seq\", public.\"Mixed_Case_Table_Name_Test_id_seq\", public.\"Recipients_id_seq\", public.\"WITH_id_seq\", public.employees2_id_seq, public.employees3_id_seq, public.employees_employee_id_seq, public.ext_test_id_seq, public.mixed_data_types_table1_id_seq, public.mixed_data_types_table2_id_seq, public.orders2_id_seq, public.ordersentry_order_id_seq, public.parent_table_id_seq, public.with_example1_id_seq, public.with_example2_id_seq, schema2.\"Case_Sensitive_Columns_id_seq\", schema2.\"Mixed_Case_Table_Name_Test_id_seq\", schema2.\"Recipients_id_seq\", schema2.\"WITH_id_seq\", schema2.employees2_id_seq, schema2.ext_test_id_seq, schema2.mixed_data_types_table1_id_seq, schema2.mixed_data_types_table2_id_seq, schema2.orders2_id_seq, schema2.parent_table_id_seq, schema2.with_example1_id_seq, schema2.with_example2_id_seq, test_views.view_table1_id_seq, test_views.view_table2_id_seq" }, { "ObjectType": "TABLE", From f22d7c686e6b1b8b59a350b5a83d80ac6845f750 Mon Sep 17 00:00:00 2001 From: ShivanshGahlot Date: Thu, 26 Dec 2024 10:00:32 +0000 Subject: [PATCH 12/12] Made suggested changes --- .../src/query/queryissue/detectors_test.go | 57 ------------------- 1 file changed, 57 deletions(-) diff --git a/yb-voyager/src/query/queryissue/detectors_test.go b/yb-voyager/src/query/queryissue/detectors_test.go index 502babf06..266f053ba 100644 --- a/yb-voyager/src/query/queryissue/detectors_test.go +++ b/yb-voyager/src/query/queryissue/detectors_test.go @@ -27,63 +27,6 @@ import ( "github.com/yugabyte/yb-voyager/yb-voyager/src/query/queryparser" ) -func TestCopyCommandUnsupportedConstructsDetector(t *testing.T) { - copyCommandSqlsMap := map[string][]string{ - // Valid COPY commands without WHERE or ON_ERROR - `COPY my_table FROM '/path/to/data.csv' WITH (FORMAT csv);`: {}, - `COPY my_table FROM '/path/to/data.csv' WITH (FORMAT text);`: {}, - `COPY my_table FROM '/path/to/data.csv';`: {}, - `COPY my_table FROM '/path/to/data.csv' WITH (DELIMITER ',');`: {}, - `COPY my_table(col1, col2) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true);`: {}, - - // COPY commands with WHERE clause - `COPY my_table FROM '/path/to/data.csv' WHERE col1 > 100;`: {COPY_FROM_WHERE}, - `COPY my_table(col1, col2) FROM '/path/to/data.csv' WHERE col2 = 'test';`: {COPY_FROM_WHERE}, - `COPY my_table FROM '/path/to/data.csv' WHERE TRUE;`: {COPY_FROM_WHERE}, - `COPY employees (id, name, age) - FROM STDIN WITH (FORMAT csv) - WHERE age > 30;`: {COPY_FROM_WHERE}, - - // COPY commands with ON_ERROR clause - `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE);`: {COPY_ON_ERROR}, - `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP);`: {COPY_ON_ERROR}, - - // COPY commands with both ON_ERROR and WHERE clause - `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR IGNORE) WHERE age > 18;`: {COPY_FROM_WHERE, COPY_ON_ERROR}, - `COPY table_name (name, age) FROM '/path/to/data.csv' WITH (FORMAT csv, HEADER true, ON_ERROR STOP) WHERE name = 'Alice';`: {COPY_FROM_WHERE, COPY_ON_ERROR}, - } - - detectConstructs := func(sql string) []QueryIssue { - detector := NewCopyCommandUnsupportedConstructsDetector(sql) - parseResult, err := queryparser.Parse(sql) - assert.NoError(t, err, "Failed to parse SQL: %s", sql) - - visited := make(map[protoreflect.Message]bool) - - processor := func(msg protoreflect.Message) error { - err := detector.Detect(msg) - if err != nil { - return err - } - return nil - } - - parseTreeMsg := queryparser.GetProtoMessageFromParseTree(parseResult) - err = queryparser.TraverseParseTree(parseTreeMsg, visited, processor) - assert.NoError(t, err) - - return detector.GetIssues() - } - - for sql, expectedIssues := range copyCommandSqlsMap { - issues := detectConstructs(sql) - assert.Equal(t, len(expectedIssues), len(issues), "Expected %d issues for SQL: %s", len(expectedIssues), sql) - for i, issue := range issues { - assert.Equal(t, expectedIssues[i], issue.Type, "Expected issue %s for SQL: %s", expectedIssues[i], sql) - } - } -} - func getDetectorIssues(t *testing.T, detector UnsupportedConstructDetector, sql string) []QueryIssue { parseResult, err := queryparser.Parse(sql) assert.NoError(t, err, "Failed to parse SQL: %s", sql)