diff --git a/src/e2e-test/features/bigquery/source/BigQuerySqlEngine.feature b/src/e2e-test/features/bigquery/source/BigQuerySqlEngine.feature index 02179b417..6da51891a 100644 --- a/src/e2e-test/features/bigquery/source/BigQuerySqlEngine.feature +++ b/src/e2e-test/features/bigquery/source/BigQuerySqlEngine.feature @@ -15,14 +15,13 @@ @BigQuery_Sink Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data transfer - @BQ_SOURCE_SQLENGINE_TEST @BQ_SINK_TEST @EXISTING_BQ_CONNECTION - Scenario:Validate successful records transfer from BigQuery source to BigQuery sink + @BQ_SOURCE_JOINER_TEST @BQ_SOURCE_JOINER2_TEST @BQ_DELETE_JOIN @BQ_SINK_TEST @EXISTING_BQ_CONNECTION + Scenario:Validate successful records transfer from BigQuery source to BigQuery sink using Join Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Source" When Select plugin: "BigQuery" from the plugins list as: "Source" - When Expand Plugin group in the LHS plugins list: "Sink" - When Select plugin: "BigQuery" from the plugins list as: "Sink" - Then Connect plugins: "BigQuery" and "BigQuery2" to establish connection + When Expand Plugin group in the LHS plugins list: "Analytics" + When Select plugin: "Joiner" from the plugins list as: "Analytics" Then Navigate to the properties page of plugin: "BigQuery" Then Click plugin property: "switch-useConnection" Then Click on the Browse Connections button @@ -33,15 +32,36 @@ Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data tr Then Click on the Get Schema button Then Validate "BigQuery" plugin properties And Close the Plugin Properties page + When Expand Plugin group in the LHS plugins list: "Source" + When Select plugin: "BigQuery" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "BigQuery" from the plugins list as: "Sink" + Then Connect plugins: "BigQuery" and "Joiner" to establish connection + Then Connect plugins: "BigQuery2" and "Joiner" to establish connection + Then Connect plugins: "Joiner" and "BigQuery3" to establish connection Then Navigate to the properties page of plugin: "BigQuery2" Then Click plugin property: "useConnection" Then Click on the Browse Connections button Then Select connection: "bqConnectionName" + Then Enter input plugin property: "referenceName" with value: "BQRefName" + Then Enter input plugin property: "dataset" with value: "dataset" + Then Enter input plugin property: "table" with value: "bqSourceTable2" + Then Validate "BigQuery2" plugin properties + And Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Joiner" + Then Select radio button plugin property: "conditionType" with value: "basic" + Then Click on the Get Schema button + Then Validate "Joiner" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "BigQuery3" + Then Click plugin property: "useConnection" + Then Click on the Browse Connections button + Then Select connection: "bqConnectionName" Then Enter input plugin property: "referenceName" with value: "BQSinkReferenceName" Then Enter input plugin property: "dataset" with value: "dataset" Then Enter input plugin property: "table" with value: "bqTargetTable" - Then Validate "BigQuery" plugin properties - And Close the Plugin Properties page + Then Validate "BigQuery3" plugin properties + Then Close the Plugin Properties page Then Save the pipeline Then Preview and run the pipeline Then Wait till pipeline preview is in running state diff --git a/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java index 7182d33a5..76a32d055 100644 --- a/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java +++ b/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java @@ -1368,4 +1368,44 @@ public static void createSourceBQTableForSqlEngine() throws IOException, Interru PluginPropertyUtils.pluginProp("bqInsertDataQueryFileSQL")); } + @Before(order = 1, value = "@BQ_SOURCE_JOINER_TEST") + public static void createSourceBQTableForJoiner() throws IOException, InterruptedException { + createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("bqCreateTableQueryFileJoin"), + PluginPropertyUtils.pluginProp("bqInsertDataQueryFileJoin")); + } + + @Before(order = 1, value = "@BQ_SOURCE_JOINER2_TEST") + public static void createsecondBQTableJoin() throws IOException, InterruptedException { + bqSourceTable2 = "E2E_SOURCE_" + UUID.randomUUID().toString().replaceAll("-", "_"); + BigQueryClient.getSoleQueryResult("create table `" + datasetName + "." + bqSourceTable2 + "` " + + "(customerid INT64, item STRING, price FLOAT64 )"); + try { + io.cdap.e2e.utils.BigQueryClient.getSoleQueryResult("INSERT INTO `" + datasetName + "." + bqSourceTable2 + "` " + + "(customerid, item, price)" + + "VALUES (1, 'donut', 0.8), " + + "(1, 'coffee', 2.05), " + + "(2, 'donut', 1.5), " + + "(2, 'plate', 0.5), " + + "(3, 'tea', 1.99), " + + "(5, 'cookie', 0.5)"); + } catch (NoSuchElementException e) { + // Insert query does not return any record. + // Iterator on TableResult values in getSoleQueryResult method throws NoSuchElementException + BeforeActions.scenario.write("Error inserting the record in the table" + e.getStackTrace()); + } + PluginPropertyUtils.addPluginProp("bqSourceTable2", bqSourceTable2); + BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " created successfully"); + } + + @After(order = 1, value = "@BQ_DELETE_JOIN") + public static void deleteBQTablesJoin() throws IOException, InterruptedException { + BigQueryClient.dropBqQuery(bqSourceTable); + BigQueryClient.dropBqQuery(bqSourceTable2); + PluginPropertyUtils.removePluginProp("bqSourceTable"); + PluginPropertyUtils.removePluginProp("bqSourceTable2"); + BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully"); + BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully"); + bqSourceTable = StringUtils.EMPTY; + bqSourceTable2 = StringUtils.EMPTY; + } } diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index e9372f919..0f21503cd 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -346,4 +346,6 @@ groupByGcsAggregateFields=[{"key":"name#Count","value":"namecount"}] deduplicateTestOutputFile=testdata/BQValidationExpectedFiles/deduplicate_Test groupByTestOutputFile=testdata/BQValidationExpectedFiles/groupby_Test bqExpectedFile=testdata/BQValidationExpectedFiles/bqexpected_Test +bqCreateTableQueryFileJoin=testdata/BigQuery/BigQueryCreateTableJoin.txt +bqInsertDataQueryFileJoin=testdata/BigQuery/BigQueryInsertDataJoin.txt ## SQLENGINE-PLUGIN-PROPERTIES-END diff --git a/src/e2e-test/resources/testdata/BQValidationExpectedFiles/bqexpected_Test b/src/e2e-test/resources/testdata/BQValidationExpectedFiles/bqexpected_Test index d87c27872..e9fb6148b 100644 --- a/src/e2e-test/resources/testdata/BQValidationExpectedFiles/bqexpected_Test +++ b/src/e2e-test/resources/testdata/BQValidationExpectedFiles/bqexpected_Test @@ -1,3 +1,8 @@ -{"address":"ppu","id":13,"name":"root"} -{"address":"ggn","id":12,"name":"joe"} -{"address":"ggn","id":12,"name":"joe"} \ No newline at end of file +{"city":"San Jose","customerid":1,"firstname":"Douglas","id":1,"item":"coffee","lastname":"Williams","phoneNumber":35834612345,"price":2.05,"state":"CA","streetAddress":"1 Vista Montana","zipcode":9513498885} +{"city":"Houston","customerid":2,"firstname":"David","id":2,"item":"plate","lastname":"Johnson","phoneNumber":1451412345,"price":0.5,"state":"TX","streetAddress":"3 Baypointe Parkway","zipcode":1738378970} +{"city":"Manhattan","customerid":3,"firstname":"Hugh","id":3,"item":"tea","lastname":"Jackman","phoneNumber":1695412345,"price":1.99,"state":"NY","streetAddress":"5, Cool Way","zipcode":6726312345} +{"city":"Orlando","customerid":null,"firstname":"Walter","id":4,"item":null,"lastname":"White","phoneNumber":7829812345,"price":null,"state":"FL","streetAddress":"3828 Piermont Dr","zipcode":7349864532} +{"city":"San Diego","customerid":5,"firstname":"Frank","id":5,"item":"cookie","lastname":"Underwood","phoneNumber":19061512345,"price":0.5,"state":"CA","streetAddress":"1609 Far St.","zipcode":2977876770} +{"city":"Las Vegas","customerid":null,"firstname":"Serena","id":6,"item":null,"lastname":"Woods","phoneNumber":78919612345,"price":null,"state":"NV","streetAddress":"123 Far St.","zipcode":4533456734} +{"city":"New York","customerid":null,"firstname":"Sarthak","id":7,"item":null,"lastname":"Dash","phoneNumber":1324812345,"price":null,"state":"NY","streetAddress":"123 Far St.","zipcode":2345365523} +{"city":"Bangalore","customerid":null,"firstname":"Rahul","id":8,"item":null,"lastname":"Dash","phoneNumber":94864612345,"price":null,"state":"KA","streetAddress":"22 MG Road.","zipcode":null} \ No newline at end of file diff --git a/src/e2e-test/resources/testdata/BigQuery/BigQueryCreateTableJoin.txt b/src/e2e-test/resources/testdata/BigQuery/BigQueryCreateTableJoin.txt new file mode 100644 index 000000000..7c7288de1 --- /dev/null +++ b/src/e2e-test/resources/testdata/BigQuery/BigQueryCreateTableJoin.txt @@ -0,0 +1,2 @@ +create table `DATASET.TABLE_NAME` (id INT64, firstname STRING, lastname STRING, streetAddress STRING, +city STRING, state STRING, zipcode BIGINT, phoneNumber BIGINT) \ No newline at end of file diff --git a/src/e2e-test/resources/testdata/BigQuery/BigQueryInsertDataJoin.txt b/src/e2e-test/resources/testdata/BigQuery/BigQueryInsertDataJoin.txt new file mode 100644 index 000000000..19978e6a4 --- /dev/null +++ b/src/e2e-test/resources/testdata/BigQuery/BigQueryInsertDataJoin.txt @@ -0,0 +1,10 @@ +INSERT INTO DATASET.TABLE_NAME (id, firstname, lastname, streetAddress, city, state, zipcode, phoneNumber) +VALUES +(5, 'Frank', 'Underwood', '1609 Far St.', 'San Diego', 'CA', 2977876770, 19061512345), +(1, 'Douglas', 'Williams', '1 Vista Montana', 'San Jose', 'CA', 9513498885, 35834612345), +(4, 'Walter', 'White', '3828 Piermont Dr', 'Orlando', 'FL', 7349864532, 7829812345), +(3, 'Hugh', 'Jackman', '5, Cool Way', 'Manhattan', 'NY', 6726312345, 1695412345), +(7, 'Sarthak', 'Dash', '123 Far St.', 'New York', 'NY', 2345365523, 1324812345), +(6, 'Serena', 'Woods', '123 Far St.', 'Las Vegas', 'NV', 4533456734, 78919612345), +(2, 'David', 'Johnson', '3 Baypointe Parkway', 'Houston', 'TX', 1738378970, 1451412345), +(8, 'Rahul', 'Dash', '22 MG Road.', 'Bangalore', 'KA',NULL, 94864612345); \ No newline at end of file