Skip to content

Commit

Permalink
ITN coverage for SQl engine
Browse files Browse the repository at this point in the history
  • Loading branch information
AnkitCLI committed Apr 5, 2024
1 parent 8b780c3 commit fd73f04
Show file tree
Hide file tree
Showing 6 changed files with 89 additions and 10 deletions.
34 changes: 27 additions & 7 deletions src/e2e-test/features/bigquery/source/BigQuerySqlEngine.feature
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,13 @@
@BigQuery_Sink
Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data transfer

@BQ_SOURCE_SQLENGINE_TEST @BQ_SINK_TEST @EXISTING_BQ_CONNECTION
Scenario:Validate successful records transfer from BigQuery source to BigQuery sink
@BQ_SOURCE_JOINER_TEST @BQ_SOURCE_JOINER2_TEST @BQ_DELETE_JOIN @BQ_SINK_TEST @EXISTING_BQ_CONNECTION
Scenario:Validate successful records transfer from BigQuery source to BigQuery sink using Join
Given Open Datafusion Project to configure pipeline
When Expand Plugin group in the LHS plugins list: "Source"
When Select plugin: "BigQuery" from the plugins list as: "Source"
When Expand Plugin group in the LHS plugins list: "Sink"
When Select plugin: "BigQuery" from the plugins list as: "Sink"
Then Connect plugins: "BigQuery" and "BigQuery2" to establish connection
When Expand Plugin group in the LHS plugins list: "Analytics"
When Select plugin: "Joiner" from the plugins list as: "Analytics"
Then Navigate to the properties page of plugin: "BigQuery"
Then Click plugin property: "switch-useConnection"
Then Click on the Browse Connections button
Expand All @@ -33,15 +32,36 @@ Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data tr
Then Click on the Get Schema button
Then Validate "BigQuery" plugin properties
And Close the Plugin Properties page
When Expand Plugin group in the LHS plugins list: "Source"
When Select plugin: "BigQuery" from the plugins list as: "Source"
When Expand Plugin group in the LHS plugins list: "Sink"
When Select plugin: "BigQuery" from the plugins list as: "Sink"
Then Connect plugins: "BigQuery" and "Joiner" to establish connection
Then Connect plugins: "BigQuery2" and "Joiner" to establish connection
Then Connect plugins: "Joiner" and "BigQuery3" to establish connection
Then Navigate to the properties page of plugin: "BigQuery2"
Then Click plugin property: "useConnection"
Then Click on the Browse Connections button
Then Select connection: "bqConnectionName"
Then Enter input plugin property: "referenceName" with value: "BQRefName"
Then Enter input plugin property: "dataset" with value: "dataset"
Then Enter input plugin property: "table" with value: "bqSourceTable2"
Then Validate "BigQuery2" plugin properties
And Close the Plugin Properties page
Then Navigate to the properties page of plugin: "Joiner"
Then Select radio button plugin property: "conditionType" with value: "basic"
Then Click on the Get Schema button
Then Validate "Joiner" plugin properties
Then Close the Plugin Properties page
Then Navigate to the properties page of plugin: "BigQuery3"
Then Click plugin property: "useConnection"
Then Click on the Browse Connections button
Then Select connection: "bqConnectionName"
Then Enter input plugin property: "referenceName" with value: "BQSinkReferenceName"
Then Enter input plugin property: "dataset" with value: "dataset"
Then Enter input plugin property: "table" with value: "bqTargetTable"
Then Validate "BigQuery" plugin properties
And Close the Plugin Properties page
Then Validate "BigQuery3" plugin properties
Then Close the Plugin Properties page
Then Save the pipeline
Then Preview and run the pipeline
Then Wait till pipeline preview is in running state
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1368,4 +1368,44 @@ public static void createSourceBQTableForSqlEngine() throws IOException, Interru
PluginPropertyUtils.pluginProp("bqInsertDataQueryFileSQL"));
}

@Before(order = 1, value = "@BQ_SOURCE_JOINER_TEST")
public static void createSourceBQTableForJoiner() throws IOException, InterruptedException {
createSourceBQTableWithQueries(PluginPropertyUtils.pluginProp("bqCreateTableQueryFileJoin"),
PluginPropertyUtils.pluginProp("bqInsertDataQueryFileJoin"));
}

@Before(order = 1, value = "@BQ_SOURCE_JOINER2_TEST")
public static void createsecondBQTableJoin() throws IOException, InterruptedException {
bqSourceTable2 = "E2E_SOURCE_" + UUID.randomUUID().toString().replaceAll("-", "_");
BigQueryClient.getSoleQueryResult("create table `" + datasetName + "." + bqSourceTable2 + "` " +
"(customerid INT64, item STRING, price FLOAT64 )");
try {
io.cdap.e2e.utils.BigQueryClient.getSoleQueryResult("INSERT INTO `" + datasetName + "." + bqSourceTable2 + "` " +
"(customerid, item, price)" +
"VALUES (1, 'donut', 0.8), " +
"(1, 'coffee', 2.05), " +
"(2, 'donut', 1.5), " +
"(2, 'plate', 0.5), " +
"(3, 'tea', 1.99), " +
"(5, 'cookie', 0.5)");
} catch (NoSuchElementException e) {
// Insert query does not return any record.
// Iterator on TableResult values in getSoleQueryResult method throws NoSuchElementException
BeforeActions.scenario.write("Error inserting the record in the table" + e.getStackTrace());
}
PluginPropertyUtils.addPluginProp("bqSourceTable2", bqSourceTable2);
BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " created successfully");
}

@After(order = 1, value = "@BQ_DELETE_JOIN")
public static void deleteBQTablesJoin() throws IOException, InterruptedException {
BigQueryClient.dropBqQuery(bqSourceTable);
BigQueryClient.dropBqQuery(bqSourceTable2);
PluginPropertyUtils.removePluginProp("bqSourceTable");
PluginPropertyUtils.removePluginProp("bqSourceTable2");
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully");
BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully");
bqSourceTable = StringUtils.EMPTY;
bqSourceTable2 = StringUtils.EMPTY;
}
}
2 changes: 2 additions & 0 deletions src/e2e-test/resources/pluginParameters.properties
Original file line number Diff line number Diff line change
Expand Up @@ -346,4 +346,6 @@ groupByGcsAggregateFields=[{"key":"name#Count","value":"namecount"}]
deduplicateTestOutputFile=testdata/BQValidationExpectedFiles/deduplicate_Test
groupByTestOutputFile=testdata/BQValidationExpectedFiles/groupby_Test
bqExpectedFile=testdata/BQValidationExpectedFiles/bqexpected_Test
bqCreateTableQueryFileJoin=testdata/BigQuery/BigQueryCreateTableJoin.txt
bqInsertDataQueryFileJoin=testdata/BigQuery/BigQueryInsertDataJoin.txt
## SQLENGINE-PLUGIN-PROPERTIES-END
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
{"address":"ppu","id":13,"name":"root"}
{"address":"ggn","id":12,"name":"joe"}
{"address":"ggn","id":12,"name":"joe"}
{"city":"San Jose","customerid":1,"firstname":"Douglas","id":1,"item":"coffee","lastname":"Williams","phoneNumber":35834612345,"price":2.05,"state":"CA","streetAddress":"1 Vista Montana","zipcode":9513498885}
{"city":"Houston","customerid":2,"firstname":"David","id":2,"item":"plate","lastname":"Johnson","phoneNumber":1451412345,"price":0.5,"state":"TX","streetAddress":"3 Baypointe Parkway","zipcode":1738378970}
{"city":"Manhattan","customerid":3,"firstname":"Hugh","id":3,"item":"tea","lastname":"Jackman","phoneNumber":1695412345,"price":1.99,"state":"NY","streetAddress":"5, Cool Way","zipcode":6726312345}
{"city":"Orlando","customerid":null,"firstname":"Walter","id":4,"item":null,"lastname":"White","phoneNumber":7829812345,"price":null,"state":"FL","streetAddress":"3828 Piermont Dr","zipcode":7349864532}
{"city":"San Diego","customerid":5,"firstname":"Frank","id":5,"item":"cookie","lastname":"Underwood","phoneNumber":19061512345,"price":0.5,"state":"CA","streetAddress":"1609 Far St.","zipcode":2977876770}
{"city":"Las Vegas","customerid":null,"firstname":"Serena","id":6,"item":null,"lastname":"Woods","phoneNumber":78919612345,"price":null,"state":"NV","streetAddress":"123 Far St.","zipcode":4533456734}
{"city":"New York","customerid":null,"firstname":"Sarthak","id":7,"item":null,"lastname":"Dash","phoneNumber":1324812345,"price":null,"state":"NY","streetAddress":"123 Far St.","zipcode":2345365523}
{"city":"Bangalore","customerid":null,"firstname":"Rahul","id":8,"item":null,"lastname":"Dash","phoneNumber":94864612345,"price":null,"state":"KA","streetAddress":"22 MG Road.","zipcode":null}
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
create table `DATASET.TABLE_NAME` (id INT64, firstname STRING, lastname STRING, streetAddress STRING,
city STRING, state STRING, zipcode BIGINT, phoneNumber BIGINT)
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
INSERT INTO DATASET.TABLE_NAME (id, firstname, lastname, streetAddress, city, state, zipcode, phoneNumber)
VALUES
(5, 'Frank', 'Underwood', '1609 Far St.', 'San Diego', 'CA', 2977876770, 19061512345),
(1, 'Douglas', 'Williams', '1 Vista Montana', 'San Jose', 'CA', 9513498885, 35834612345),
(4, 'Walter', 'White', '3828 Piermont Dr', 'Orlando', 'FL', 7349864532, 7829812345),
(3, 'Hugh', 'Jackman', '5, Cool Way', 'Manhattan', 'NY', 6726312345, 1695412345),
(7, 'Sarthak', 'Dash', '123 Far St.', 'New York', 'NY', 2345365523, 1324812345),
(6, 'Serena', 'Woods', '123 Far St.', 'Las Vegas', 'NV', 4533456734, 78919612345),
(2, 'David', 'Johnson', '3 Baypointe Parkway', 'Houston', 'TX', 1738378970, 1451412345),
(8, 'Rahul', 'Dash', '22 MG Road.', 'Bangalore', 'KA',NULL, 94864612345);

0 comments on commit fd73f04

Please sign in to comment.