new scenarios e2e bq review comments addressed

src/e2e-test/features/bigquery/sink/BigQueryToBigQuerySink.feature

@@ -249,7 +249,7 @@ Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data tr
     Then Validate the data transferred from BigQuery to BigQuery with actual And expected file for: "bqUpsertExpectedFile"
 
   @BQ_NULL_MODE_SOURCE_TEST @BQ_SINK_TEST @EXISTING_BQ_CONNECTION
-  Scenario: Validate Successful record transfer from BigQuery source plugin to BigQuery sink plugin with all null values in one column and few null values in different column.
+  Scenario: Validate Successful record transfer from BigQuery source plugin to BigQuery sink plugin having all null values in one column and few null values in another column in Source table
     Given Open Datafusion Project to configure pipeline
     When Expand Plugin group in the LHS plugins list: "Source"
     When Select plugin: "BigQuery" from the plugins list as: "Source"
@@ -297,7 +297,7 @@ Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data tr
     Then Validate the values of records transferred to BQ sink is equal to the values from source BigQuery table
 
   @BQ_UPDATE_SOURCE_DEDUPE_TEST @BQ_UPDATE_SINK_DEDUPE_TEST @EXISTING_BQ_CONNECTION
-  Scenario: Verify successful record transfer from BigQuery source to BigQuery sink using advance operation update with dedupe property.
+  Scenario: Verify successful record transfer from BigQuery source to BigQuery sink using advance operation update with Dedupe By Property.
     Given Open Datafusion Project to configure pipeline
     When Expand Plugin group in the LHS plugins list: "Source"
     When Select plugin: "BigQuery" from the plugins list as: "Source"
@@ -328,9 +328,9 @@ Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data tr
     Then Verify input plugin property: "dataset" contains value: "dataset"
     Then Enter input plugin property: "table" with value: "bqTargetTable"
     And Select radio button plugin property: "operation" with value: "update"
-    Then Enter Value for plugin property table key : "relationTableKey" with values: "Name"
+    Then Enter Value for plugin property table key : "relationTableKey" with values: "relationTableKeyValue"
     Then Select dropdown plugin property: "dedupeBy" with option value: "dedupeByOrder"
-    Then Enter key for plugin property: "dedupeBy" with values: "ID"
+    Then Enter key for plugin property: "dedupeBy" with values: "dedupeByValue"
     Then Validate "BigQuery" plugin properties
     Then Close the BigQuery properties
     Then Save the pipeline
@@ -346,7 +346,7 @@ Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data tr
     Then Open and capture logs
     Then Close the pipeline logs
     Then Verify the pipeline status is "Succeeded"
-    Then Validate the data transferred from BigQuery to BigQuery with actual And expected file for: "bqUpdatededupeExpectedFile"
+    Then Validate the data transferred from BigQuery to BigQuery with actual And expected file for: "bqUpdateDedupeExpectedFile"
 
   @BQ_INSERT_INT_SOURCE_TEST @BQ_EXISTING_SINK_TEST @EXISTING_BQ_CONNECTION
   Scenario: Verify successful record transfer for the Insert operation with partition type Integer and destination table is existing already.
@@ -401,8 +401,8 @@ Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data tr
     Then Verify the pipeline status is "Succeeded"
     Then Validate the data transferred from BigQuery to BigQuery with actual And expected file for: "bqInsertExpectedFile"
 
-  @BQ_TIME_STAMP_SOURCE_TEST @BQ_SINK_TEST @EXISTING_BQ_CONNECTION
-  Scenario: Verify successful record transfer for the Insert operation from BigQuery source plugin to BigQuery sink with partition type Time.
+  @BQ_TIME_SOURCE_TEST @BQ_SINK_TEST @EXISTING_BQ_CONNECTION
+  Scenario: Verify successful record transfer for the Insert operation from BigQuery source plugin to BigQuery sink with partition type Time and partition field is date.
     Given Open Datafusion Project to configure pipeline
     When Expand Plugin group in the LHS plugins list: "Source"
     When Select plugin: "BigQuery" from the plugins list as: "Source"
@@ -449,10 +449,110 @@ Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data tr
     Then Open and capture logs
     Then Close the pipeline logs
     Then Verify the pipeline status is "Succeeded"
+    Then Validate the data transferred from BigQuery to BigQuery with actual And expected file for: "bqDateExpectedFile"
+
+  @BQ_TIME_SOURCE_TEST @BQ_SINK_TEST @EXISTING_BQ_CONNECTION
+  Scenario: Verify successful record transfer for the Insert operation from BigQuery source plugin to BigQuery sink with partition type Time and partition field is datetime.
+    Given Open Datafusion Project to configure pipeline
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "BigQuery" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "BigQuery" from the plugins list as: "Sink"
+    Then Connect plugins: "BigQuery" and "BigQuery2" to establish connection
+    Then Navigate to the properties page of plugin: "BigQuery"
+    Then Click plugin property: "switch-useConnection"
+    Then Click on the Browse Connections button
+    Then Select connection: "bqConnectionName"
+    Then Click on the Browse button inside plugin properties
+    Then Select connection data row with name: "dataset"
+    Then Select connection data row with name: "bqSourceTable"
+    Then Wait till connection data loading completes with a timeout of 60 seconds
+    Then Verify input plugin property: "dataset" contains value: "dataset"
+    Then Verify input plugin property: "table" contains value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Validate "BigQuery" plugin properties
+    And Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Click plugin property: "useConnection"
+    Then Click on the Browse Connections button
+    Then Select connection: "bqConnectionName"
+    Then Enter input plugin property: "referenceName" with value: "BQSinkReferenceName"
+    Then Click on the Browse button inside plugin properties
+    Then Click SELECT button inside connection data row with name: "dataset"
+    Then Wait till connection data loading completes with a timeout of 60 seconds
+    Then Verify input plugin property: "dataset" contains value: "dataset"
+    Then Enter input plugin property: "table" with value: "bqTargetTable"
+    Then Enter input plugin property: "partitionByField" with value: "bqPartitionFieldDateTime"
+    Then Click plugin property: "updateTableSchema"
+    Then Validate "BigQuery" plugin properties
+    Then Close the BigQuery properties
+    Then Save the pipeline
+    Then Preview and run the pipeline
+    Then Wait till pipeline preview is in running state
+    Then Open and capture pipeline preview logs
+    Then Verify the preview run status of pipeline in the logs is "succeeded"
+    Then Close the pipeline logs
+    Then Close the preview
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Close the pipeline logs
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate the data transferred from BigQuery to BigQuery with actual And expected file for: "bqDateTimeExpectedFile"
+
+  @BQ_TIME_SOURCE_TEST @BQ_SINK_TEST @EXISTING_BQ_CONNECTION
+  Scenario: Verify successful record transfer for the Insert operation from BigQuery source plugin to BigQuery sink with partition type Time and partition field is timestamp.
+    Given Open Datafusion Project to configure pipeline
+    When Expand Plugin group in the LHS plugins list: "Source"
+    When Select plugin: "BigQuery" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "BigQuery" from the plugins list as: "Sink"
+    Then Connect plugins: "BigQuery" and "BigQuery2" to establish connection
+    Then Navigate to the properties page of plugin: "BigQuery"
+    Then Click plugin property: "switch-useConnection"
+    Then Click on the Browse Connections button
+    Then Select connection: "bqConnectionName"
+    Then Click on the Browse button inside plugin properties
+    Then Select connection data row with name: "dataset"
+    Then Select connection data row with name: "bqSourceTable"
+    Then Wait till connection data loading completes with a timeout of 60 seconds
+    Then Verify input plugin property: "dataset" contains value: "dataset"
+    Then Verify input plugin property: "table" contains value: "bqSourceTable"
+    Then Click on the Get Schema button
+    Then Validate "BigQuery" plugin properties
+    And Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Click plugin property: "useConnection"
+    Then Click on the Browse Connections button
+    Then Select connection: "bqConnectionName"
+    Then Enter input plugin property: "referenceName" with value: "BQSinkReferenceName"
+    Then Click on the Browse button inside plugin properties
+    Then Click SELECT button inside connection data row with name: "dataset"
+    Then Wait till connection data loading completes with a timeout of 60 seconds
+    Then Verify input plugin property: "dataset" contains value: "dataset"
+    Then Enter input plugin property: "table" with value: "bqTargetTable"
+    Then Enter input plugin property: "partitionByField" with value: "bqPartitionFieldTimeStamp"
+    Then Click plugin property: "updateTableSchema"
+    Then Validate "BigQuery" plugin properties
+    Then Close the BigQuery properties
+    Then Save the pipeline
+    Then Preview and run the pipeline
+    Then Wait till pipeline preview is in running state
+    Then Open and capture pipeline preview logs
+    Then Verify the preview run status of pipeline in the logs is "succeeded"
+    Then Close the pipeline logs
+    Then Close the preview
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Open and capture logs
+    Then Close the pipeline logs
+    Then Verify the pipeline status is "Succeeded"
     Then Validate the data transferred from BigQuery to BigQuery with actual And expected file for: "bqTimeStampExpectedFile"
 
   @BQ_UPSERT_DEDUPE_SOURCE_TEST @BQ_UPSERT_DEDUPE_SINK_TEST @EXISTING_BQ_CONNECTION
-  Scenario:Validate successful records transfer from BigQuery source to BigQuery sink with Upsert operation with dedupe source data and destination table already exists and update table schema is false
+  Scenario:Validate successful records transfer from BigQuery source to BigQuery sink with Upsert operation with dedupe source data and existing destination table where update table schema is set to false
     Given Open Datafusion Project to configure pipeline
     When Expand Plugin group in the LHS plugins list: "Source"
     When Select plugin: "BigQuery" from the plugins list as: "Source"
@@ -486,7 +586,7 @@ Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data tr
     Then Click on the Add Button of the property: "relationTableKey" with value:
       | TableKeyDedupe |
     Then Select dropdown plugin property: "dedupeBy" with option value: "dedupeBy"
-    Then Enter key for plugin property: "dedupeBy" with values: "Price"
+    Then Enter key for plugin property: "dedupeBy" with values: "dedupeByValueUpsert"
     Then Validate "BigQuery" plugin properties
     And Close the Plugin Properties page
     Then Save the pipeline
@@ -504,7 +604,7 @@ Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data tr
     Then Verify the pipeline status is "Succeeded"
     Then Validate the data transferred from BigQuery to BigQuery with actual And expected file for: "bqUpsertDedupeFile"
 
-  @BQ_RECORD_SOURCE_TEST @BQ_SINK_TEST
+  @BQ_RECORD_SOURCE_TEST @BQ_SECOND_RECORD_SOURCE_TEST @BQ_SINK_TEST
   Scenario: Validate successful record transfer from two BigQuery source plugins with different schema record names, taking one extra column in BigQuery source plugin 1,and
   using wrangler transformation plugin for removing the extra column and transferring the data in BigQuery sink plugin containing all the columns from both the source plugin.
     Given Open Datafusion Project to configure pipeline
@@ -517,6 +617,13 @@ Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data tr
     Then Click on the Get Schema button
     Then Click on the Validate button
     Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "BigQuery2"
+    Then Replace input plugin property: "project" with value: "projectId"
+    Then Replace input plugin property: "dataset" with value: "dataset"
+    Then Replace input plugin property: "table" with value: "bqSourceTable2"
+    Then Click on the Get Schema button
+    Then Click on the Validate button
+    Then Close the Plugin Properties page
     Then Navigate to the properties page of plugin: "BigQuery3"
     Then Replace input plugin property: "project" with value: "projectId"
     Then Replace input plugin property: "table" with value: "bqTargetTable"
@@ -572,5 +679,5 @@ Feature: BigQuery sink - Verification of BigQuery to BigQuery successful data tr
     Then Run the Pipeline in Runtime
     Then Wait till pipeline is in running state
     Then Open and capture logs
-    Then Close the pipeline logs
     Then Verify the pipeline status is "Succeeded"
+    Then Close the pipeline logs

src/e2e-test/java/io/cdap/plugin/bigquery/runners/sinkrunner/TestRunner.java

@@ -27,7 +27,7 @@
   features = {"src/e2e-test/features"},
   glue = {"io.cdap.plugin.bigquery.stepsdesign", "io.cdap.plugin.gcs.stepsdesign",
     "stepsdesign", "io.cdap.plugin.common.stepsdesign"},
-  tags = {"@BigQuery_Sink not @CDAP-20830"},
+  tags = {"@BigQuery_Sink and not @CDAP-20830"},
   //TODO: Enable test once issue is fixed https://cdap.atlassian.net/browse/CDAP-20830
   monochrome = true,
   plugin = {"pretty", "html:target/cucumber-html-report/bigquery-sink",

src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java

@@ -59,6 +59,7 @@ public class TestSetupHooks {
   public static String gcsTargetBucketName = StringUtils.EMPTY;
   public static String bqTargetTable = StringUtils.EMPTY;
   public static String bqSourceTable = StringUtils.EMPTY;
+  public static String bqSourceTable2 = StringUtils.EMPTY;
   public static String bqSourceView = StringUtils.EMPTY;
   public static String pubSubTargetTopic = StringUtils.EMPTY;
   public static String spannerInstance = StringUtils.EMPTY;
@@ -270,7 +271,11 @@ public static void createTempSourceBQTable() throws IOException, InterruptedExce
   }
 
   @After(order = 1, value = "@BQ_SOURCE_TEST or @BQ_PARTITIONED_SOURCE_TEST or @BQ_SOURCE_DATATYPE_TEST or " +
-    "@BQ_INSERT_SOURCE_TEST or @BQ_UPDATE_SINK_TEST")
+    "@BQ_INSERT_SOURCE_TEST or @BQ_UPDATE_SINK_TEST or @BQ_UPSERT_SOURCE_TEST or @BQ_UPSERT_SINK_TEST or " +
+    "@BQ_NULL_MODE_SOURCE_TEST or @BQ_UPDATE_SOURCE_DEDUPE_TEST or @BQ_UPDATE_SINK_DEDUPE_TEST or " +
+    "@BQ_INSERT_INT_SOURCE_TEST or @BQ_EXISTING_SINK_TEST or @BQ_TIME_STAMP_SOURCE_TEST or " +
+    "@BQ_UPSERT_DEDUPE_SOURCE_TEST or @BQ_UPSERT_DEDUPE_SINK_TEST or @BQ_RECORD_SOURCE_TEST or " +
+    "@BQ_SECOND_RECORD_SOURCE_TEST or @BQ_INSERT_SINK_TEST")
   public static void deleteTempSourceBQTable() throws IOException, InterruptedException {
     BigQueryClient.dropBqQuery(bqSourceTable);
     PluginPropertyUtils.removePluginProp("bqSourceTable");
@@ -1091,19 +1096,23 @@ public static void createSinkBQExistingTable() throws IOException, InterruptedEx
     BeforeActions.scenario.write("BQ Target Table " + bqTargetTable + " updated successfully");
   }
 
-  @Before(order = 1, value = "@BQ_TIME_STAMP_SOURCE_TEST")
+  @Before(order = 1, value = "@BQ_TIME_SOURCE_TEST")
   public static void createTimeStampBQTable() throws IOException, InterruptedException {
     bqSourceTable = "E2E_SOURCE_" + UUID.randomUUID().toString().replaceAll("-", "_");
     PluginPropertyUtils.addPluginProp("bqSourceTable", bqSourceTable);
     BeforeActions.scenario.write("BQ source table name - " + bqSourceTable);
     BigQueryClient.getSoleQueryResult("create table `" + datasetName + "." + bqSourceTable + "` " +
-                                        "(ID STRING, transaction_date DATE, Firstname STRING)");
+                                        "(ID STRING, transaction_date DATE, Firstname STRING," +
+                                        " transaction_dt DATETIME, updated_on TIMESTAMP )");
     try {
       BigQueryClient.getSoleQueryResult("INSERT INTO `" + datasetName + "." + bqSourceTable + "` " +
-                                          "(ID,  transaction_date, Firstname)" +
-                                          "VALUES" + "('Agra', '2021-02-20', 'Neera')," +
-                                          "('Noida', '2021-02-21','')," +
-                                          "('Gurgaon', '2021-02-22', 'singh')");
+                                          "(ID,  transaction_date, Firstname, transaction_dt, updated_on  )" +
+                                          "VALUES" + "('Agra', '2021-02-20', 'Neera','2019-07-07 11:24:00', " +
+                                          "'2019-03-10 04:50:01 UTC')," +
+                                          "('Noida', '2021-02-21','', '2019-07-07 11:24:00', " +
+                                          "'2019-03-10 04:50:01 UTC')," +
+                                          "('Gurgaon', '2021-02-22', 'singh', '2019-07-07 11:24:00', " +
+                                          "'2019-03-10 04:50:01 UTC' )");
     } catch (NoSuchElementException e) {
       // Insert query does not return any record.
       // Iterator on TableResult values in getSoleQueryResult method throws NoSuchElementException
@@ -1175,6 +1184,24 @@ public static void createSourceBQRecordTable() throws IOException, InterruptedEx
     BeforeActions.scenario.write("BQ Source Table " + bqSourceTable + " created successfully");
   }
 
+  @Before(order = 1, value = "@BQ_SECOND_RECORD_SOURCE_TEST")
+  public static void createSourceBQSecondRecordTable() throws IOException, InterruptedException {
+    bqSourceTable2 = "E2E_SOURCE_" + UUID.randomUUID().toString().replaceAll("-", "_");
+    io.cdap.e2e.utils.BigQueryClient.getSoleQueryResult("create table `" + datasetName + "." + bqSourceTable2 + "` " +
+                                                          "(ID INT64, Name STRING, " + "Price FLOAT64 ) ");
+    try {
+      io.cdap.e2e.utils.BigQueryClient.getSoleQueryResult("INSERT INTO `" + datasetName + "." + bqSourceTable2 + "` " +
+                                                            "(ID,  Name, Price)" +
+                                                            "VALUES" + "(1, 'string_1', 0.1)");
+    } catch (NoSuchElementException e) {
+      // Insert query does not return any record.
+      // Iterator on TableResult values in getSoleQueryResult method throws NoSuchElementException
+      BeforeActions.scenario.write("Error inserting the record in the table" + e.getStackTrace());
+    }
+    PluginPropertyUtils.addPluginProp("bqSourceTable2", bqSourceTable2);
+    BeforeActions.scenario.write("BQ Source Table " + bqSourceTable2 + " created successfully");
+  }
+
   @Before(order = 1, value = "@BQ_INSERT_SINK_TEST")
   public static void createSinkBQInsertTable() throws IOException, InterruptedException {
 

src/e2e-test/resources/pluginDataCyAttributes.properties

@@ -42,4 +42,6 @@ spannerConnectionRow=connector-Spanner
 testConnection=connection-test-button
 connectionCreate=connection-submit-button
 parsingOptionConfirm=parsing-config-confirm
+dedupeBy=dedupeBy
+relationTableKey=relationTableKey
 ## CONNECTION-MANAGEMENT-END