Merge pull request data-integrations#1363 from cloudsufi/SpannerE2EAd…

…ditional

Spanner To Spanner Additional Cases as per ITN class.

src/e2e-test/features/spanner/source/SpannerToSpanner_Additional.feature

@@ -0,0 +1,119 @@
+@Spanner_Source
+Feature: Spanner source - Verification of Additional Spanner to Spanner successful data transfer without using connections
+
+  @SPANNER_TEST @EXISTING_SPANNER_SINK
+  Scenario: To verify data is getting transferred from no schema Spanner source to Spanner sink having existing table
+    Given Open Datafusion Project to configure pipeline
+    When Select plugin: "Spanner" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "Spanner" from the plugins list as: "Sink"
+    Then Connect plugins: "Spanner" and "Spanner2" to establish connection
+    Then Navigate to the properties page of plugin: "Spanner"
+    Then Enter Spanner property reference name
+    Then Enter Spanner property projectId "projectId"
+    Then Override Service account details if set in environment variables
+    Then Enter input plugin property: "instanceId" with value: "spannerInstance"
+    Then Enter input plugin property: "databaseName" with value: "spannerDatabase"
+    Then Enter input plugin property: "tableName" with value: "spannerSourceTable"
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "Spanner2"
+    Then Enter Spanner property reference name
+    Then Enter Spanner property projectId "projectId"
+    Then Override Service account details if set in environment variables
+    Then Enter input plugin property: "instanceId" with value: "spannerInstance"
+    Then Enter input plugin property: "databaseName" with value: "spannerDatabase"
+    Then Enter input plugin property: "tableName" with value: "spannerExistingTargetTable"
+    Then Enter Spanner sink property primary key "spannerSinkPrimaryKeySpanner"
+    Then Validate "Spanner" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Preview and run the pipeline
+    Then Wait till pipeline preview is in running state
+    Then Open and capture pipeline preview logs
+    Then Verify the preview run status of pipeline in the logs is "succeeded"
+    Then Close the pipeline logs
+    Then Close the preview
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate records transferred to already existing target spanner table with record counts of source table
+
+  @SPANNER_TEST @EXISTING_SPANNER_SINK
+  Scenario: To verify data is getting transferred from Spanner source to Spanner sink having existing table
+    Given Open Datafusion Project to configure pipeline
+    When Select plugin: "Spanner" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "Spanner" from the plugins list as: "Sink"
+    Then Connect plugins: "Spanner" and "Spanner2" to establish connection
+    Then Navigate to the properties page of plugin: "Spanner"
+    Then Enter Spanner property reference name
+    Then Enter Spanner property projectId "projectId"
+    Then Override Service account details if set in environment variables
+    Then Enter input plugin property: "instanceId" with value: "spannerInstance"
+    Then Enter input plugin property: "databaseName" with value: "spannerDatabase"
+    Then Enter input plugin property: "tableName" with value: "spannerSourceTable"
+    Then Validate output schema with expectedSchema "spannerSourceSchema"
+    Then Validate "Spanner" plugin properties
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "Spanner2"
+    Then Enter Spanner property reference name
+    Then Enter Spanner property projectId "projectId"
+    Then Override Service account details if set in environment variables
+    Then Enter input plugin property: "instanceId" with value: "spannerInstance"
+    Then Enter input plugin property: "databaseName" with value: "spannerDatabase"
+    Then Enter input plugin property: "tableName" with value: "spannerExistingTargetTable"
+    Then Enter Spanner sink property primary key "spannerSinkPrimaryKeySpanner"
+    Then Validate "Spanner" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Preview and run the pipeline
+    Then Wait till pipeline preview is in running state
+    Then Open and capture pipeline preview logs
+    Then Verify the preview run status of pipeline in the logs is "succeeded"
+    Then Close the pipeline logs
+    Then Close the preview
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate records transferred to already existing target spanner table with record counts of source table
+
+   @SPANNER_TEST @SPANNER_SINK_TEST
+  Scenario: To verify data is getting transferred from no schema Spanner source to Spanner non existing sink table
+    Given Open Datafusion Project to configure pipeline
+    When Select plugin: "Spanner" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "Spanner" from the plugins list as: "Sink"
+    Then Connect plugins: "Spanner" and "Spanner2" to establish connection
+    Then Navigate to the properties page of plugin: "Spanner"
+    Then Enter Spanner property reference name
+    Then Enter Spanner property projectId "projectId"
+    Then Override Service account details if set in environment variables
+    Then Enter input plugin property: "instanceId" with value: "spannerInstance"
+    Then Enter input plugin property: "databaseName" with value: "spannerDatabase"
+    Then Enter input plugin property: "tableName" with value: "spannerSourceTable"
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "Spanner2"
+    Then Enter Spanner property reference name
+    Then Enter Spanner property projectId "projectId"
+    Then Override Service account details if set in environment variables
+    Then Enter input plugin property: "instanceId" with value: "spannerInstance"
+    Then Enter input plugin property: "databaseName" with value: "spannerTargetDatabase"
+    Then Enter input plugin property: "tableName" with value: "spannerTargetTable"
+    Then Enter Spanner sink property primary key "spannerSinkPrimaryKeySpanner"
+    Then Validate "Spanner" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Preview and run the pipeline
+    Then Wait till pipeline preview is in running state
+    Then Open and capture pipeline preview logs
+    Then Verify the preview run status of pipeline in the logs is "succeeded"
+    Then Close the pipeline logs
+    Then Close the preview
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate records transferred to target spanner table with record counts of source spanner table
+

src/e2e-test/features/spanner/source/SpannerToSpanner_WithMacro.feature

@@ -0,0 +1,55 @@
+@Spanner_Source
+Feature: Spanner source - Verification of Spanner to Spanner Successful data transfer with macro arguments
+
+  @SPANNER_SINK_TEST @SPANNER_TEST
+  Scenario: To verify data is getting transferred from Spanner to Spanner with macro arguments
+    Given Open Datafusion Project to configure pipeline
+    When Select plugin: "Spanner" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "Spanner" from the plugins list as: "Sink"
+    Then Connect plugins: "Spanner" and "Spanner2" to establish connection
+    Then Navigate to the properties page of plugin: "Spanner"
+    Then Enter Spanner property reference name
+    Then Enter Spanner property projectId "projectId"
+    Then Override Service account details if set in environment variables
+    Then Click on the Macro button of Property: "instanceId" and set the value to: "macroStringInstance"
+    Then Click on the Macro button of Property: "databaseName" and set the value to: "macroStringDatabase"
+    Then Click on the Macro button of Property: "tableName" and set the value to: "macroStringSourceTable"
+    Then Validate "Spanner" plugin properties
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "Spanner2"
+    Then Enter Spanner property reference name
+    Then Enter Spanner property projectId "projectId"
+    Then Override Service account details if set in environment variables
+    Then Click on the Macro button of Property: "instanceId" and set the value to: "macroStringInstance"
+    Then Click on the Macro button of Property: "databaseName" and set the value to: "macroStringTargetDatabase"
+    Then Click on the Macro button of Property: "tableName" and set the value to: "macroStringTargetTable"
+    Then Click on the Macro button of Property: "keys" and set the value to: "macroStringPrimaryKey"
+    Then Validate "Spanner" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Preview and run the pipeline
+    Then Enter runtime argument value "spannerInstance" for key "macroStringInstance"
+    Then Enter runtime argument value "spannerDatabase" for key "macroStringDatabase"
+    Then Enter runtime argument value "spannerSourceTable" for key "macroStringSourceTable"
+    Then Enter runtime argument value "spannerTargetDatabase" for key "macroStringTargetDatabase"
+    Then Enter runtime argument value "spannerTargetTable" for key "macroStringTargetTable"
+    Then Enter runtime argument value "spannerSinkPrimaryKeySpanner" for key "macroStringPrimaryKey"
+    Then Run the preview of pipeline with runtime arguments
+    Then Wait till pipeline preview is in running state
+    Then Open and capture pipeline preview logs
+    Then Verify the preview run status of pipeline in the logs is "succeeded"
+    Then Close the pipeline logs
+    Then Close the preview
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Enter runtime argument value "spannerInstance" for key "macroStringInstance"
+    Then Enter runtime argument value "spannerDatabase" for key "macroStringDatabase"
+    Then Enter runtime argument value "spannerSourceTable" for key "macroStringSourceTable"
+    Then Enter runtime argument value "spannerTargetDatabase" for key "macroStringTargetDatabase"
+    Then Enter runtime argument value "spannerTargetTable" for key "macroStringTargetTable"
+    Then Enter runtime argument value "spannerSinkPrimaryKeySpanner" for key "macroStringPrimaryKey"
+    Then Run the Pipeline in Runtime with runtime arguments
+    Then Wait till pipeline is in running state
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate records transferred to target spanner table with record counts of source spanner table

src/e2e-test/features/spanner/source/SpannerToSpanner_withConnections.feature

@@ -163,3 +163,52 @@ Feature: Spanner source - Verification of Spanner to Spanner successful data tra
     Then Wait till pipeline is in running state
     Then Verify the pipeline status is "Succeeded"
     Then Validate records transferred to target spanner table with record counts of source spanner table
+
+  @EXISTING_SPANNER_SINK @EXISTING_SPANNER_CONNECTION @Spanner_Source_Required
+  Scenario: To verify data is getting transferred from Spanner source  to existing Spanner sink
+    Given Open Datafusion Project to configure pipeline
+    When Select plugin: "Spanner" from the plugins list as: "Source"
+    When Expand Plugin group in the LHS plugins list: "Sink"
+    When Select plugin: "Spanner" from the plugins list as: "Sink"
+    Then Connect plugins: "Spanner" and "Spanner2" to establish connection
+    Then Navigate to the properties page of plugin: "Spanner"
+    Then Click plugin property: "switch-useConnection"
+    Then Click on the Browse Connections button
+    Then Select connection: "spannerConnectionName"
+    Then Click on the Browse button inside plugin properties
+    Then Select connection data row with name: "spannerInstance"
+    Then Select connection data row with name: "spannerDatabase"
+    Then Select connection data row with name: "spannerSourceTable"
+    Then Wait till connection data loading completes with a timeout of 60 seconds
+    Then Verify input plugin property: "instanceId" contains value: "spannerInstance"
+    Then Verify input plugin property: "databaseName" contains value: "spannerDatabase"
+    Then Verify input plugin property: "tableName" contains value: "spannerSourceTable"
+    Then Validate output schema with expectedSchema "spannerSourceSchema"
+    Then Validate "Spanner" plugin properties
+    Then Close the Plugin Properties page
+    Then Navigate to the properties page of plugin: "Spanner2"
+    Then Enter input plugin property: "referenceName" with value: "SpannerSinkReferenceName"
+    Then Click plugin property: "useConnection"
+    Then Click on the Browse Connections button
+    Then Select connection: "spannerConnectionName"
+    Then Click on the Browse button inside plugin properties
+    Then Click SELECT button inside connection data row with name: "spannerInstance"
+    Then Wait till connection data loading completes with a timeout of 60 seconds
+    Then Verify input plugin property: "instanceId" contains value: "spannerInstance"
+    Then Enter input plugin property: "databaseName" with value: "spannerDatabase"
+    Then Enter input plugin property: "tableName" with value: "spannerExistingTargetTable"
+    Then Enter Spanner sink property primary key "spannerSinkPrimaryKeySpanner"
+    Then Validate "Spanner" plugin properties
+    Then Close the Plugin Properties page
+    Then Save the pipeline
+    Then Preview and run the pipeline
+    Then Wait till pipeline preview is in running state
+    Then Open and capture pipeline preview logs
+    Then Verify the preview run status of pipeline in the logs is "succeeded"
+    Then Close the pipeline logs
+    Then Close the preview
+    Then Deploy the pipeline
+    Then Run the Pipeline in Runtime
+    Then Wait till pipeline is in running state
+    Then Verify the pipeline status is "Succeeded"
+    Then Validate records transferred to already existing target spanner table with record counts of source table

src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java

@@ -69,6 +69,7 @@ public class TestSetupHooks {
   public static String spannerTargetTable = StringUtils.EMPTY;
   public static boolean firstSpannerTestFlag = true;
   public static String datasetName = PluginPropertyUtils.pluginProp("dataset");
+  public static String spannerExistingTargetTable = StringUtils.EMPTY;
 
   @Before(order = 1)
   public static void overrideServiceAccountFilePathIfProvided() {
@@ -1333,4 +1334,32 @@ public static void replaceTableDetailsInInsertQuerySql() {
   public static void setInsertQueryBackWithTableDetailsPlaceholderSql() {
     setQueryBackWithTableDetailsPlaceholder("bqExecuteInsert");
   }
+
+  @Before(order = 2, value = "@EXISTING_SPANNER_SINK")
+  public static void makeExistingTargetSpannerDBAndTableName() {
+    try {
+      spannerTargetDatabase = spannerDatabase;
+      spannerExistingTargetTable = PluginPropertyUtils.pluginProp("spannerExistingTargetTable");
+      String createQuery = null;
+      try {
+        createQuery = new String(Files.readAllBytes(Paths.get(TestSetupHooks.class.getResource
+                ("/" + PluginPropertyUtils.pluginProp("spannerTestDataCreateExistingSinkTableQueriesFile")).toURI()))
+                , StandardCharsets.UTF_8);
+
+      } catch (Exception e) {
+        BeforeActions.scenario.write("Exception in reading "
+                + PluginPropertyUtils.pluginProp("spannerTestDataCreateExistingSinkTableQueriesFile")
+                + " - " + e.getMessage());
+        Assert.fail("Exception in Spanner testdata prerequisite -"
+                + "error in reading create existing table queries file."
+                + e.getMessage());
+      }
+      SpannerClient.executeDMLQuery(spannerInstance, spannerTargetDatabase, createQuery);
+      PluginPropertyUtils.addPluginProp("spannerTargetDatabase", spannerTargetDatabase);
+      PluginPropertyUtils.addPluginProp("spannerExistingTargetTable", spannerExistingTargetTable);
+    } catch (Exception e) {
+      e.printStackTrace();
+    }
+  }
+
 }

src/e2e-test/java/io/cdap/plugin/spanner/stepsdesign/SpannerBase.java

@@ -152,4 +152,21 @@ public void validateRecordsTransferredToTargetSpannerTableWithRecordCountsOfSour
 
     Assert.assertEquals(spannerSourceTableRecordCount, spannerTargetTableRecordCount);
   }
+
+  @Then("Validate records transferred to already existing target spanner table with record counts of source table")
+  public void validateRecordsTransferredToExistingSpannerTableWithRecordCountsOfSourceSpannerTable() {
+    int spannerSourceTableRecordCount = SpannerClient
+            .getCountOfRecordsInTable(PluginPropertyUtils.pluginProp("spannerInstance"),
+                    PluginPropertyUtils.pluginProp("spannerDatabase"),
+                    PluginPropertyUtils.pluginProp("spannerSourceTable"));
+    BeforeActions.scenario.write("No of records from Spanner Source table :" + spannerSourceTableRecordCount);
+    int spannerTargetTableRecordCount = SpannerClient
+            .getCountOfRecordsInTable(PluginPropertyUtils.pluginProp("spannerInstance"),
+                    PluginPropertyUtils.pluginProp("spannerDatabase"),
+                    PluginPropertyUtils.pluginProp("spannerExistingTargetTable"));
+    BeforeActions.scenario.write("No of records transferred to existing Spanner target table:"
+            + spannerTargetTableRecordCount);
+
+    Assert.assertEquals(spannerSourceTableRecordCount, spannerTargetTableRecordCount);
+  }
 }

src/e2e-test/resources/pluginParameters.properties

@@ -278,6 +278,8 @@ spannerTestDataInsertDataQueriesFile=testdata/SpannerInsertDataQueries.txt
 spannerTestDataInsertBasicDataQueriesFile=testdata/SpannerInsertEmployeeBasicDataQueries.txt
 spannerSourceTable=EmployeeDetails
 spannerSourceBasicTable=EmployeeDetails_Basic
+spannerExistingTargetTable=ExistingSinkTable
+spannerTestDataCreateExistingSinkTableQueriesFile=testdata/SpannerCreateExistingSinkTableQueries.txt
 spannerQuery=Select * from EmployeeDetails where AgeInYears < 40
 spannerCountQuery=Select count(*) from EmployeeDetails where AgeInYears < 40
 spannerSourceSchema=[{"key":"EmployeeDepartment","value":"string"},{"key":"EmployeeName","value":"string"},{"key":"Salary","value":"long"},\

src/e2e-test/resources/testdata/SpannerCreateExistingSinkTableQueries.txt

@@ -0,0 +1 @@
+CREATE TABLE ExistingSinkTable ( EmployeeDepartment STRING(1024), EmployeeName STRING(1024), Salary INT64, Workhours INT64, DateOfBirth DATE, AgeInYears FLOAT64, IsActive BOOL, InTime TIMESTAMP, Punch BYTES(25), Activities ARRAY<STRING(4)> , Numbers ARRAY<INT64>, Attendance ARRAY<BOOL>, FloatNumbers ARRAY<FLOAT64>, BytesArray ARRAY<BYTES(25)>, DateArray ARRAY<DATE>, TimestampArray ARRAY<TIMESTAMP>) PRIMARY KEY(EmployeeDepartment, EmployeeName)