From 2fbf3c05ddce98c1ce2a0116eb848448a508ea1d Mon Sep 17 00:00:00 2001 From: navdeepkumar Date: Wed, 24 Jan 2024 18:34:51 +0530 Subject: [PATCH] Spanner To Spanner Additional Cases as per ITN class. --- .../SpannerToSpanner_Additional.feature | 119 ++++++++++++++++++ .../source/SpannerToSpanner_WithMacro.feature | 55 ++++++++ .../SpannerToSpanner_withConnections.feature | 49 ++++++++ .../common/stepsdesign/TestSetupHooks.java | 29 +++++ .../spanner/stepsdesign/SpannerBase.java | 17 +++ .../resources/pluginParameters.properties | 2 + .../SpannerCreateExistingSinkTableQueries.txt | 1 + 7 files changed, 272 insertions(+) create mode 100644 src/e2e-test/features/spanner/source/SpannerToSpanner_Additional.feature create mode 100644 src/e2e-test/features/spanner/source/SpannerToSpanner_WithMacro.feature create mode 100644 src/e2e-test/resources/testdata/SpannerCreateExistingSinkTableQueries.txt diff --git a/src/e2e-test/features/spanner/source/SpannerToSpanner_Additional.feature b/src/e2e-test/features/spanner/source/SpannerToSpanner_Additional.feature new file mode 100644 index 0000000000..d77d2e7177 --- /dev/null +++ b/src/e2e-test/features/spanner/source/SpannerToSpanner_Additional.feature @@ -0,0 +1,119 @@ +@Spanner_Source +Feature: Spanner source - Verification of Additional Spanner to Spanner successful data transfer without using connections + + @SPANNER_TEST @EXISTING_SPANNER_SINK + Scenario: To verify data is getting transferred from no schema Spanner source to Spanner sink having existing table + Given Open Datafusion Project to configure pipeline + When Select plugin: "Spanner" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Spanner" from the plugins list as: "Sink" + Then Connect plugins: "Spanner" and "Spanner2" to establish connection + Then Navigate to the properties page of plugin: "Spanner" + Then Enter Spanner property reference name + Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables + Then Enter input plugin property: "instanceId" with value: "spannerInstance" + Then Enter input plugin property: "databaseName" with value: "spannerDatabase" + Then Enter input plugin property: "tableName" with value: "spannerSourceTable" + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Spanner2" + Then Enter Spanner property reference name + Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables + Then Enter input plugin property: "instanceId" with value: "spannerInstance" + Then Enter input plugin property: "databaseName" with value: "spannerDatabase" + Then Enter input plugin property: "tableName" with value: "spannerExistingTargetTable" + Then Enter Spanner sink property primary key "spannerSinkPrimaryKeySpanner" + Then Validate "Spanner" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to already existing target spanner table with record counts of source table + + @SPANNER_TEST @EXISTING_SPANNER_SINK + Scenario: To verify data is getting transferred from Spanner source to Spanner sink having existing table + Given Open Datafusion Project to configure pipeline + When Select plugin: "Spanner" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Spanner" from the plugins list as: "Sink" + Then Connect plugins: "Spanner" and "Spanner2" to establish connection + Then Navigate to the properties page of plugin: "Spanner" + Then Enter Spanner property reference name + Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables + Then Enter input plugin property: "instanceId" with value: "spannerInstance" + Then Enter input plugin property: "databaseName" with value: "spannerDatabase" + Then Enter input plugin property: "tableName" with value: "spannerSourceTable" + Then Validate output schema with expectedSchema "spannerSourceSchema" + Then Validate "Spanner" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Spanner2" + Then Enter Spanner property reference name + Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables + Then Enter input plugin property: "instanceId" with value: "spannerInstance" + Then Enter input plugin property: "databaseName" with value: "spannerDatabase" + Then Enter input plugin property: "tableName" with value: "spannerExistingTargetTable" + Then Enter Spanner sink property primary key "spannerSinkPrimaryKeySpanner" + Then Validate "Spanner" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to already existing target spanner table with record counts of source table + + @SPANNER_TEST @SPANNER_SINK_TEST + Scenario: To verify data is getting transferred from no schema Spanner source to Spanner non existing sink table + Given Open Datafusion Project to configure pipeline + When Select plugin: "Spanner" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Spanner" from the plugins list as: "Sink" + Then Connect plugins: "Spanner" and "Spanner2" to establish connection + Then Navigate to the properties page of plugin: "Spanner" + Then Enter Spanner property reference name + Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables + Then Enter input plugin property: "instanceId" with value: "spannerInstance" + Then Enter input plugin property: "databaseName" with value: "spannerDatabase" + Then Enter input plugin property: "tableName" with value: "spannerSourceTable" + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Spanner2" + Then Enter Spanner property reference name + Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables + Then Enter input plugin property: "instanceId" with value: "spannerInstance" + Then Enter input plugin property: "databaseName" with value: "spannerTargetDatabase" + Then Enter input plugin property: "tableName" with value: "spannerTargetTable" + Then Enter Spanner sink property primary key "spannerSinkPrimaryKeySpanner" + Then Validate "Spanner" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target spanner table with record counts of source spanner table + diff --git a/src/e2e-test/features/spanner/source/SpannerToSpanner_WithMacro.feature b/src/e2e-test/features/spanner/source/SpannerToSpanner_WithMacro.feature new file mode 100644 index 0000000000..5e70e46e2f --- /dev/null +++ b/src/e2e-test/features/spanner/source/SpannerToSpanner_WithMacro.feature @@ -0,0 +1,55 @@ +@Spanner_Source +Feature: Spanner source - Verification of Spanner to Spanner Successful data transfer with macro arguments + + @SPANNER_SINK_TEST @SPANNER_TEST + Scenario: To verify data is getting transferred from Spanner to Spanner with macro arguments + Given Open Datafusion Project to configure pipeline + When Select plugin: "Spanner" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Spanner" from the plugins list as: "Sink" + Then Connect plugins: "Spanner" and "Spanner2" to establish connection + Then Navigate to the properties page of plugin: "Spanner" + Then Enter Spanner property reference name + Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables + Then Click on the Macro button of Property: "instanceId" and set the value to: "macroStringInstance" + Then Click on the Macro button of Property: "databaseName" and set the value to: "macroStringDatabase" + Then Click on the Macro button of Property: "tableName" and set the value to: "macroStringSourceTable" + Then Validate "Spanner" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Spanner2" + Then Enter Spanner property reference name + Then Enter Spanner property projectId "projectId" + Then Override Service account details if set in environment variables + Then Click on the Macro button of Property: "instanceId" and set the value to: "macroStringInstance" + Then Click on the Macro button of Property: "databaseName" and set the value to: "macroStringTargetDatabase" + Then Click on the Macro button of Property: "tableName" and set the value to: "macroStringTargetTable" + Then Click on the Macro button of Property: "keys" and set the value to: "macroStringPrimaryKey" + Then Validate "Spanner" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Enter runtime argument value "spannerInstance" for key "macroStringInstance" + Then Enter runtime argument value "spannerDatabase" for key "macroStringDatabase" + Then Enter runtime argument value "spannerSourceTable" for key "macroStringSourceTable" + Then Enter runtime argument value "spannerTargetDatabase" for key "macroStringTargetDatabase" + Then Enter runtime argument value "spannerTargetTable" for key "macroStringTargetTable" + Then Enter runtime argument value "spannerSinkPrimaryKeySpanner" for key "macroStringPrimaryKey" + Then Run the preview of pipeline with runtime arguments + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Enter runtime argument value "spannerInstance" for key "macroStringInstance" + Then Enter runtime argument value "spannerDatabase" for key "macroStringDatabase" + Then Enter runtime argument value "spannerSourceTable" for key "macroStringSourceTable" + Then Enter runtime argument value "spannerTargetDatabase" for key "macroStringTargetDatabase" + Then Enter runtime argument value "spannerTargetTable" for key "macroStringTargetTable" + Then Enter runtime argument value "spannerSinkPrimaryKeySpanner" for key "macroStringPrimaryKey" + Then Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to target spanner table with record counts of source spanner table \ No newline at end of file diff --git a/src/e2e-test/features/spanner/source/SpannerToSpanner_withConnections.feature b/src/e2e-test/features/spanner/source/SpannerToSpanner_withConnections.feature index 1fba3e5fec..a0a57bdf11 100644 --- a/src/e2e-test/features/spanner/source/SpannerToSpanner_withConnections.feature +++ b/src/e2e-test/features/spanner/source/SpannerToSpanner_withConnections.feature @@ -163,3 +163,52 @@ Feature: Spanner source - Verification of Spanner to Spanner successful data tra Then Wait till pipeline is in running state Then Verify the pipeline status is "Succeeded" Then Validate records transferred to target spanner table with record counts of source spanner table + + @EXISTING_SPANNER_SINK @EXISTING_SPANNER_CONNECTION @Spanner_Source_Required + Scenario: To verify data is getting transferred from Spanner source to existing Spanner sink + Given Open Datafusion Project to configure pipeline + When Select plugin: "Spanner" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "Spanner" from the plugins list as: "Sink" + Then Connect plugins: "Spanner" and "Spanner2" to establish connection + Then Navigate to the properties page of plugin: "Spanner" + Then Click plugin property: "switch-useConnection" + Then Click on the Browse Connections button + Then Select connection: "spannerConnectionName" + Then Click on the Browse button inside plugin properties + Then Select connection data row with name: "spannerInstance" + Then Select connection data row with name: "spannerDatabase" + Then Select connection data row with name: "spannerSourceTable" + Then Wait till connection data loading completes with a timeout of 60 seconds + Then Verify input plugin property: "instanceId" contains value: "spannerInstance" + Then Verify input plugin property: "databaseName" contains value: "spannerDatabase" + Then Verify input plugin property: "tableName" contains value: "spannerSourceTable" + Then Validate output schema with expectedSchema "spannerSourceSchema" + Then Validate "Spanner" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Spanner2" + Then Enter input plugin property: "referenceName" with value: "SpannerSinkReferenceName" + Then Click plugin property: "useConnection" + Then Click on the Browse Connections button + Then Select connection: "spannerConnectionName" + Then Click on the Browse button inside plugin properties + Then Click SELECT button inside connection data row with name: "spannerInstance" + Then Wait till connection data loading completes with a timeout of 60 seconds + Then Verify input plugin property: "instanceId" contains value: "spannerInstance" + Then Enter input plugin property: "databaseName" with value: "spannerDatabase" + Then Enter input plugin property: "tableName" with value: "spannerExistingTargetTable" + Then Enter Spanner sink property primary key "spannerSinkPrimaryKeySpanner" + Then Validate "Spanner" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Verify the pipeline status is "Succeeded" + Then Validate records transferred to already existing target spanner table with record counts of source table \ No newline at end of file diff --git a/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java index 256fc5b5b7..fc92c2e6eb 100644 --- a/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java +++ b/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java @@ -69,6 +69,7 @@ public class TestSetupHooks { public static String spannerTargetTable = StringUtils.EMPTY; public static boolean firstSpannerTestFlag = true; public static String datasetName = PluginPropertyUtils.pluginProp("dataset"); + public static String spannerExistingTargetTable = StringUtils.EMPTY; @Before(order = 1) public static void overrideServiceAccountFilePathIfProvided() { @@ -1333,4 +1334,32 @@ public static void replaceTableDetailsInInsertQuerySql() { public static void setInsertQueryBackWithTableDetailsPlaceholderSql() { setQueryBackWithTableDetailsPlaceholder("bqExecuteInsert"); } + + @Before(order = 2, value = "@EXISTING_SPANNER_SINK") + public static void makeExistingTargetSpannerDBAndTableName() { + try { + spannerTargetDatabase = spannerDatabase; + spannerExistingTargetTable = PluginPropertyUtils.pluginProp("spannerExistingTargetTable"); + String createQuery = null; + try { + createQuery = new String(Files.readAllBytes(Paths.get(TestSetupHooks.class.getResource + ("/" + PluginPropertyUtils.pluginProp("spannerTestDataCreateExistingSinkTableQueriesFile")).toURI())) + , StandardCharsets.UTF_8); + + } catch (Exception e) { + BeforeActions.scenario.write("Exception in reading " + + PluginPropertyUtils.pluginProp("spannerTestDataCreateExistingSinkTableQueriesFile") + + " - " + e.getMessage()); + Assert.fail("Exception in Spanner testdata prerequisite -" + + "error in reading create existing table queries file." + + e.getMessage()); + } + SpannerClient.executeDMLQuery(spannerInstance, spannerTargetDatabase, createQuery); + PluginPropertyUtils.addPluginProp("spannerTargetDatabase", spannerTargetDatabase); + PluginPropertyUtils.addPluginProp("spannerExistingTargetTable", spannerExistingTargetTable); + } catch (Exception e) { + e.printStackTrace(); + } + } + } diff --git a/src/e2e-test/java/io/cdap/plugin/spanner/stepsdesign/SpannerBase.java b/src/e2e-test/java/io/cdap/plugin/spanner/stepsdesign/SpannerBase.java index e1d4a35b31..3ce36789ed 100644 --- a/src/e2e-test/java/io/cdap/plugin/spanner/stepsdesign/SpannerBase.java +++ b/src/e2e-test/java/io/cdap/plugin/spanner/stepsdesign/SpannerBase.java @@ -152,4 +152,21 @@ public void validateRecordsTransferredToTargetSpannerTableWithRecordCountsOfSour Assert.assertEquals(spannerSourceTableRecordCount, spannerTargetTableRecordCount); } + + @Then("Validate records transferred to already existing target spanner table with record counts of source table") + public void validateRecordsTransferredToExistingSpannerTableWithRecordCountsOfSourceSpannerTable() { + int spannerSourceTableRecordCount = SpannerClient + .getCountOfRecordsInTable(PluginPropertyUtils.pluginProp("spannerInstance"), + PluginPropertyUtils.pluginProp("spannerDatabase"), + PluginPropertyUtils.pluginProp("spannerSourceTable")); + BeforeActions.scenario.write("No of records from Spanner Source table :" + spannerSourceTableRecordCount); + int spannerTargetTableRecordCount = SpannerClient + .getCountOfRecordsInTable(PluginPropertyUtils.pluginProp("spannerInstance"), + PluginPropertyUtils.pluginProp("spannerDatabase"), + PluginPropertyUtils.pluginProp("spannerExistingTargetTable")); + BeforeActions.scenario.write("No of records transferred to existing Spanner target table:" + + spannerTargetTableRecordCount); + + Assert.assertEquals(spannerSourceTableRecordCount, spannerTargetTableRecordCount); + } } diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index 7ccc98f4bc..5f37646dc0 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -278,6 +278,8 @@ spannerTestDataInsertDataQueriesFile=testdata/SpannerInsertDataQueries.txt spannerTestDataInsertBasicDataQueriesFile=testdata/SpannerInsertEmployeeBasicDataQueries.txt spannerSourceTable=EmployeeDetails spannerSourceBasicTable=EmployeeDetails_Basic +spannerExistingTargetTable=ExistingSinkTable +spannerTestDataCreateExistingSinkTableQueriesFile=testdata/SpannerCreateExistingSinkTableQueries.txt spannerQuery=Select * from EmployeeDetails where AgeInYears < 40 spannerCountQuery=Select count(*) from EmployeeDetails where AgeInYears < 40 spannerSourceSchema=[{"key":"EmployeeDepartment","value":"string"},{"key":"EmployeeName","value":"string"},{"key":"Salary","value":"long"},\ diff --git a/src/e2e-test/resources/testdata/SpannerCreateExistingSinkTableQueries.txt b/src/e2e-test/resources/testdata/SpannerCreateExistingSinkTableQueries.txt new file mode 100644 index 0000000000..76dc3c06b7 --- /dev/null +++ b/src/e2e-test/resources/testdata/SpannerCreateExistingSinkTableQueries.txt @@ -0,0 +1 @@ +CREATE TABLE ExistingSinkTable ( EmployeeDepartment STRING(1024), EmployeeName STRING(1024), Salary INT64, Workhours INT64, DateOfBirth DATE, AgeInYears FLOAT64, IsActive BOOL, InTime TIMESTAMP, Punch BYTES(25), Activities ARRAY , Numbers ARRAY, Attendance ARRAY, FloatNumbers ARRAY, BytesArray ARRAY, DateArray ARRAY, TimestampArray ARRAY) PRIMARY KEY(EmployeeDepartment, EmployeeName) \ No newline at end of file