From b0dc520dc26e25ffe49541483d539524355d6dda Mon Sep 17 00:00:00 2001 From: priyabhatnagar Date: Mon, 21 Aug 2023 19:07:25 +0530 Subject: [PATCH] gcscopy e2e tests review comments for validation --- .github/workflows/e2e.yml | 2 +- pom.xml | 1 + src/e2e-test/features/gcscopy/GCSCopy.feature | 153 ++++++++++++++++ .../gcscopy/GCSCopyErrorScenarios.feature | 60 +++++++ .../gcscopy/GCSCopy_WithMacro.feature | 41 +++++ src/e2e-test/features/gcsmove/GCSMove.feature | 4 +- .../plugin/gcs/stepsdesign/package-info.java | 2 +- .../gcscopy/actions/GCSCopyActions.java | 42 +++++ .../plugin/gcscopy/actions/package-info.java | 4 + .../gcscopy/locators/GCSCopyLocators.java | 37 ++++ .../plugin/gcscopy/locators/package-info.java | 4 + .../plugin/gcscopy/runners/TestRunner.java | 37 ++++ .../gcscopy/runners/TestRunnerRequired.java | 37 ++++ .../plugin/gcscopy/runners/package-info.java | 4 + .../plugin/gcscopy/stepsdesign/GCSCopy.java | 166 ++++++++++++++++++ .../stepsdesign/GCSCopyValidation.java | 101 +++++++++++ .../gcscopy/stepsdesign/package-info.java | 4 + .../plugin/gcsmove/runners/TestRunner.java | 3 +- .../gcsmove/runners/TestRunnerRequired.java | 1 - .../resources/errorMessage.properties | 3 + .../resources/pluginParameters.properties | 13 +- .../resources/testdata/GCSCOPY_CSV_TEST.csv | 11 ++ 22 files changed, 722 insertions(+), 8 deletions(-) create mode 100644 src/e2e-test/features/gcscopy/GCSCopy.feature create mode 100644 src/e2e-test/features/gcscopy/GCSCopyErrorScenarios.feature create mode 100644 src/e2e-test/features/gcscopy/GCSCopy_WithMacro.feature create mode 100644 src/e2e-test/java/io/cdap/plugin/gcscopy/actions/GCSCopyActions.java create mode 100644 src/e2e-test/java/io/cdap/plugin/gcscopy/actions/package-info.java create mode 100644 src/e2e-test/java/io/cdap/plugin/gcscopy/locators/GCSCopyLocators.java create mode 100644 src/e2e-test/java/io/cdap/plugin/gcscopy/locators/package-info.java create mode 100644 src/e2e-test/java/io/cdap/plugin/gcscopy/runners/TestRunner.java create mode 100644 src/e2e-test/java/io/cdap/plugin/gcscopy/runners/TestRunnerRequired.java create mode 100644 src/e2e-test/java/io/cdap/plugin/gcscopy/runners/package-info.java create mode 100644 src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopy.java create mode 100644 src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopyValidation.java create mode 100644 src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/package-info.java create mode 100644 src/e2e-test/resources/testdata/GCSCOPY_CSV_TEST.csv diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 619b83d76f..1db5015306 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -40,7 +40,7 @@ jobs: ) strategy: matrix: - tests: [bigquery, common, gcs, pubsub, spanner, gcscreate, gcsdelete, gcsmove, bigqueryexecute] + tests: [bigquery, common, gcs, pubsub, spanner, gcscreate, gcsdelete, gcsmove, bigqueryexecute, gcscopy] fail-fast: false steps: # Pinned 1.0.0 version diff --git a/pom.xml b/pom.xml index 92ff32c99c..d9a1db5fae 100644 --- a/pom.xml +++ b/pom.xml @@ -1157,6 +1157,7 @@ integration-test + verify diff --git a/src/e2e-test/features/gcscopy/GCSCopy.feature b/src/e2e-test/features/gcscopy/GCSCopy.feature new file mode 100644 index 0000000000..5145dece39 --- /dev/null +++ b/src/e2e-test/features/gcscopy/GCSCopy.feature @@ -0,0 +1,153 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@GCSCopy +Feature:GCSCopy - Verification of successful objects copy from one bucket to another + + @CMEK @GCS_CSV_TEST @GCS_SINK_TEST + Scenario:Validate successful copy object from one bucket to another new bucket along with data validation. + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Conditions and Actions" + When Select plugin: "GCS Copy" from the plugins list as: "Conditions and Actions" + When Navigate to the properties page of plugin: "GCS Copy" + And Replace input plugin property: "project" with value: "projectId" + And Enter GCSCopy property source path "gcsCsvFile" + And Enter GCSCopy property destination path + Then Override Service account details if set in environment variables + Then Enter GCSCopy property encryption key name "cmekGCS" if cmek is enabled + Then Validate "GCS Copy" plugin properties + Then Close the Plugin Properties page + Then Save and Deploy Pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate GCSCopy successfully copies object "gcsCsvFile" to destination bucket + Then Validate the data of GCS Copy source bucket and destination bucket "gcsCopyCsvExpectedFilePath" + + @GCS_READ_RECURSIVE_TEST @GCS_SINK_TEST @GCSCopy_Required + Scenario: Validate successful copy objects from one bucket to another with Copy All Subdirectories set to true along with data validation. + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Conditions and Actions" + When Select plugin: "GCS Copy" from the plugins list as: "Conditions and Actions" + When Navigate to the properties page of plugin: "GCS Copy" + And Replace input plugin property: "project" with value: "projectId" + And Enter GCSCopy property source path "gcsCopyReadRecursivePath" + And Enter GCSCopy property destination path + Then Override Service account details if set in environment variables + Then Enter GCSCopy property encryption key name "cmekGCS" if cmek is enabled + Then Select radio button plugin property: "recursive" with value: "true" + Then Validate "GCS Copy" plugin properties + Then Close the Plugin Properties page + Then Save and Deploy Pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate GCSCopy successfully copies object "gcsCopyReadRecursivePath" to destination bucket + Then Validate the data of GCS Copy source bucket and destination bucket "gcsCopyRecursivePath" + + @GCS_READ_RECURSIVE_TEST @GCS_SINK_TEST @GCSCopy_Required + Scenario: Validate successful copy objects from one bucket to another with Copy All Subdirectories set to false along with data validation. + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Conditions and Actions" + When Select plugin: "GCS Copy" from the plugins list as: "Conditions and Actions" + When Navigate to the properties page of plugin: "GCS Copy" + And Replace input plugin property: "project" with value: "projectId" + And Enter GCSCopy property source path "gcsCopyReadRecursivePath" + And Enter GCSCopy property destination path + Then Override Service account details if set in environment variables + Then Enter GCSCopy property encryption key name "cmekGCS" if cmek is enabled + Then Select radio button plugin property: "recursive" with value: "false" + Then Validate "GCS Copy" plugin properties + Then Close the Plugin Properties page + Then Save and Deploy Pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate GCSCopy did not copy subdirectory "gcsCopyReadRecursiveSubDirectory" to destination bucket + Then Validate the data of GCS Copy source bucket and destination bucket "gcsCopyRecursivePath" + + @GCS_CSV_TEST @GCS_SINK_EXISTING_BUCKET_TEST @GCSCopy_Required + Scenario: Validate successful copy objects from one bucket to another existing bucket with Overwrite Existing Files set to true along with data validation. + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Conditions and Actions" + When Select plugin: "GCS Copy" from the plugins list as: "Conditions and Actions" + When Navigate to the properties page of plugin: "GCS Copy" + And Replace input plugin property: "project" with value: "projectId" + And Enter GCSCopy property source path "gcsCsvFile" + Then Enter GCSCopy property destination path "gcsCopyReadRecursivePath" + Then Override Service account details if set in environment variables + Then Enter GCSCopy property encryption key name "cmekGCS" if cmek is enabled + Then Select radio button plugin property: "overwrite" with value: "true" + Then Validate "GCS Copy" plugin properties + Then Close the Plugin Properties page + Then Save and Deploy Pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate GCSCopy successfully copies object "gcsCsvFile" to destination bucket + Then Validate the data of GCS Copy source bucket and destination bucket "gcsCopyCsvExpectedFilePath" + + @GCS_CSV_TEST @GCS_SINK_EXISTING_BUCKET_TEST + Scenario: Validate successful copy objects from one bucket to another existing bucket with Overwrite Existing Files set to false along with data validation. + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Conditions and Actions" + When Select plugin: "GCS Copy" from the plugins list as: "Conditions and Actions" + When Navigate to the properties page of plugin: "GCS Copy" + And Replace input plugin property: "project" with value: "projectId" + And Enter GCSCopy property source path "gcsCsvFile" + Then Enter GCSCopy property destination path "gcsCopyReadRecursivePath" + Then Override Service account details if set in environment variables + Then Enter GCSCopy property encryption key name "cmekGCS" if cmek is enabled + Then Select radio button plugin property: "overwrite" with value: "false" + Then Validate "GCS Copy" plugin properties + Then Close the Plugin Properties page + Then Save and Deploy Pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + And Verify the pipeline status is "Failed" + Then Close the pipeline logs + Then Validate GCSCopy failed to copy object "gcsCsvFile" to destination bucket + Then Validate the data of GCS Copy source bucket and destination bucket "gcsCopyCsvExpectedFilePath" + + @GCS_CSV_TEST @GCS_SINK_TEST + Scenario:Validate successful Copy object from one bucket to another new bucket with location set to non-default value + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Conditions and Actions" + When Select plugin: "GCS Copy" from the plugins list as: "Conditions and Actions" + When Navigate to the properties page of plugin: "GCS Copy" + And Replace input plugin property: "project" with value: "projectId" + And Enter GCSCopy property source path "gcsCsvFile" + And Enter GCSCopy property destination path + Then Override Service account details if set in environment variables + Then Replace input plugin property: "location" with value: "locationEU" + Then Enter GCSCopy property encryption key name "cmekGCS" if cmek is enabled + Then Validate "GCS Copy" plugin properties + Then Close the Plugin Properties page + Then Save and Deploy Pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate GCSCopy successfully copied object "gcsCsvFile" to destination bucket in location "locationEU" + Then Validate the data of GCS Copy source bucket and destination bucket "gcsCopyCsvExpectedFilePath" \ No newline at end of file diff --git a/src/e2e-test/features/gcscopy/GCSCopyErrorScenarios.feature b/src/e2e-test/features/gcscopy/GCSCopyErrorScenarios.feature new file mode 100644 index 0000000000..5ce468b81f --- /dev/null +++ b/src/e2e-test/features/gcscopy/GCSCopyErrorScenarios.feature @@ -0,0 +1,60 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@GCSCopy +Feature: GCSCopy - Validate GCSCopy plugin error scenarios + + Scenario:Verify GCSCopy plugin properties validation errors for mandatory fields + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Conditions and Actions" + When Select plugin: "GCS Copy" from the plugins list as: "Conditions and Actions" + When Navigate to the properties page of plugin: "GCS Copy" + Then Click on the Validate button + Then Verify mandatory property error for below listed properties: + | sourcePath | + | destPath | + + @GCS_SINK_TEST + Scenario:Verify GCSCopy plugin error message for invalid bucket name in Source Path + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Conditions and Actions" + When Select plugin: "GCS Copy" from the plugins list as: "Conditions and Actions" + When Navigate to the properties page of plugin: "GCS Copy" + And Enter input plugin property: "sourcePath" with value: "invalidsourcePath" + And Enter input plugin property: "destPath" with value: "gcsTargetBucketName" + Then Click on the Validate button + Then Verify that the Plugin Property: "sourcePath" is displaying an in-line error message: "errorMessageInvalidSourcePath" + + @GCS_CSV_TEST + Scenario:Verify GCSCopy plugin error message for invalid bucket name in Destination Path + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Conditions and Actions" + When Select plugin: "GCS Copy" from the plugins list as: "Conditions and Actions" + When Navigate to the properties page of plugin: "GCS Copy" + And Enter input plugin property: "sourcePath" with value: "gcsCsvFile" + And Enter input plugin property: "destPath" with value: "invaliddestPath" + Then Click on the Validate button + Then Verify that the Plugin Property: "destPath" is displaying an in-line error message: "errorMessageInvalidDestPath" + + @GCS_CSV_TEST @GCS_SINK_TEST + Scenario:Verify GCSCopy plugin error message for invalid Encryption Key Name + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Conditions and Actions" + When Select plugin: "GCS Copy" from the plugins list as: "Conditions and Actions" + When Navigate to the properties page of plugin: "GCS Copy" + And Enter input plugin property: "sourcePath" with value: "gcsCsvFile" + And Enter input plugin property: "destPath" with value: "gcsTargetBucketName" + And Enter input plugin property: "cmekKey" with value: "invalidEncryptionKey" + Then Click on the Validate button + Then Verify that the Plugin Property: "cmekKey" is displaying an in-line error message: "errorMessageInvalidEncryptionKey" diff --git a/src/e2e-test/features/gcscopy/GCSCopy_WithMacro.feature b/src/e2e-test/features/gcscopy/GCSCopy_WithMacro.feature new file mode 100644 index 0000000000..e33faa48c5 --- /dev/null +++ b/src/e2e-test/features/gcscopy/GCSCopy_WithMacro.feature @@ -0,0 +1,41 @@ +# Copyright © 2023 Cask Data, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +@GCSCopy +Feature:GCSCopy - Verification of successful objects copy from one bucket to another with macro arguments + + @CMEK @GCS_CSV_TEST @GCS_SINK_TEST + Scenario:Validate successful copy object from one bucket to another new bucket with macro arguments + Given Open Datafusion Project to configure pipeline + When Expand Plugin group in the LHS plugins list: "Conditions and Actions" + When Select plugin: "GCS Copy" from the plugins list as: "Conditions and Actions" + When Navigate to the properties page of plugin: "GCS Copy" + Then Click on the Macro button of Property: "project" and set the value to: "projectId" + Then Click on the Macro button of Property: "sourcePath" and set the value to: "SourcePath" + Then Click on the Macro button of Property: "destPath" and set the value to: "DestPath" + Then Override Service account details if set in environment variables + Then Enter GCSCopy property encryption key name "cmekGCS" if cmek is enabled + Then Validate "GCS Copy" plugin properties + Then Close the Plugin Properties page + Then Save and Deploy Pipeline + Then Run the Pipeline in Runtime + Then Enter runtime argument value "projectId" for key "projectId" + Then Enter runtime argument value "gcsCsvFile" for GCSCopy property sourcePath key "SourcePath" + Then Enter runtime argument value for GCSCopy property destination path key "DestPath" + Then Run the Pipeline in Runtime with runtime arguments + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate GCSCopy successfully copies object "gcsCsvFile" to destination bucket diff --git a/src/e2e-test/features/gcsmove/GCSMove.feature b/src/e2e-test/features/gcsmove/GCSMove.feature index 57e2152795..e79e572483 100644 --- a/src/e2e-test/features/gcsmove/GCSMove.feature +++ b/src/e2e-test/features/gcsmove/GCSMove.feature @@ -82,7 +82,7 @@ Feature:GCSMove - Verification of successful objects move from one bucket to ano Then Verify the pipeline status is "Succeeded" Then Validate GCSMove successfully moved object "gcsCsvFile" to destination bucket - @GCS_CSV_TEST @GCS_SINK_EXISTING_BUCKET_TEST @PLUGIN-1134 + @GCS_CSV_TEST @GCS_SINK_EXISTING_BUCKET_TEST Scenario:Validate successful move objects from one bucket to another existing bucket with Overwrite Existing Files set to false Given Open Datafusion Project to configure pipeline When Expand Plugin group in the LHS plugins list: "Conditions and Actions" @@ -99,7 +99,7 @@ Feature:GCSMove - Verification of successful objects move from one bucket to ano Then Run the Pipeline in Runtime Then Wait till pipeline is in running state Then Open and capture logs - Then Verify the pipeline status is "failed" + And Verify the pipeline status is "Failed" Then Validate GCSMove failed to move object "gcsCsvFile" to destination bucket @GCS_CSV_TEST @GCS_SINK_TEST diff --git a/src/e2e-test/java/io/cdap/plugin/gcs/stepsdesign/package-info.java b/src/e2e-test/java/io/cdap/plugin/gcs/stepsdesign/package-info.java index e4b6b888e1..a886730d38 100644 --- a/src/e2e-test/java/io/cdap/plugin/gcs/stepsdesign/package-info.java +++ b/src/e2e-test/java/io/cdap/plugin/gcs/stepsdesign/package-info.java @@ -1,4 +1,4 @@ /** - * Package contains the stepDesign for the GCS features. + * Package contains the step definitions for the GCS plugin. */ package io.cdap.plugin.gcs.stepsdesign; diff --git a/src/e2e-test/java/io/cdap/plugin/gcscopy/actions/GCSCopyActions.java b/src/e2e-test/java/io/cdap/plugin/gcscopy/actions/GCSCopyActions.java new file mode 100644 index 0000000000..e216e582c8 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/gcscopy/actions/GCSCopyActions.java @@ -0,0 +1,42 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.gcscopy.actions; + +import io.cdap.e2e.utils.ElementHelper; +import io.cdap.e2e.utils.SeleniumHelper; +import io.cdap.plugin.gcscopy.locators.GCSCopyLocators; + +/** + * GCS Copy plugin related actions. + */ + +public class GCSCopyActions { + static { + SeleniumHelper.getPropertiesLocators(GCSCopyLocators.class); + } + public static void enterSourcePath(String sourcePath) { + ElementHelper.replaceElementValue(GCSCopyLocators.gcsCopySourcePath, sourcePath); + } + + public static void enterDestinationPath(String destinationPath) { + ElementHelper.replaceElementValue(GCSCopyLocators.gcsCopyDestinationPath, destinationPath); + } + + public static void enterEncryptionKeyName(String cmek) { + GCSCopyLocators.gcsCopyEncryptionKey.sendKeys(cmek); + } +} diff --git a/src/e2e-test/java/io/cdap/plugin/gcscopy/actions/package-info.java b/src/e2e-test/java/io/cdap/plugin/gcscopy/actions/package-info.java new file mode 100644 index 0000000000..96c6070452 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/gcscopy/actions/package-info.java @@ -0,0 +1,4 @@ +/** + * Package contains the actions for the GCS Copy plugin. + */ +package io.cdap.plugin.gcscopy.actions; diff --git a/src/e2e-test/java/io/cdap/plugin/gcscopy/locators/GCSCopyLocators.java b/src/e2e-test/java/io/cdap/plugin/gcscopy/locators/GCSCopyLocators.java new file mode 100644 index 0000000000..79b87cf173 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/gcscopy/locators/GCSCopyLocators.java @@ -0,0 +1,37 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.gcscopy.locators; + +import org.openqa.selenium.WebElement; +import org.openqa.selenium.support.FindBy; +import org.openqa.selenium.support.How; + +/** + * GCS Copy plugin related locators. + */ +public class GCSCopyLocators { + + @FindBy(how = How.XPATH, using = "//input[@data-testid='sourcePath']") + public static WebElement gcsCopySourcePath; + + @FindBy(how = How.XPATH, using = "//input[@data-testid='destPath']") + public static WebElement gcsCopyDestinationPath; + + @FindBy(how = How.XPATH, using = "//input[@data-testid='cmekKey']") + public static WebElement gcsCopyEncryptionKey; + +} diff --git a/src/e2e-test/java/io/cdap/plugin/gcscopy/locators/package-info.java b/src/e2e-test/java/io/cdap/plugin/gcscopy/locators/package-info.java new file mode 100644 index 0000000000..c8cb778988 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/gcscopy/locators/package-info.java @@ -0,0 +1,4 @@ +/** + * Package contains the locators for the GCS Copy plugin. + */ +package io.cdap.plugin.gcscopy.locators; diff --git a/src/e2e-test/java/io/cdap/plugin/gcscopy/runners/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/gcscopy/runners/TestRunner.java new file mode 100644 index 0000000000..91ae566df9 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/gcscopy/runners/TestRunner.java @@ -0,0 +1,37 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.gcscopy.runners; + +import io.cucumber.junit.Cucumber; +import io.cucumber.junit.CucumberOptions; +import org.junit.runner.RunWith; + +/** + * Test Runner to execute GCSCopy test cases. + */ + @RunWith(Cucumber.class) + @CucumberOptions( + features = {"src/e2e-test/features"}, + glue = {"io.cdap.plugin.gcscopy.stepsdesign", "stepsdesign" , "io.cdap.plugin.common.stepsdesign"}, + tags = {"@GCSCopy"}, + monochrome = true, + plugin = {"pretty", "html:target/cucumber-html-report/gcscopy-action", + "json:target/cucumber-reports/cucumber-gcscopy-action.json", + "junit:target/cucumber-reports/cucumber-gcscopy-action.xml"} + ) + public class TestRunner { + } diff --git a/src/e2e-test/java/io/cdap/plugin/gcscopy/runners/TestRunnerRequired.java b/src/e2e-test/java/io/cdap/plugin/gcscopy/runners/TestRunnerRequired.java new file mode 100644 index 0000000000..d0a44c1f7e --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/gcscopy/runners/TestRunnerRequired.java @@ -0,0 +1,37 @@ +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.gcscopy.runners; + +import io.cucumber.junit.Cucumber; +import io.cucumber.junit.CucumberOptions; +import org.junit.runner.RunWith; + +/** + * Test Runner to execute only GCSCopy test cases. + */ +@RunWith(Cucumber.class) +@CucumberOptions( + features = {"src/e2e-test/features"}, + glue = {"io.cdap.plugin.gcscopy.stepsdesign", "stepsdesign", "io.cdap.plugin.common.stepsdesign"}, + tags = {"@GCSCopy_Required"}, + monochrome = true, + plugin = {"pretty", "html:target/cucumber-html-report/gcscopy-action", + "json:target/cucumber-reports/cucumber-gcscopy-action.json", + "junit:target/cucumber-reports/cucumber-gcscopy-action.xml"} +) +public class TestRunnerRequired { +} diff --git a/src/e2e-test/java/io/cdap/plugin/gcscopy/runners/package-info.java b/src/e2e-test/java/io/cdap/plugin/gcscopy/runners/package-info.java new file mode 100644 index 0000000000..6a95f2a483 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/gcscopy/runners/package-info.java @@ -0,0 +1,4 @@ +/** + * Package contains the test runner for the GCS Copy plugin. + */ +package io.cdap.plugin.gcscopy.runners; diff --git a/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopy.java b/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopy.java new file mode 100644 index 0000000000..4525ecfcaf --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopy.java @@ -0,0 +1,166 @@ + +/* + * Copyright © 2023 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.gcscopy.stepsdesign; + +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.StorageException; +import io.cdap.e2e.pages.locators.CdfStudioLocators; +import io.cdap.e2e.utils.ElementHelper; +import io.cdap.e2e.utils.PluginPropertyUtils; +import io.cdap.e2e.utils.StorageClient; +import io.cdap.plugin.common.stepsdesign.TestSetupHooks; +import io.cdap.plugin.gcscopy.actions.GCSCopyActions; +import io.cucumber.java.en.Then; +import org.junit.Assert; +import stepsdesign.BeforeActions; + +import java.io.IOException; +import java.net.URISyntaxException; + +/** + * GCS Copy plugin related stepsdesign. + */ + +public class GCSCopy { + + @Then("Validate GCSCopy successfully copies object {string} to destination bucket") + public void validateGCSCopySuccessfullyCopiedObjectToDestinationBucket(String path) throws IOException { + String sourceGCSBucket = TestSetupHooks.gcsSourceBucketName; + String gcsObject = PluginPropertyUtils.pluginProp(path); + System.out.println(gcsObject + "Name of the gcsObject"); + boolean isPresentAtSource = false; + for (Blob blob : StorageClient.listObjects(sourceGCSBucket).iterateAll()) { + System.out.println(blob.getName()); + if (blob.getName().contains(gcsObject)) { + isPresentAtSource = true; + break; + } + } + if (!isPresentAtSource) { + + Assert.fail("Object is not present in source bucket" + sourceGCSBucket); + } + BeforeActions.scenario.write("Object is copied from source GCS Bucket " + sourceGCSBucket + " successfully"); + String targetGCSBucket = TestSetupHooks.gcsTargetBucketName; + try { + for (Blob blob : StorageClient.listObjects(targetGCSBucket).iterateAll()) { + if (blob.getName().contains(gcsObject)) { + BeforeActions.scenario.write("Object copied to gcs bucket " + targetGCSBucket + " successfully"); + return; + } + } + Assert.fail("Object not copied to target gcs bucket " + targetGCSBucket); + } catch (StorageException | IOException e) { + if (e.getMessage().contains("The specified bucket does not exist")) { + Assert.fail("Target gcs bucket " + targetGCSBucket + " not created - " + e.getMessage()); + } else { + Assert.fail(e.getMessage()); + } + } + } + + @Then("Enter GCSCopy property encryption key name {string} if cmek is enabled") + public void enterGCSCopyPropertyEncryptionKeyNameIfCmekIsEnabled(String cmek) { + String cmekGCS = PluginPropertyUtils.pluginProp(cmek); + if (cmekGCS != null) { + GCSCopyActions.enterEncryptionKeyName(cmekGCS); + BeforeActions.scenario.write("Entered encryption key name - " + cmekGCS); + } + } + + @Then("Enter GCSCopy property source path {string}") + public void enterGCSCopyPropertySourcePath(String path) { + GCSCopyActions.enterSourcePath("gs://" + TestSetupHooks.gcsSourceBucketName + + "/" + PluginPropertyUtils.pluginProp(path)); + } + + @Then("Enter GCSCopy property destination path") + public void enterGCSCopyPropertyDestinationPath() { + GCSCopyActions.enterDestinationPath("gs://" + TestSetupHooks.gcsTargetBucketName); + } + + @Then("Enter GCSCopy property destination path {string}") + public void enterGCSCopyPropertyDestinationPath(String path) { + GCSCopyActions.enterDestinationPath("gs://" + TestSetupHooks.gcsTargetBucketName + + "/" + PluginPropertyUtils.pluginProp(path)); + } + + @Then("Validate GCSCopy successfully copied object {string} to destination bucket in location {string}") + public void validateGCSCopySuccessfullyCopiedObjectToDestinationBucketInLocation(String path, String location) { + String targetGCSBucket = TestSetupHooks.gcsTargetBucketName; + try { + if (StorageClient.getBucketMetadata(targetGCSBucket).getLocation().equalsIgnoreCase(PluginPropertyUtils.pluginProp + (location))) { + validateGCSCopySuccessfullyCopiedObjectToDestinationBucket(path); + return; + } + Assert.fail("Target gcs bucket " + targetGCSBucket + " is not created in location " + location); + } catch (StorageException | IOException e) { + if (e.getMessage().contains("The specified bucket does not exist")) { + Assert.fail("Target gcs bucket " + targetGCSBucket + " not created - " + e.getMessage()); + } else { + Assert.fail(e.getMessage()); + } + } + } + + @Then("Enter runtime argument value {string} for GCSCopy property sourcePath key {string}") + public void enterRuntimeArgumentValueForGCSCopyPropertySourcePathKey(String value, String runtimeArgumentKey) { + ElementHelper.sendKeys(CdfStudioLocators.runtimeArgsValue(runtimeArgumentKey), + "gs://" + TestSetupHooks.gcsSourceBucketName + "/" + PluginPropertyUtils.pluginProp(value)); + } + + @Then("Enter runtime argument value for GCSCopy property destination path key {string}") + public void enterRuntimeArgumentValueForGCSCopyPropertyDestinationPathKey(String runtimeArgumentKey) { + ElementHelper.sendKeys(CdfStudioLocators.runtimeArgsValue(runtimeArgumentKey), + "gs://" + TestSetupHooks.gcsTargetBucketName); + } + + @Then("Validate GCSCopy did not copy subdirectory {string} to destination bucket") + public void validateGCSCopyDidNotCopySubdirectoryToDestinationBucket(String subdirectory) throws IOException { + String sourceGCSBucket = TestSetupHooks.gcsSourceBucketName; + String gcsObject = PluginPropertyUtils.pluginProp(subdirectory); + for (Blob blob : StorageClient.listObjects(sourceGCSBucket).iterateAll()) { + if (blob.getName().contains(gcsObject)) { + + BeforeActions.scenario.write("Subdirectory is not copied from source bucket" + sourceGCSBucket); + return; + } + } + Assert.fail("Subdirectory is copied from source GCS Bucket " + sourceGCSBucket); + } + + @Then("Validate GCSCopy failed to copy object {string} to destination bucket") + public void validateGCSCopyFailedToCopyObjectToDestinationBucket(String path) throws IOException { + String sourceGCSBucket = TestSetupHooks.gcsSourceBucketName; + String gcsObject = PluginPropertyUtils.pluginProp(path); + for (Blob blob : StorageClient.listObjects(sourceGCSBucket).iterateAll()) { + if (blob.getName().equals(gcsObject)) { + BeforeActions.scenario.write("Object is not copied from source bucket" + sourceGCSBucket); + return; + } + } + Assert.fail("Object is deleted from source GCS Bucket " + sourceGCSBucket); + } + + @Then("Validate the data of GCS Copy source bucket and destination bucket {string}") + public void validateTheDataFromGCSSourceToGCSSinkWithExpectedCsvFileAndTargetGCSBucket(String path) { + GCSCopyValidation.validateGCSSourceToGCSSinkWithCSVFormat(TestSetupHooks.gcsTargetBucketName, + PluginPropertyUtils.pluginProp(path)); + } +} diff --git a/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopyValidation.java b/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopyValidation.java new file mode 100644 index 0000000000..b4e9403bc9 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopyValidation.java @@ -0,0 +1,101 @@ +package io.cdap.plugin.gcscopy.stepsdesign; + +import au.com.bytecode.opencsv.CSVReader; +import com.google.api.gax.paging.Page; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.gson.JsonObject; +import io.cdap.e2e.utils.PluginPropertyUtils; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +/** + * Package contains the Validation for the GCSCopy plugin. + */ +public class GCSCopyValidation { + + private static final String projectId = PluginPropertyUtils.pluginProp("projectId"); +// private static final String csvFilePath = PluginPropertyUtils.pluginProp("gcsCopyCsvExpectedFilePath"); + + public static boolean validateGCSSourceToGCSSinkWithCSVFormat(String bucketName, String filepath) { + Map expectedCSVData = readCsvAndConvertToJson(filepath); + Map actualGcsCsvData = listBucketObjects(bucketName); + + boolean isMatched = actualGcsCsvData.equals(expectedCSVData); + + return isMatched; + } + + public static Map readCsvAndConvertToJson(String filepath) { + Map csvDataMap = new HashMap<>(); + try (CSVReader csvReader = new CSVReader(new java.io.FileReader(filepath))) { + // Read the header line to get column names + String[] headers = csvReader.readNext(); + String[] line; + while ((line = csvReader.readNext()) != null) { + JsonObject jsonObject = new JsonObject(); + + for (int j = 0; j < headers.length; j++) { + jsonObject.addProperty(headers[j], line[j]); + } + String id = line[0]; + csvDataMap.put(id, jsonObject); + } + } catch (IOException e) { + e.printStackTrace(); + } + return csvDataMap; + } + + public static Map listBucketObjects(String bucketName) { + Map bucketObjectData = new HashMap<>(); + Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService(); + Page blobs = storage.list(bucketName); + + List bucketObjects = StreamSupport.stream(blobs.iterateAll().spliterator(), true) + .filter(blob -> blob.getSize() != 0) + .collect(Collectors.toList()); + + Stream objectNamesWithData = bucketObjects.stream().map(blob -> blob.getName()); + List bucketObjectNames = objectNamesWithData.collect(Collectors.toList()); + + if (!bucketObjectNames.isEmpty()) { + String objectName = bucketObjectNames.get(0); + if (objectName.contains("part-r")) { + Map dataMap2 = fetchObjectData(projectId, bucketName, objectName); + bucketObjectData.putAll(dataMap2); + } + } + + return bucketObjectData; + } + + public static Map fetchObjectData(String projectId, String bucketName, String objectName) { + Map dataMap = new HashMap<>(); + Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService(); + byte[] objectData = storage.readAllBytes(bucketName, objectName); + String objectDataAsString = new String(objectData, StandardCharsets.UTF_8); + String[] lines = objectDataAsString.split("\n"); + String[] headers = lines[0].split(","); + + for (int i = 1; i < lines.length; i++) { + String[] values = lines[i].split(","); + JsonObject jsonObject = new JsonObject(); + for (int j = 0; j < headers.length; j++) { + jsonObject.addProperty(headers[j], values[j]); + } + String id = values[0]; + dataMap.put(id, jsonObject); + } + return dataMap; + } + +} diff --git a/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/package-info.java b/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/package-info.java new file mode 100644 index 0000000000..8085a97fbf --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/package-info.java @@ -0,0 +1,4 @@ +/** + * Package contains the step definitions for the GCS Copy plugin. + */ +package io.cdap.plugin.gcscopy.stepsdesign; diff --git a/src/e2e-test/java/io/cdap/plugin/gcsmove/runners/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/gcsmove/runners/TestRunner.java index 2088ccebad..0cfc443c58 100644 --- a/src/e2e-test/java/io/cdap/plugin/gcsmove/runners/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/gcsmove/runners/TestRunner.java @@ -27,8 +27,7 @@ features = {"src/e2e-test/features"}, glue = {"io.cdap.plugin.gcsmove.stepsdesign", "io.cdap.plugin.gcs.stepsdesign", "stepsdesign", "io.cdap.plugin.common.stepsdesign"}, - tags = {"@GCSMove and not @PLUGIN-1134"}, - //TODO: Enable test once issue is fixed https://cdap.atlassian.net/browse/PLUGIN-1134 + tags = {"@GCSMove"}, monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/gcsmove-action", "json:target/cucumber-reports/cucumber-gcsmove-action.json", diff --git a/src/e2e-test/java/io/cdap/plugin/gcsmove/runners/TestRunnerRequired.java b/src/e2e-test/java/io/cdap/plugin/gcsmove/runners/TestRunnerRequired.java index 6d3989ac7f..05180a5ce4 100644 --- a/src/e2e-test/java/io/cdap/plugin/gcsmove/runners/TestRunnerRequired.java +++ b/src/e2e-test/java/io/cdap/plugin/gcsmove/runners/TestRunnerRequired.java @@ -28,7 +28,6 @@ glue = {"io.cdap.plugin.gcsmove.stepsdesign", "io.cdap.plugin.gcs.stepsdesign", "stepsdesign", "io.cdap.plugin.common.stepsdesign"}, tags = {"@GCSMove_Required"}, - //TODO: Enable test once issue is fixed https://cdap.atlassian.net/browse/PLUGIN-1134 monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report/gcsmove-action-required", "json:target/cucumber-reports/cucumber-gcsmove-action-required.json", diff --git a/src/e2e-test/resources/errorMessage.properties b/src/e2e-test/resources/errorMessage.properties index 9a45c144fe..e84e28fe6c 100644 --- a/src/e2e-test/resources/errorMessage.properties +++ b/src/e2e-test/resources/errorMessage.properties @@ -26,4 +26,7 @@ errorMessageInvalidReferenceName=Invalid reference name errorMessageInvalidBucketName=Invalid bucket name in path errorMessageInvalidFormat=Input has multi-level structure that cannot be represented appropriately as csv. \ Consider using json, avro or parquet to write data. +errorMessageInvalidSourcePath=Invalid bucket name in path 'abc@'. Bucket name should +errorMessageInvalidDestPath=Invalid bucket name in path 'abc@'. Bucket name should +errorMessageInvalidEncryptionKey=CryptoKeyName.parse: formattedString not in valid format: Parameter "abc@" must be diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index 0e680fd5b2..7caede873d 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -245,13 +245,24 @@ spannerSourceBasicSchema=[{"key":"EmployeeDepartment","value":"string"},{"key":" ## GCSMove-PLUGIN-PROPERTIES-START gcsMoveMandatoryProperties=sourcePath,destPath gcsMoveValidGcsPath=gs://valid-bucket-format -gcsMoveReadRecursivePath=testdata +gcsMoveReadRecursivePath=testdata/ gcsMoveReadRecursiveSubDirectory=testdata/GCS_RECURSIVE_TEST locationEU=eu gcsDoneSuccessFile=__SUCCESS gcsDoneFailedFile=__FAILED ## GCSMove-PLUGIN-PROPERTIES-END +## GCSCopy-PLUGIN-PROPERTIES-START +gcsCopyMandatoryProperties=sourcePath,destPath +invalidsourcePath=abc@ +invaliddestPath=abc@ +invalidEncryptionKey=abc@ +gcsCopyReadRecursivePath=testdata/ +gcsCopyReadRecursiveSubDirectory=testdata/GCS_RECURSIVE_TEST +gcsCopyCsvExpectedFilePath=src/e2e-test/resources/testdata/GCSCOPY_CSV_TEST.csv +gcsCopyRecursivePath=src/e2e-test/resources/testdata/GCS_RECURSIVE_TEST +## GCSCopy-PLUGIN-PROPERTIES-END + ## BQEXECUTE-PLUGIN-PROPERTIES-START bqExecuteQuery=SELECT * FROM `PROJECT_NAME.DATASET.TABLENAME` bqExecuteRowAsArgQuery=SELECT DISTINCT ProjectId as bqExecuteArgProjectID, Dataset as bqExecuteArgDataset FROM `PROJECT_NAME.DATASET.TABLENAME` limit 1 diff --git a/src/e2e-test/resources/testdata/GCSCOPY_CSV_TEST.csv b/src/e2e-test/resources/testdata/GCSCOPY_CSV_TEST.csv new file mode 100644 index 0000000000..24a37bf289 --- /dev/null +++ b/src/e2e-test/resources/testdata/GCSCOPY_CSV_TEST.csv @@ -0,0 +1,11 @@ +id,EmployeeDepartment,Employeename,Salary,wotkhours +1,Plumber,Surya,10000,3 +2,Plumber,Sahil,70000,5 +3,Plumber,Waugh,10000,6 +4,Plumber,Steve,10000,7 +5,Plumber,Smith,20000,8 +6,Electrician,Ragini,30000,2 +7,Electrician,Williamson,10000,9 +8,Electrician,James,80000,10 +9,Electrician,Steve Smith,10000,23 +10,Electrician,Mark Waugh,10000,32 \ No newline at end of file