From b9787cba97024eb74c928ad81068d62ce1c9f6b3 Mon Sep 17 00:00:00 2001 From: priyabhatnagar Date: Mon, 11 Sep 2023 17:19:17 +0530 Subject: [PATCH] gcscopy e2e tests review comments for validation --- src/e2e-test/features/gcscopy/GCSCopy.feature | 8 +- .../plugin/gcscopy/stepsdesign/GCSCopy.java | 20 ++-- .../stepsdesign/GCSCopyValidation.java | 101 ++++++++++++++++++ .../resources/pluginParameters.properties | 2 + .../resources/testdata/GCSCOPY_CSV_TEST.csv | 11 ++ 5 files changed, 135 insertions(+), 7 deletions(-) create mode 100644 src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopyValidation.java create mode 100644 src/e2e-test/resources/testdata/GCSCOPY_CSV_TEST.csv diff --git a/src/e2e-test/features/gcscopy/GCSCopy.feature b/src/e2e-test/features/gcscopy/GCSCopy.feature index b0b96a4570..5145dece39 100644 --- a/src/e2e-test/features/gcscopy/GCSCopy.feature +++ b/src/e2e-test/features/gcscopy/GCSCopy.feature @@ -35,6 +35,7 @@ Feature:GCSCopy - Verification of successful objects copy from one bucket to ano Then Verify the pipeline status is "Succeeded" Then Close the pipeline logs Then Validate GCSCopy successfully copies object "gcsCsvFile" to destination bucket + Then Validate the data of GCS Copy source bucket and destination bucket "gcsCopyCsvExpectedFilePath" @GCS_READ_RECURSIVE_TEST @GCS_SINK_TEST @GCSCopy_Required Scenario: Validate successful copy objects from one bucket to another with Copy All Subdirectories set to true along with data validation. @@ -57,6 +58,7 @@ Feature:GCSCopy - Verification of successful objects copy from one bucket to ano Then Verify the pipeline status is "Succeeded" Then Close the pipeline logs Then Validate GCSCopy successfully copies object "gcsCopyReadRecursivePath" to destination bucket + Then Validate the data of GCS Copy source bucket and destination bucket "gcsCopyRecursivePath" @GCS_READ_RECURSIVE_TEST @GCS_SINK_TEST @GCSCopy_Required Scenario: Validate successful copy objects from one bucket to another with Copy All Subdirectories set to false along with data validation. @@ -79,6 +81,7 @@ Feature:GCSCopy - Verification of successful objects copy from one bucket to ano Then Verify the pipeline status is "Succeeded" Then Close the pipeline logs Then Validate GCSCopy did not copy subdirectory "gcsCopyReadRecursiveSubDirectory" to destination bucket + Then Validate the data of GCS Copy source bucket and destination bucket "gcsCopyRecursivePath" @GCS_CSV_TEST @GCS_SINK_EXISTING_BUCKET_TEST @GCSCopy_Required Scenario: Validate successful copy objects from one bucket to another existing bucket with Overwrite Existing Files set to true along with data validation. @@ -101,6 +104,7 @@ Feature:GCSCopy - Verification of successful objects copy from one bucket to ano Then Verify the pipeline status is "Succeeded" Then Close the pipeline logs Then Validate GCSCopy successfully copies object "gcsCsvFile" to destination bucket + Then Validate the data of GCS Copy source bucket and destination bucket "gcsCopyCsvExpectedFilePath" @GCS_CSV_TEST @GCS_SINK_EXISTING_BUCKET_TEST Scenario: Validate successful copy objects from one bucket to another existing bucket with Overwrite Existing Files set to false along with data validation. @@ -123,6 +127,7 @@ Feature:GCSCopy - Verification of successful objects copy from one bucket to ano And Verify the pipeline status is "Failed" Then Close the pipeline logs Then Validate GCSCopy failed to copy object "gcsCsvFile" to destination bucket + Then Validate the data of GCS Copy source bucket and destination bucket "gcsCopyCsvExpectedFilePath" @GCS_CSV_TEST @GCS_SINK_TEST Scenario:Validate successful Copy object from one bucket to another new bucket with location set to non-default value @@ -144,4 +149,5 @@ Feature:GCSCopy - Verification of successful objects copy from one bucket to ano Then Open and capture logs Then Verify the pipeline status is "Succeeded" Then Close the pipeline logs - Then Validate GCSCopy successfully copied object "gcsCsvFile" to destination bucket in location "locationEU" \ No newline at end of file + Then Validate GCSCopy successfully copied object "gcsCsvFile" to destination bucket in location "locationEU" + Then Validate the data of GCS Copy source bucket and destination bucket "gcsCopyCsvExpectedFilePath" \ No newline at end of file diff --git a/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopy.java b/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopy.java index f75c7800d8..4525ecfcaf 100644 --- a/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopy.java +++ b/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopy.java @@ -30,6 +30,7 @@ import stepsdesign.BeforeActions; import java.io.IOException; +import java.net.URISyntaxException; /** * GCS Copy plugin related stepsdesign. @@ -41,14 +42,15 @@ public class GCSCopy { public void validateGCSCopySuccessfullyCopiedObjectToDestinationBucket(String path) throws IOException { String sourceGCSBucket = TestSetupHooks.gcsSourceBucketName; String gcsObject = PluginPropertyUtils.pluginProp(path); + System.out.println(gcsObject + "Name of the gcsObject"); boolean isPresentAtSource = false; for (Blob blob : StorageClient.listObjects(sourceGCSBucket).iterateAll()) { - if (blob.getName().equals(gcsObject)) { + System.out.println(blob.getName()); + if (blob.getName().contains(gcsObject)) { isPresentAtSource = true; break; } } - if (!isPresentAtSource) { Assert.fail("Object is not present in source bucket" + sourceGCSBucket); @@ -57,14 +59,14 @@ public void validateGCSCopySuccessfullyCopiedObjectToDestinationBucket(String pa String targetGCSBucket = TestSetupHooks.gcsTargetBucketName; try { for (Blob blob : StorageClient.listObjects(targetGCSBucket).iterateAll()) { - if (blob.getName().equals(gcsObject)) { + if (blob.getName().contains(gcsObject)) { BeforeActions.scenario.write("Object copied to gcs bucket " + targetGCSBucket + " successfully"); return; } } Assert.fail("Object not copied to target gcs bucket " + targetGCSBucket); } catch (StorageException | IOException e) { - if (e.getMessage().equals("The specified bucket does not exist")) { + if (e.getMessage().contains("The specified bucket does not exist")) { Assert.fail("Target gcs bucket " + targetGCSBucket + " not created - " + e.getMessage()); } else { Assert.fail(e.getMessage()); @@ -140,8 +142,8 @@ public void validateGCSCopyDidNotCopySubdirectoryToDestinationBucket(String subd return; } } - Assert.fail("Subdirectory is copied from source GCS Bucket " + sourceGCSBucket); - } + Assert.fail("Subdirectory is copied from source GCS Bucket " + sourceGCSBucket); + } @Then("Validate GCSCopy failed to copy object {string} to destination bucket") public void validateGCSCopyFailedToCopyObjectToDestinationBucket(String path) throws IOException { @@ -155,4 +157,10 @@ public void validateGCSCopyFailedToCopyObjectToDestinationBucket(String path) th } Assert.fail("Object is deleted from source GCS Bucket " + sourceGCSBucket); } + + @Then("Validate the data of GCS Copy source bucket and destination bucket {string}") + public void validateTheDataFromGCSSourceToGCSSinkWithExpectedCsvFileAndTargetGCSBucket(String path) { + GCSCopyValidation.validateGCSSourceToGCSSinkWithCSVFormat(TestSetupHooks.gcsTargetBucketName, + PluginPropertyUtils.pluginProp(path)); } +} diff --git a/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopyValidation.java b/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopyValidation.java new file mode 100644 index 0000000000..b4e9403bc9 --- /dev/null +++ b/src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopyValidation.java @@ -0,0 +1,101 @@ +package io.cdap.plugin.gcscopy.stepsdesign; + +import au.com.bytecode.opencsv.CSVReader; +import com.google.api.gax.paging.Page; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; +import com.google.gson.JsonObject; +import io.cdap.e2e.utils.PluginPropertyUtils; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +/** + * Package contains the Validation for the GCSCopy plugin. + */ +public class GCSCopyValidation { + + private static final String projectId = PluginPropertyUtils.pluginProp("projectId"); +// private static final String csvFilePath = PluginPropertyUtils.pluginProp("gcsCopyCsvExpectedFilePath"); + + public static boolean validateGCSSourceToGCSSinkWithCSVFormat(String bucketName, String filepath) { + Map expectedCSVData = readCsvAndConvertToJson(filepath); + Map actualGcsCsvData = listBucketObjects(bucketName); + + boolean isMatched = actualGcsCsvData.equals(expectedCSVData); + + return isMatched; + } + + public static Map readCsvAndConvertToJson(String filepath) { + Map csvDataMap = new HashMap<>(); + try (CSVReader csvReader = new CSVReader(new java.io.FileReader(filepath))) { + // Read the header line to get column names + String[] headers = csvReader.readNext(); + String[] line; + while ((line = csvReader.readNext()) != null) { + JsonObject jsonObject = new JsonObject(); + + for (int j = 0; j < headers.length; j++) { + jsonObject.addProperty(headers[j], line[j]); + } + String id = line[0]; + csvDataMap.put(id, jsonObject); + } + } catch (IOException e) { + e.printStackTrace(); + } + return csvDataMap; + } + + public static Map listBucketObjects(String bucketName) { + Map bucketObjectData = new HashMap<>(); + Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService(); + Page blobs = storage.list(bucketName); + + List bucketObjects = StreamSupport.stream(blobs.iterateAll().spliterator(), true) + .filter(blob -> blob.getSize() != 0) + .collect(Collectors.toList()); + + Stream objectNamesWithData = bucketObjects.stream().map(blob -> blob.getName()); + List bucketObjectNames = objectNamesWithData.collect(Collectors.toList()); + + if (!bucketObjectNames.isEmpty()) { + String objectName = bucketObjectNames.get(0); + if (objectName.contains("part-r")) { + Map dataMap2 = fetchObjectData(projectId, bucketName, objectName); + bucketObjectData.putAll(dataMap2); + } + } + + return bucketObjectData; + } + + public static Map fetchObjectData(String projectId, String bucketName, String objectName) { + Map dataMap = new HashMap<>(); + Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService(); + byte[] objectData = storage.readAllBytes(bucketName, objectName); + String objectDataAsString = new String(objectData, StandardCharsets.UTF_8); + String[] lines = objectDataAsString.split("\n"); + String[] headers = lines[0].split(","); + + for (int i = 1; i < lines.length; i++) { + String[] values = lines[i].split(","); + JsonObject jsonObject = new JsonObject(); + for (int j = 0; j < headers.length; j++) { + jsonObject.addProperty(headers[j], values[j]); + } + String id = values[0]; + dataMap.put(id, jsonObject); + } + return dataMap; + } + +} diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index fcdcc54399..7caede873d 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -259,6 +259,8 @@ invaliddestPath=abc@ invalidEncryptionKey=abc@ gcsCopyReadRecursivePath=testdata/ gcsCopyReadRecursiveSubDirectory=testdata/GCS_RECURSIVE_TEST +gcsCopyCsvExpectedFilePath=src/e2e-test/resources/testdata/GCSCOPY_CSV_TEST.csv +gcsCopyRecursivePath=src/e2e-test/resources/testdata/GCS_RECURSIVE_TEST ## GCSCopy-PLUGIN-PROPERTIES-END ## BQEXECUTE-PLUGIN-PROPERTIES-START diff --git a/src/e2e-test/resources/testdata/GCSCOPY_CSV_TEST.csv b/src/e2e-test/resources/testdata/GCSCOPY_CSV_TEST.csv new file mode 100644 index 0000000000..24a37bf289 --- /dev/null +++ b/src/e2e-test/resources/testdata/GCSCOPY_CSV_TEST.csv @@ -0,0 +1,11 @@ +id,EmployeeDepartment,Employeename,Salary,wotkhours +1,Plumber,Surya,10000,3 +2,Plumber,Sahil,70000,5 +3,Plumber,Waugh,10000,6 +4,Plumber,Steve,10000,7 +5,Plumber,Smith,20000,8 +6,Electrician,Ragini,30000,2 +7,Electrician,Williamson,10000,9 +8,Electrician,James,80000,10 +9,Electrician,Steve Smith,10000,23 +10,Electrician,Mark Waugh,10000,32 \ No newline at end of file