-
Notifications
You must be signed in to change notification settings - Fork 85
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
gcscopy e2e tests review comments for validation
- Loading branch information
1 parent
13b333f
commit b9787cb
Showing
5 changed files
with
135 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
101 changes: 101 additions & 0 deletions
101
src/e2e-test/java/io/cdap/plugin/gcscopy/stepsdesign/GCSCopyValidation.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,101 @@ | ||
| package io.cdap.plugin.gcscopy.stepsdesign; | ||
|
|
||
| import au.com.bytecode.opencsv.CSVReader; | ||
| import com.google.api.gax.paging.Page; | ||
| import com.google.cloud.storage.Blob; | ||
| import com.google.cloud.storage.Storage; | ||
| import com.google.cloud.storage.StorageOptions; | ||
| import com.google.gson.JsonObject; | ||
| import io.cdap.e2e.utils.PluginPropertyUtils; | ||
|
|
||
| import java.io.IOException; | ||
| import java.nio.charset.StandardCharsets; | ||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
| import java.util.stream.Collectors; | ||
| import java.util.stream.Stream; | ||
| import java.util.stream.StreamSupport; | ||
|
|
||
| /** | ||
| * Package contains the Validation for the GCSCopy plugin. | ||
| */ | ||
| public class GCSCopyValidation { | ||
|
|
||
| private static final String projectId = PluginPropertyUtils.pluginProp("projectId"); | ||
| // private static final String csvFilePath = PluginPropertyUtils.pluginProp("gcsCopyCsvExpectedFilePath"); | ||
|
|
||
| public static boolean validateGCSSourceToGCSSinkWithCSVFormat(String bucketName, String filepath) { | ||
| Map<String, JsonObject> expectedCSVData = readCsvAndConvertToJson(filepath); | ||
| Map<String, JsonObject> actualGcsCsvData = listBucketObjects(bucketName); | ||
|
|
||
| boolean isMatched = actualGcsCsvData.equals(expectedCSVData); | ||
|
|
||
| return isMatched; | ||
| } | ||
|
|
||
| public static Map<String, JsonObject> readCsvAndConvertToJson(String filepath) { | ||
| Map<String, JsonObject> csvDataMap = new HashMap<>(); | ||
| try (CSVReader csvReader = new CSVReader(new java.io.FileReader(filepath))) { | ||
| // Read the header line to get column names | ||
| String[] headers = csvReader.readNext(); | ||
| String[] line; | ||
| while ((line = csvReader.readNext()) != null) { | ||
| JsonObject jsonObject = new JsonObject(); | ||
|
|
||
| for (int j = 0; j < headers.length; j++) { | ||
| jsonObject.addProperty(headers[j], line[j]); | ||
| } | ||
| String id = line[0]; | ||
| csvDataMap.put(id, jsonObject); | ||
| } | ||
| } catch (IOException e) { | ||
| e.printStackTrace(); | ||
| } | ||
| return csvDataMap; | ||
| } | ||
|
|
||
| public static Map<String, JsonObject> listBucketObjects(String bucketName) { | ||
| Map<String, JsonObject> bucketObjectData = new HashMap<>(); | ||
| Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService(); | ||
| Page<Blob> blobs = storage.list(bucketName); | ||
|
|
||
| List<Blob> bucketObjects = StreamSupport.stream(blobs.iterateAll().spliterator(), true) | ||
| .filter(blob -> blob.getSize() != 0) | ||
| .collect(Collectors.toList()); | ||
|
|
||
| Stream<String> objectNamesWithData = bucketObjects.stream().map(blob -> blob.getName()); | ||
| List<String> bucketObjectNames = objectNamesWithData.collect(Collectors.toList()); | ||
|
|
||
| if (!bucketObjectNames.isEmpty()) { | ||
| String objectName = bucketObjectNames.get(0); | ||
| if (objectName.contains("part-r")) { | ||
| Map<String, JsonObject> dataMap2 = fetchObjectData(projectId, bucketName, objectName); | ||
| bucketObjectData.putAll(dataMap2); | ||
| } | ||
| } | ||
|
|
||
| return bucketObjectData; | ||
| } | ||
|
|
||
| public static Map<String, JsonObject> fetchObjectData(String projectId, String bucketName, String objectName) { | ||
| Map<String, JsonObject> dataMap = new HashMap<>(); | ||
| Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService(); | ||
| byte[] objectData = storage.readAllBytes(bucketName, objectName); | ||
| String objectDataAsString = new String(objectData, StandardCharsets.UTF_8); | ||
| String[] lines = objectDataAsString.split("\n"); | ||
| String[] headers = lines[0].split(","); | ||
|
|
||
| for (int i = 1; i < lines.length; i++) { | ||
| String[] values = lines[i].split(","); | ||
| JsonObject jsonObject = new JsonObject(); | ||
| for (int j = 0; j < headers.length; j++) { | ||
| jsonObject.addProperty(headers[j], values[j]); | ||
| } | ||
| String id = values[0]; | ||
| dataMap.put(id, jsonObject); | ||
| } | ||
| return dataMap; | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| id,EmployeeDepartment,Employeename,Salary,wotkhours | ||
| 1,Plumber,Surya,10000,3 | ||
| 2,Plumber,Sahil,70000,5 | ||
| 3,Plumber,Waugh,10000,6 | ||
| 4,Plumber,Steve,10000,7 | ||
| 5,Plumber,Smith,20000,8 | ||
| 6,Electrician,Ragini,30000,2 | ||
| 7,Electrician,Williamson,10000,9 | ||
| 8,Electrician,James,80000,10 | ||
| 9,Electrician,Steve Smith,10000,23 | ||
| 10,Electrician,Mark Waugh,10000,32 |