From 1b8106d8e2b6cc66ad9526f9ec3ff2358f5c73c0 Mon Sep 17 00:00:00 2001 From: AnkitCLI Date: Thu, 6 Jun 2024 22:45:39 +0530 Subject: [PATCH] e2e-Normalize-ITN --- .../normalize/FileToNormalize.feature | 49 +++++++++++++++++++ .../common/stepsdesign/TestSetupHooks.java | 11 ++++- .../resources/pluginParameters.properties | 2 + .../testdata/file/CSV_Normalize_TEST_3.csv | 5 ++ .../OUTPUT_FOR_NORMALIZE_TEST.csv | 13 +++++ 5 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 core-plugins/src/e2e-test/resources/testdata/file/CSV_Normalize_TEST_3.csv create mode 100644 core-plugins/src/e2e-test/resources/testdata/file/expected_outputs/OUTPUT_FOR_NORMALIZE_TEST.csv diff --git a/core-plugins/src/e2e-test/features/normalize/FileToNormalize.feature b/core-plugins/src/e2e-test/features/normalize/FileToNormalize.feature index bb00df369..26f311ba4 100644 --- a/core-plugins/src/e2e-test/features/normalize/FileToNormalize.feature +++ b/core-plugins/src/e2e-test/features/normalize/FileToNormalize.feature @@ -147,3 +147,52 @@ Feature: Normalize transform - Verify File source data transfer using Normalize Then Verify the pipeline status is "Succeeded" Then Close the pipeline logs Then Validate output file generated by file sink plugin "fileSinkTargetBucket" is equal to expected output file "normalizeCsvAllDataTypeOutputFile" + + @NORMALIZE_TEST3 @FILE_SINK_TEST + Scenario: To verify data is getting transferred from File source to File sink successfully with Normalized fields using empty attribute + Given Open Datafusion Project to configure pipeline + When Select plugin: "File" from the plugins list as: "Source" + When Expand Plugin group in the LHS plugins list: "Transform" + When Select plugin: "Transpose" from the plugins list as: "Transform" + Then Connect plugins: "File" and "Transpose" to establish connection + When Expand Plugin group in the LHS plugins list: "Sink" + When Select plugin: "File" from the plugins list as: "Sink" + Then Connect plugins: "Transpose" and "File2" to establish connection + Then Navigate to the properties page of plugin: "File" + Then Enter input plugin property: "referenceName" with value: "FileReferenceName" + Then Enter input plugin property: "path" with value: "normalizeTest3" + Then Select dropdown plugin property: "format" with option value: "csv" + Then Click plugin property: "skipHeader" + Then Click on the Get Schema button + Then Verify the Output Schema matches the Expected Schema: "normalizeCsvAllDataTypeFileSchema" + Then Validate "File" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "Transpose" + Then Enter Normalize plugin Fields to be Mapped "normalizeFileValidFieldsMapping" + Then Enter Normalize plugin Fields to be Normalized "normalizeFieldsToBeNormalizedFile" + Then Select Normalize plugin output schema action: "clear" + Then Enter Normalize plugin outputSchema "normalizeBQValidOutputSchema" + Then Validate "Transpose" plugin properties + Then Close the Plugin Properties page + Then Navigate to the properties page of plugin: "File2" + Then Enter input plugin property: "referenceName" with value: "FileReferenceName" + Then Enter input plugin property: "path" with value: "fileSinkTargetBucket" + Then Replace input plugin property: "pathSuffix" with value: "yyyy-MM-dd-HH-mm-ss" + Then Select dropdown plugin property: "format" with option value: "csv" + Then Click plugin property: "writeHeader" + Then Validate "File2" plugin properties + Then Close the Plugin Properties page + Then Save the pipeline + Then Preview and run the pipeline + Then Wait till pipeline preview is in running state + Then Open and capture pipeline preview logs + Then Verify the preview run status of pipeline in the logs is "succeeded" + Then Close the pipeline logs + Then Close the preview + Then Deploy the pipeline + Then Run the Pipeline in Runtime + Then Wait till pipeline is in running state + Then Open and capture logs + Then Verify the pipeline status is "Succeeded" + Then Close the pipeline logs + Then Validate output file generated by file sink plugin "fileSinkTargetBucket" is equal to expected output file "normalizeCsvAllDataTypeOutputFile1" diff --git a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java index ff161c70d..4e378b76c 100644 --- a/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java +++ b/core-plugins/src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java @@ -244,7 +244,16 @@ public static void createBucketWithNormalizeTest1File() throws IOException, URIS BeforeActions.scenario.write("Normalize 1st bucket name - " + gcsSourceBucketName1); } - @After(order = 1, value = "@NORMALIZE_TEST1") + @Before(order = 1, value = "@NORMALIZE_TEST3") + public static void createBucketWithNormalizeTest3File() throws IOException, URISyntaxException { + gcsSourceBucketName1 = createGCSBucketWithFile(PluginPropertyUtils.pluginProp( + "normalizeCsvAllDataTypeFile3")); + PluginPropertyUtils.addPluginProp("normalizeTest3", "gs://" + gcsSourceBucketName1 + "/" + + PluginPropertyUtils.pluginProp("normalizeCsvAllDataTypeFile3")); + BeforeActions.scenario.write("Normalize 1st bucket name - " + gcsSourceBucketName1); + } + + @After(order = 1, value = "@NORMALIZE_TEST1 or @NORMALIZE_TEST3") public static void deleteSourceBucketWithNormalizeTest1File() { deleteGCSBucket(gcsSourceBucketName1); gcsSourceBucketName1 = StringUtils.EMPTY; diff --git a/core-plugins/src/e2e-test/resources/pluginParameters.properties b/core-plugins/src/e2e-test/resources/pluginParameters.properties index 667c45eec..f89e75c42 100644 --- a/core-plugins/src/e2e-test/resources/pluginParameters.properties +++ b/core-plugins/src/e2e-test/resources/pluginParameters.properties @@ -131,7 +131,9 @@ normalizeCsvFileOutputSchema={ "type": "record", "name": "text", "fields": [ \ normalizeBQValidDatatypeOutputSchema=[{"key":"ID","value":"int"},{"key":"AttributeType","value":"string"},\ {"key":"AttributeValue","value":"string"},{"key":"Date","value":"string"}] normalizeCsvAllDataTypeFile=testdata/file/CSV_Normalize_TEST_1.csv +normalizeCsvAllDataTypeFile3=testdata/file/CSV_Normalize_TEST_3.csv normalizeCsvAllDataTypeOutputFile=e2e-tests/expected_outputs/CSV_NormalizeOutput.csv +normalizeCsvAllDataTypeOutputFile1=e2e-tests/file/expected_outputs/OUTPUT_FOR_NORMALIZE_TEST.csv normalizeCsvAllDataTypeFileSchema=[{"key":"id","value":"int"},{"key":"name","value":"string"},\ {"key":"yearofbirth","value":"int"},{"key":"isdeleted","value":"boolean"},{"key":"email","value":"string"},\ {"key":"createddate","value":"string"},{"key":"revenue","value":"string"},{"key":"points","value":"string"},\ diff --git a/core-plugins/src/e2e-test/resources/testdata/file/CSV_Normalize_TEST_3.csv b/core-plugins/src/e2e-test/resources/testdata/file/CSV_Normalize_TEST_3.csv new file mode 100644 index 000000000..6cc0a6a38 --- /dev/null +++ b/core-plugins/src/e2e-test/resources/testdata/file/CSV_Normalize_TEST_3.csv @@ -0,0 +1,5 @@ +id,name,yearofbirth,isdeleted,email,createddate,revenue,points,longdatatype,doubledatatype,date,null,BytesData +1,albert einstein,1879,true,sumitsri@gmail.com,,900750000.01,3.14235678,-9223372036854770000,22.8,1996-07-21,,10111011101110111011 +2,isaac newton,1643,false,sumitsri@gmail.com,,900750000.01,3.14235678906787648,-9223372036854770000,123.08,1996-07-21,str,10111011101110111011 +3,marie curie,1867,true,sumitsri@gmail.com,2021-09-20 11:27:50 UTC,900750000.01,3.14235678,-9223372036854770000,124.97,1996-07-21,,10111011101110111011 +4,galilée,1564,false,sumitsri@gmail.com,2021-09-20 11:27:50 UTC,900750000.01,3.14235678,-2^63,234.89,1996-07-21,,10111011101110111011 \ No newline at end of file diff --git a/core-plugins/src/e2e-test/resources/testdata/file/expected_outputs/OUTPUT_FOR_NORMALIZE_TEST.csv b/core-plugins/src/e2e-test/resources/testdata/file/expected_outputs/OUTPUT_FOR_NORMALIZE_TEST.csv new file mode 100644 index 000000000..d07c13bc1 --- /dev/null +++ b/core-plugins/src/e2e-test/resources/testdata/file/expected_outputs/OUTPUT_FOR_NORMALIZE_TEST.csv @@ -0,0 +1,13 @@ +AttributeValue,AttributeType,ID,Date +sumitsri@gmail.com,email,1,null +900750000.01,revenue,1,null +3.14235678,points,1,null +sumitsri@gmail.com,email,2,null +900750000.01,revenue,2,null +3.14235678906787648,points,2,null +sumitsri@gmail.com,email,3,2021-09-20 11:27:50 UTC +900750000.01,revenue,3,2021-09-20 11:27:50 UTC +3.14235678,points,3,2021-09-20 11:27:50 UTC +sumitsri@gmail.com,email,4,2021-09-20 11:27:50 UTC +900750000.01,revenue,4,2021-09-20 11:27:50 UTC +3.14235678,points,4,2021-09-20 11:27:50 UTC \ No newline at end of file