From 3a50ad068f26b657d6858f3a1c6c9e251c1b80bf Mon Sep 17 00:00:00 2001 From: AnkitCLI Date: Thu, 28 Mar 2024 22:44:27 +0530 Subject: [PATCH] addressed comments --- .../stepsdesign/BigQueryMultiTable.java | 19 +++++ .../BigQueryMultiTableValidation.java | 85 ++++++++++--------- .../resources/pluginParameters.properties | 2 +- 3 files changed, 67 insertions(+), 39 deletions(-) diff --git a/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryMultiTable.java b/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryMultiTable.java index 38936c82c5..7904c8db03 100644 --- a/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryMultiTable.java +++ b/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryMultiTable.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.bigquery.stepsdesign; import io.cdap.e2e.utils.PluginPropertyUtils; @@ -8,6 +24,9 @@ import java.util.Arrays; import java.util.List; +/** + * BigQuery MultiTable related common stepDesigns. + */ public class BigQueryMultiTable { @Then("Validate data transferred from BigQuery To BigQueryMultiTable is equal") public void validateDataTransferredFromBigQueryToBigQueryMultiTableIsEqual() throws IOException, InterruptedException { diff --git a/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryMultiTableValidation.java b/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryMultiTableValidation.java index 722d95820e..c11057c9fb 100644 --- a/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryMultiTableValidation.java +++ b/src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryMultiTableValidation.java @@ -1,3 +1,19 @@ +/* + * Copyright © 2024 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + package io.cdap.plugin.bigquery.stepsdesign; import com.google.cloud.bigquery.FieldValue; @@ -14,8 +30,12 @@ import java.util.List; import java.util.stream.Collectors; +/** + * BQValidation. + */ public class BigQueryMultiTableValidation { static Gson gson = new Gson(); + public static boolean validateBQToBigQueryMultiTableInTwoTables(List sourceTables) throws IOException, InterruptedException { List targetTables = getTableByName().stream().sorted().collect(Collectors.toList()); if (sourceTables.size() != targetTables.size()) { @@ -24,51 +44,39 @@ public static boolean validateBQToBigQueryMultiTableInTwoTables(List sou for (int i = 0; i < sourceTables.size(); i++) { String currentSourceTable = sourceTables.get(i); String currentTargetTable = targetTables.get(i); - List bigQueryRows = new ArrayList<>(); - getBigQueryTableData(currentSourceTable, bigQueryRows); - List bigQueryResponse = new ArrayList<>(); - for (Object row : bigQueryRows) { - JsonObject jsonData = gson.fromJson(String.valueOf(row), JsonObject.class); - bigQueryResponse.add(jsonData); - } - List bigQueryRows2 = new ArrayList<>(); - getBigQueryTableData(currentTargetTable, bigQueryRows2); - - List bigQueryResponse2 = new ArrayList<>(); - for (Object row : bigQueryRows2) { - JsonObject jsonData = gson.fromJson(String.valueOf(row), JsonObject.class); - bigQueryResponse2.add(jsonData); - } - boolean isValid = compareBigQueryDataAndBQMT(bigQueryResponse, bigQueryResponse2); - if (!isValid) { + if (!validateBQToBQSingleTable(currentSourceTable, currentTargetTable)) { return false; // Return false if validation fails for any table } } - return true; // Return true if validation passes for all tables + // Return true if validation passes for all tables + return true; } public static boolean validateBQToBigQueryMultiTableInOneTable(String sourceTable) throws IOException, InterruptedException { String currentTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable"); - String currentSourceTable = sourceTable; - List bigQueryRows = new ArrayList<>(); - getBigQueryTableData(currentSourceTable, bigQueryRows); - List bigQueryResponse = new ArrayList<>(); - for (Object row : bigQueryRows) { - JsonObject jsonData = gson.fromJson(String.valueOf(row), JsonObject.class); - bigQueryResponse.add(jsonData); - } - List bigQueryRows2 = new ArrayList<>(); - getBigQueryTableData(currentTargetTable, bigQueryRows2); - List bigQueryResponse2 = new ArrayList<>(); - for (Object row : bigQueryRows2) { - JsonObject jsonData = gson.fromJson(String.valueOf(row), JsonObject.class); - bigQueryResponse2.add(jsonData); - } - boolean isValid = compareBigQueryDataAndBQMT(bigQueryResponse, bigQueryResponse2); - if (!isValid) { - return false; // Return false if validation fails for any table - } - return true; // Return true if validation passes for all tables + return validateBQToBQSingleTable(sourceTable, currentTargetTable); + } + + public static boolean validateBQToBQSingleTable(String sourceTable, String targetTable) throws IOException, InterruptedException { + // Fetch data from the source BigQuery table + List bigQueryRows = new ArrayList<>(); + getBigQueryTableData(sourceTable, bigQueryRows); + + // Convert fetched data into JSON objects + List bigQueryResponse = new ArrayList<>(); + for (Object row : bigQueryRows) { + JsonObject jsonData = gson.fromJson(String.valueOf(row), JsonObject.class); + bigQueryResponse.add(jsonData); + } + // Fetch data from the target BigQuery table + List bigQueryRows2 = new ArrayList<>(); + getBigQueryTableData(targetTable, bigQueryRows2); + List bigQueryResponse2 = new ArrayList<>(); + for (Object row : bigQueryRows2) { + JsonObject jsonData = gson.fromJson(String.valueOf(row), JsonObject.class); + bigQueryResponse2.add(jsonData); + } + return compareBigQueryDataAndBQMT(bigQueryResponse, bigQueryResponse2); } private static void getBigQueryTableData(String table, List bigQueryRows) throws IOException, @@ -107,6 +115,7 @@ public static List getTableByName() throws IOException, InterruptedExcep } return tableNames; } + private static boolean compareBigQueryDataAndBQMT(List bigQueryResponse, List bqmtData) throws NullPointerException { if (bigQueryResponse.size() != bqmtData.size()) { return false; diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index 4d93a865ee..1e5a869129 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -1,6 +1,6 @@ projectId=cdf-athena datasetprojectId=cdf-athena -dataset=ankitdataset +dataset=test_bqmt_automation wrongSourcePath=gs://00000000-e2e-0014a44f-81be-4501-8360-0ddca192492 serviceAccountType=filePath serviceAccount=auto-detect