Skip to content

Commit

Permalink
Merge pull request #80 from data-integrations/error-prop-release
Browse files Browse the repository at this point in the history
error propagation for API calls
  • Loading branch information
harshdeeppruthi authored Aug 7, 2024
2 parents 6e57ce2 + 2c34526 commit 064675d
Show file tree
Hide file tree
Showing 28 changed files with 494 additions and 318 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import io.cdap.e2e.utils.AssertionHelper;
import io.cdap.e2e.utils.BigQueryClient;
import io.cdap.e2e.utils.PluginPropertyUtils;
import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException;
import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl;
import io.cdap.plugin.servicenow.locators.ServiceNowPropertiesPage;
import io.cdap.plugin.servicenow.source.ServiceNowSourceConfig;
Expand Down Expand Up @@ -50,22 +51,22 @@ public class ServiceNowSinkPropertiesPageActions {
private static Gson gson = new Gson();

public static void getRecordFromServiceNowTable(String query, String tableName)
throws OAuthProblemException, OAuthSystemException, IOException {
throws ServiceNowAPIException {
config = new ServiceNowSourceConfig(
"", "", "", "", "",
System.getenv("SERVICE_NOW_CLIENT_ID"),
System.getenv("SERVICE_NOW_CLIENT_SECRET"),
System.getenv("SERVICE_NOW_REST_API_ENDPOINT"),
System.getenv("SERVICE_NOW_USERNAME"),
System.getenv("SERVICE_NOW_PASSWORD"),
"", "", "", null);
"", "", "", "", "",
System.getenv("SERVICE_NOW_CLIENT_ID"),
System.getenv("SERVICE_NOW_CLIENT_SECRET"),
System.getenv("SERVICE_NOW_REST_API_ENDPOINT"),
System.getenv("SERVICE_NOW_USERNAME"),
System.getenv("SERVICE_NOW_PASSWORD"),
"", "", "", null);

ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
responseFromServiceNowTable = tableAPIClient.getRecordFromServiceNowTable(tableName, query);
}

public static void verifyIfRecordCreatedInServiceNowIsCorrect(String query, String tableName)
throws IOException, InterruptedException, OAuthProblemException, OAuthSystemException {
throws IOException, InterruptedException, ServiceNowAPIException {

getRecordFromServiceNowTable(query, tableName);
TableResult bigQueryTableData = getBigQueryTableData(TestSetupHooks.bqSourceDataset, TestSetupHooks.bqSourceTable);
Expand All @@ -79,7 +80,7 @@ public static void verifyIfRecordCreatedInServiceNowIsCorrect(String query, Stri
}

public static void verifyIfRecordUpdatedInServiceNowIsCorrect(String query, String tableName)
throws IOException, InterruptedException, OAuthProblemException, OAuthSystemException {
throws IOException, InterruptedException, ServiceNowAPIException {

getRecordFromServiceNowTable(query, tableName);
TableResult bigQueryTableData = getBigQueryTableData(TestSetupHooks.bqSourceDataset, TestSetupHooks.bqSourceTable);
Expand All @@ -96,14 +97,14 @@ public static void verifyIfRecordUpdatedInServiceNowIsCorrect(String query, Stri
}

public static TableResult getBigQueryTableData(String dataset, String table)
throws IOException, InterruptedException {
throws IOException, InterruptedException {
String projectId = PluginPropertyUtils.pluginProp("projectId");
String selectQuery = "SELECT TO_JSON(t) result FROM `" + projectId + "." + dataset + "." + table + "` AS t";
return BigQueryClient.getQueryResult(selectQuery);
}

public static boolean compareValueOfBothResponses(Map<String, String> serviceNowResponseMap,
Map<String, Object> bigQueryResponseMap) {
Map<String, Object> bigQueryResponseMap) {
if (serviceNowResponseMap.isEmpty() || bigQueryResponseMap.isEmpty()) {
return false;
}
Expand All @@ -116,7 +117,7 @@ public static boolean compareValueOfBothResponses(Map<String, String> serviceNow

if (bigQueryValue instanceof Double) {
String bigDecimalValue = new BigDecimal(String.valueOf(bigQueryValue)).setScale(
ServiceNowConstants.DEFAULT_SCALE, RoundingMode.HALF_UP).toString();
ServiceNowConstants.DEFAULT_SCALE, RoundingMode.HALF_UP).toString();
result = serviceNowValue.equals(bigDecimalValue);
} else if (checkBigQueryDateFormat(bigQueryValue.toString()) != null) {
SimpleDateFormat serviceNowDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package io.cdap.plugin.servicenowsink.stepsdesign;

import io.cdap.e2e.utils.PluginPropertyUtils;
import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException;
import io.cdap.plugin.servicenow.util.ServiceNowConstants;
import io.cdap.plugin.servicenowsink.actions.ServiceNowSinkPropertiesPageActions;
import io.cdap.plugin.tests.hooks.TestSetupHooks;
Expand All @@ -36,19 +37,19 @@ public class DesignTimeSteps {

@And("Verify If new record created in ServiceNow application for table {string} is correct")
public void verifyIfNewRecordCreatedInServiceNowApplicationForTableIsCorrect(String tableName)
throws IOException, InterruptedException, OAuthProblemException, OAuthSystemException {
throws IOException, InterruptedException, ServiceNowAPIException {
String tableValueFromPluginPropertiesFile = PluginPropertyUtils.pluginProp(tableName);

switch (tableValueFromPluginPropertiesFile) {
case "proc_rec_slip_item":
query = "number=" + TestSetupHooks.receivingSlipLineRecordUniqueNumber;
ServiceNowSinkPropertiesPageActions.verifyIfRecordCreatedInServiceNowIsCorrect(query,
tableValueFromPluginPropertiesFile);
tableValueFromPluginPropertiesFile);
break;
case "agent_assist_recommendation":
query = "name=" + TestSetupHooks.agentAssistRecommendationUniqueName;
ServiceNowSinkPropertiesPageActions.verifyIfRecordCreatedInServiceNowIsCorrect(query,
tableValueFromPluginPropertiesFile);
tableValueFromPluginPropertiesFile);
break;
default:
throw new IllegalStateException("Unexpected value: " + tableValueFromPluginPropertiesFile);
Expand All @@ -57,11 +58,11 @@ public void verifyIfNewRecordCreatedInServiceNowApplicationForTableIsCorrect(Str

@Then("Verify If an updated record in ServiceNow application for table {string} is correct")
public void verifyIfAnUpdatedRecordInServiceNowApplicationForTableIsCorrect(String tableName)
throws OAuthProblemException, OAuthSystemException, IOException, InterruptedException {
throws IOException, InterruptedException, ServiceNowAPIException {
String tableValueFromPluginPropertiesFile = PluginPropertyUtils.pluginProp(tableName);
query = ServiceNowConstants.SYSTEM_ID + "=" + TestSetupHooks.systemId;
ServiceNowSinkPropertiesPageActions.verifyIfRecordUpdatedInServiceNowIsCorrect(query,
tableValueFromPluginPropertiesFile);
tableValueFromPluginPropertiesFile);
}

@Then("Verify error in Input Schema for non creatable fields")
Expand Down
76 changes: 40 additions & 36 deletions src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import com.google.cloud.bigquery.BigQueryException;
import io.cdap.e2e.utils.BigQueryClient;
import io.cdap.e2e.utils.PluginPropertyUtils;
import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException;
import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl;
import io.cdap.plugin.servicenow.source.ServiceNowSourceConfig;
import io.cdap.plugin.utils.enums.ApplicationInReportingMode;
Expand Down Expand Up @@ -52,17 +53,17 @@ public class TestSetupHooks {
public static void initializeServiceNowSourceConfig() {
BeforeActions.scenario.write("Initialize ServiceNowSourceConfig");
config = new ServiceNowSourceConfig(
"", "", "", "", "",
System.getenv("SERVICE_NOW_CLIENT_ID"),
System.getenv("SERVICE_NOW_CLIENT_SECRET"),
System.getenv("SERVICE_NOW_REST_API_ENDPOINT"),
System.getenv("SERVICE_NOW_USERNAME"),
System.getenv("SERVICE_NOW_PASSWORD"),
"", "", "", null);
"", "", "", "", "",
System.getenv("SERVICE_NOW_CLIENT_ID"),
System.getenv("SERVICE_NOW_CLIENT_SECRET"),
System.getenv("SERVICE_NOW_REST_API_ENDPOINT"),
System.getenv("SERVICE_NOW_USERNAME"),
System.getenv("SERVICE_NOW_PASSWORD"),
"", "", "", null);
}

@Before(order = 2, value = "@SN_PRODUCT_CATALOG_ITEM")
public static void createRecordInProductCatalogItemTable() throws IOException {
public static void createRecordInProductCatalogItemTable() throws IOException, ServiceNowAPIException {
BeforeActions.scenario.write("Create new record in Product Catalog Item table");
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
String uniqueId = "TestProductCatalogItem" + RandomStringUtils.randomAlphanumeric(10);
Expand All @@ -72,7 +73,8 @@ public static void createRecordInProductCatalogItemTable() throws IOException {
}

@Before(order = 2, value = "@SN_RECEIVING_SLIP_LINE")
public static void createRecordInReceivingSlipLineTable() throws IOException {
public static void createRecordInReceivingSlipLineTable()
throws IOException, ServiceNowAPIException {
BeforeActions.scenario.write("Create new record in Receiving Slip Line table");
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
String uniqueId = "TestReceivingSlipLine" + RandomStringUtils.randomAlphanumeric(10);
Expand All @@ -82,7 +84,8 @@ public static void createRecordInReceivingSlipLineTable() throws IOException {
}

@Before(order = 2, value = "@SN_UPDATE_AGENT_ASSIST_RECOMMENDATION")
public static void updateRecordInAgentAssistRecommendationTable() throws IOException {
public static void updateRecordInAgentAssistRecommendationTable()
throws IOException, ServiceNowAPIException {
BeforeActions.scenario.write("Create new record in Agent Assist Recommendation table");
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
String uniqueId = "TestAgentAssist" + RandomStringUtils.randomAlphanumeric(10);
Expand All @@ -92,7 +95,8 @@ public static void updateRecordInAgentAssistRecommendationTable() throws IOExcep
}

@Before(order = 2, value = "@SN_UPDATE_VENDOR_CATALOG_ITEM")
public static void updateRecordInAgentVendorCatalogItem() throws IOException {
public static void updateRecordInAgentVendorCatalogItem()
throws IOException, ServiceNowAPIException {
BeforeActions.scenario.write("Create new record in Vendor Catalog Item table");
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
String uniqueId = "TestVendorCatalog" + RandomStringUtils.randomAlphanumeric(10);
Expand All @@ -102,12 +106,12 @@ public static void updateRecordInAgentVendorCatalogItem() throws IOException {
}

@Before(order = 2, value = "@SN_UPDATE_SERVICE_OFFERING")
public static void updateRecordInServiceOffering() throws IOException {
public static void updateRecordInServiceOffering() throws IOException, ServiceNowAPIException {
BeforeActions.scenario.write("Create new record in Service Offering table");
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
String uniqueId = "TestServiceOffering" + RandomStringUtils.randomAlphanumeric(10);
String recordDetails = "{'purchase_date':'2022-05-28','end_date':'2022-06-05 15:00:00'," +
" 'start_date':'2022-05-25 15:00:00','number':'" + uniqueId + "'}";
" 'start_date':'2022-05-25 15:00:00','number':'" + uniqueId + "'}";
StringEntity entity = new StringEntity(recordDetails);
systemId = tableAPIClient.createRecord(TablesInTableMode.SERVICE_OFFERING.value, entity);
}
Expand All @@ -120,8 +124,8 @@ public static void createTempSourceBQTableForReceivingSlipLineTable() throws IOE
receivingSlipLineRecordUniqueNumber = "ProcRecSlip" + stringUniqueId;

BigQueryClient.getSoleQueryResult("create table `" + bqSourceDataset + "." + bqSourceTable + "` as " +
"SELECT * FROM UNNEST([ STRUCT('" + receivingSlipLineRecordUniqueNumber + "' " +
"AS number, (DATETIME '2022-06-08 00:00:00') AS received)])");
"SELECT * FROM UNNEST([ STRUCT('" + receivingSlipLineRecordUniqueNumber + "' " +
"AS number, (DATETIME '2022-06-08 00:00:00') AS received)])");
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " created successfully");
}

Expand All @@ -133,8 +137,8 @@ public static void createTempSourceBQTableForAgentFile() throws IOException, Int
agentAssistRecommendationUniqueName = "Agent" + stringUniqueId;

BigQueryClient.getSoleQueryResult("create table `" + bqSourceDataset + "." + bqSourceTable + "` as " +
"SELECT * FROM UNNEST([ STRUCT(" + active + " AS active," +
" '" + agentAssistRecommendationUniqueName + "' AS name)])");
"SELECT * FROM UNNEST([ STRUCT(" + active + " AS active," +
" '" + agentAssistRecommendationUniqueName + "' AS name)])");
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " created successfully");
}

Expand All @@ -150,8 +154,8 @@ public static void createTempSourceBQTableForAgentAssistRecomendation() throws I
vendorCatalogItemUniqueName = "VendorCatalog" + stringUniqueId;

BigQueryClient.getSoleQueryResult("create table `" + bqSourceDataset + "." + bqSourceTable + "` as " +
"SELECT * FROM UNNEST([ STRUCT(" + outOfStock + " AS out_of_stock,' "
+ vendorCatalogItemUniqueName + " ' AS product_id)])");
"SELECT * FROM UNNEST([ STRUCT(" + outOfStock + " AS out_of_stock,' "
+ vendorCatalogItemUniqueName + " ' AS product_id)])");
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " created successfully");
}

Expand All @@ -161,10 +165,10 @@ public static void createTempSourceBQTableForServiceOffering() throws IOExceptio
bqSourceTable = "testTable" + stringUniqueId;
serviceOfferingUniqueNumber = "ServiceOffering" + stringUniqueId;
BigQueryClient.getSoleQueryResult("create table `" + bqSourceDataset + "." + bqSourceTable + "` as " +
"SELECT * FROM UNNEST([ STRUCT( (DATE '2022-06-10') AS purchase_date," +
" (DATETIME '2022-06-08 16:00:00') AS end_date," +
" (TIMESTAMP '2022-05-10 15:00:00-00:00') AS start_date,' "
+ serviceOfferingUniqueNumber + " ' AS number)])");
"SELECT * FROM UNNEST([ STRUCT( (DATE '2022-06-10') AS purchase_date," +
" (DATETIME '2022-06-08 16:00:00') AS end_date," +
" (TIMESTAMP '2022-05-10 15:00:00-00:00') AS start_date,' "
+ serviceOfferingUniqueNumber + " ' AS number)])");
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " created successfully");
}

Expand All @@ -177,8 +181,8 @@ public static void updateTempSourceBQTableForReceivingSlipLineTable() throws IOE
String number = "updatedReceiving" + uniqueId;

BigQueryClient.getSoleQueryResult("create table `" + bqSourceDataset + "." + bqSourceTable + "` as " +
"SELECT * FROM UNNEST([ STRUCT('" + number + "' AS number," +
" '" + systemId + "' AS sys_id )])");
"SELECT * FROM UNNEST([ STRUCT('" + number + "' AS number," +
" '" + systemId + "' AS sys_id )])");
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " created successfully");
}

Expand All @@ -190,9 +194,9 @@ public static void updateTempSourceBQTableForAgentFile() throws IOException, Int
String name = "Agent";

BigQueryClient.getSoleQueryResult("create table `" + bqSourceDataset + "." + bqSourceTable + "` as " +
"SELECT * FROM UNNEST([ STRUCT(" + active + " AS active," +
" '" + name + "' AS name," +
" '" + systemId + "' AS sys_id)])");
"SELECT * FROM UNNEST([ STRUCT(" + active + " AS active," +
" '" + name + "' AS name," +
" '" + systemId + "' AS sys_id)])");
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " created successfully");
}

Expand All @@ -206,9 +210,9 @@ public static void createTempSourceBQTableForVendorCatalogItem() throws IOExcept
String name = "check";

BigQueryClient.getSoleQueryResult("create table `" + bqSourceDataset + "." + bqSourceTable + "` as " +
"SELECT * FROM UNNEST([ STRUCT(" + outOfStock + " AS out_of_stock,'"
+ name + "' AS sys_update_name," +
" '" + systemId + "' AS sys_id)])");
"SELECT * FROM UNNEST([ STRUCT(" + outOfStock + " AS out_of_stock,'"
+ name + "' AS sys_update_name," +
" '" + systemId + "' AS sys_id)])");
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " created successfully");
}

Expand All @@ -218,11 +222,11 @@ public static void updateTempSourceBQTableForServiceOffering() throws IOExceptio
bqSourceTable = "testTable" + stringUniqueId;
serviceOfferingUniqueNumber = "ServiceOffering" + stringUniqueId;
BigQueryClient.getSoleQueryResult("create table `" + bqSourceDataset + "." + bqSourceTable + "` as " +
"SELECT * FROM UNNEST([ STRUCT((DATE '2022-06-10') AS purchase_date," +
" (DATETIME '2022-06-08 16:00:00') AS end_date," +
" (TIMESTAMP '2022-05-10 15:00:00-00:00') AS start_date," +
" '" + systemId + "' AS sys_id,' "
+ serviceOfferingUniqueNumber + " ' AS number)])");
"SELECT * FROM UNNEST([ STRUCT((DATE '2022-06-10') AS purchase_date," +
" (DATETIME '2022-06-08 16:00:00') AS end_date," +
" (TIMESTAMP '2022-05-10 15:00:00-00:00') AS start_date," +
" '" + systemId + "' AS sys_id,' "
+ serviceOfferingUniqueNumber + " ' AS number)])");
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " created successfully");
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ public void validateTable(String tableName, SourceValueType valueType, FailureCo
// Get the response JSON and fetch the header X-Total-Count. Set the value to recordCount
requestBuilder.setResponseHeaders(ServiceNowConstants.HEADER_NAME_TOTAL_COUNT);

apiResponse = serviceNowTableAPIClient.executeGet(requestBuilder.build());
apiResponse = serviceNowTableAPIClient.executeGetWithRetries(requestBuilder.build());
if (serviceNowTableAPIClient.parseResponseToResultListOfMap(apiResponse.getResponseBody()).isEmpty()) {
// Removed config property as in case of MultiSource, only first table error was populating.
collector.addFailure("Table: " + tableName + " is empty.", "");
Expand Down

This file was deleted.

Loading

0 comments on commit 064675d

Please sign in to comment.