From f218124bf492e3eb645f221bdec6b064280064b8 Mon Sep 17 00:00:00 2001 From: Harshdeep Singh Pruthi Date: Tue, 18 Jun 2024 15:26:11 +0530 Subject: [PATCH] error propagation for API calls --- .../servicenow/ServiceNowBaseConfig.java | 2 +- .../apiclient/NonRetryableException.java | 35 -------- .../apiclient/RetryableException.java | 35 -------- .../apiclient/ServiceNowAPIException.java | 77 ++++++++++++++++ .../ServiceNowTableAPIClientImpl.java | 83 +++++++++-------- .../connector/ServiceNowConnector.java | 18 ++-- .../servicenow/restapi/RestAPIClient.java | 75 +++++++++++++--- .../servicenow/restapi/RestAPIResponse.java | 88 ++++++++++--------- .../service/ServiceNowSinkAPIRequestImpl.java | 6 +- .../source/ServiceNowInputFormat.java | 3 +- .../source/ServiceNowMultiInputFormat.java | 3 +- .../source/ServiceNowMultiRecordReader.java | 5 +- .../source/ServiceNowRecordReader.java | 3 +- .../ServiceNowTableAPIClientImplTest.java | 82 +++++++++++++++++ .../connector/ServiceNowConnectorTest.java | 4 +- .../servicenow/restapi/RestAPIClientTest.java | 17 ++-- .../sink/ServiceNowRecordWriterTest.java | 18 ++-- .../sink/ServiceNowSinkConfigTest.java | 16 ++-- .../servicenow/sink/ServiceNowSinkTest.java | 12 ++- .../source/ServiceNowInputFormatTest.java | 15 ++-- .../ServiceNowMultiRecordReaderTest.java | 42 +++++---- .../ServiceNowMultiSourceConfigTest.java | 20 ++--- .../source/ServiceNowMultiSourceTest.java | 15 ++-- .../source/ServiceNowSourceConfigTest.java | 14 ++- .../source/ServiceNowSourceTest.java | 15 ++-- 25 files changed, 439 insertions(+), 264 deletions(-) delete mode 100644 src/main/java/io/cdap/plugin/servicenow/apiclient/NonRetryableException.java delete mode 100644 src/main/java/io/cdap/plugin/servicenow/apiclient/RetryableException.java create mode 100644 src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowAPIException.java create mode 100644 src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java diff --git a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java index eebb386..945a9e3 100644 --- a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java @@ -137,7 +137,7 @@ public void validateTable(String tableName, SourceValueType valueType, FailureCo // Get the response JSON and fetch the header X-Total-Count. Set the value to recordCount requestBuilder.setResponseHeaders(ServiceNowConstants.HEADER_NAME_TOTAL_COUNT); - apiResponse = serviceNowTableAPIClient.executeGet(requestBuilder.build()); + apiResponse = serviceNowTableAPIClient.executeGetWithRetries(requestBuilder.build()); if (serviceNowTableAPIClient.parseResponseToResultListOfMap(apiResponse.getResponseBody()).isEmpty()) { // Removed config property as in case of MultiSource, only first table error was populating. collector.addFailure("Table: " + tableName + " is empty.", ""); diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/NonRetryableException.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/NonRetryableException.java deleted file mode 100644 index 7bf2342..0000000 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/NonRetryableException.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright © 2022 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.servicenow.apiclient; - -/** Custom Exception Class for handling retrying API calls */ -public class NonRetryableException extends RuntimeException { - - private static final long serialVersionUID = 1L; - - public NonRetryableException() { - super(); - } - - public NonRetryableException(String message) { - super(message); - } - - public NonRetryableException(String message, Throwable throwable) { - super(message, throwable); - } -} diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/RetryableException.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/RetryableException.java deleted file mode 100644 index bf1cd71..0000000 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/RetryableException.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright © 2022 Cask Data, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package io.cdap.plugin.servicenow.apiclient; - -/** Custom Exception Class for handling retrying API calls */ -public class RetryableException extends RuntimeException { - - private static final long serialVersionUID = 1L; - - public RetryableException() { - super(); - } - - public RetryableException(String message) { - super(message); - } - - public RetryableException(String message, Throwable throwable) { - super(message, throwable); - } -} diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowAPIException.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowAPIException.java new file mode 100644 index 0000000..cf1f592 --- /dev/null +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowAPIException.java @@ -0,0 +1,77 @@ +package io.cdap.plugin.servicenow.apiclient; + +import io.cdap.plugin.servicenow.util.ServiceNowConstants; + +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.oltu.oauth2.common.exception.OAuthSystemException; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; +import javax.annotation.Nullable; + +/** + * Custom Exception class for propagating API errors/exceptions back to caller. + */ +public class ServiceNowAPIException extends Exception { + + @Nullable private final HttpResponse httpResponse; + @Nullable private final boolean manualRetry; + + private static final Set RETRYABLE_CODES = new HashSet<>(Arrays.asList(429, + HttpStatus.SC_BAD_GATEWAY, + HttpStatus.SC_SERVICE_UNAVAILABLE, + HttpStatus.SC_REQUEST_TIMEOUT, + HttpStatus.SC_GATEWAY_TIMEOUT)); + + public ServiceNowAPIException(String message, @Nullable HttpResponse httpResponse) { + super(message); + this.httpResponse = httpResponse; + this.manualRetry = false; + } + + public ServiceNowAPIException(Throwable t, @Nullable HttpResponse httpResponse) { + super(t); + this.httpResponse = httpResponse; + this.manualRetry = false; + + } + + public ServiceNowAPIException(String message, Throwable t, + @Nullable HttpResponse httpResponse, boolean manualRetry) { + super(message, t); + this.httpResponse = httpResponse; + this.manualRetry = manualRetry; + } + + public String getUnderlyingMessage() { + if (this.getCause() != null) { + return this.getCause().getMessage(); + } + return null; + } + + @Nullable + public HttpResponse getHttpResponse() { + return httpResponse; + } + + public int getStatusCode() { + if (httpResponse != null && httpResponse.getStatusLine() != null) { + return httpResponse.getStatusLine().getStatusCode(); + } + return 0; + } + + public boolean isErrorRetryable() { + if (manualRetry) { + return true; + } + Throwable t = this.getCause(); + return t instanceof OAuthSystemException + || (this.getMessage() != null + && this.getMessage().contains(ServiceNowConstants.MAXIMUM_EXECUTION_TIME_EXCEEDED)) + || RETRYABLE_CODES.contains(getStatusCode()); + } +} diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java index 2e2dcc6..2971be6 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java @@ -56,6 +56,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; import javax.annotation.Nullable; /** @@ -76,10 +77,18 @@ public ServiceNowTableAPIClientImpl(ServiceNowConnectorConfig conf) { this.conf = conf; } - public String getAccessToken() throws OAuthSystemException, OAuthProblemException { - return generateAccessToken(String.format(OAUTH_URL_TEMPLATE, conf.getRestApiEndpoint()), - conf.getClientId(), - conf.getClientSecret(), conf.getUser(), conf.getPassword()); + public String getAccessToken() throws ServiceNowAPIException { + try { + return generateAccessToken(String.format(OAUTH_URL_TEMPLATE, conf.getRestApiEndpoint()), + conf.getClientId(), + conf.getClientSecret(), conf.getUser(), conf.getPassword()); + } catch (OAuthProblemException | OAuthSystemException e) { + throw new ServiceNowAPIException("An error occurred while authenticating.", e, null, false); + } + } + + private boolean isExceptionRetryable(Throwable t) { + return t instanceof ServiceNowAPIException && ((ServiceNowAPIException) t).isErrorRetryable(); } /** @@ -90,6 +99,7 @@ public String getAccessTokenRetryableMode() throws ExecutionException, RetryExce Callable fetchToken = this::getAccessToken; Retryer retryer = RetryerBuilder.newBuilder() + .retryIfException(this::isExceptionRetryable) .retryIfExceptionOfType(OAuthSystemException.class) .withWaitStrategy(WaitStrategies.fixedWait(ServiceNowConstants.BASE_DELAY, TimeUnit.MILLISECONDS)) .withStopStrategy(StopStrategies.stopAfterAttempt(ServiceNowConstants.MAX_NUMBER_OF_RETRY_ATTEMPTS)) @@ -109,8 +119,14 @@ public String getAccessTokenRetryableMode() throws ExecutionException, RetryExce * @param limit The number of records to be fetched * @return The list of Map; each Map representing a table row */ - public List> fetchTableRecords(String tableName, SourceValueType valueType, String startDate, - String endDate, int offset, int limit) throws IOException { + public List> fetchTableRecords( + String tableName, + SourceValueType valueType, + String startDate, + String endDate, + int offset, + int limit) + throws ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( this.conf.getRestApiEndpoint(), tableName, false) .setExcludeReferenceLink(true) @@ -123,16 +139,10 @@ public List> fetchTableRecords(String tableName, SourceValue applyDateRangeToRequest(requestBuilder, startDate, endDate); - try { - String accessToken = getAccessToken(); - requestBuilder.setAuthHeader(accessToken); - RestAPIResponse apiResponse = executeGet(requestBuilder.build()); - return parseResponseToResultListOfMap(apiResponse.getResponseBody()); - } catch (OAuthSystemException e) { - throw new RetryableException("Authentication error occurred", e); - } catch (OAuthProblemException e) { - throw new IOException("Problem occurred while authenticating", e); - } + String accessToken = getAccessToken(); + requestBuilder.setAuthHeader(accessToken); + RestAPIResponse apiResponse = executeGetWithRetries(requestBuilder.build()); + return parseResponseToResultListOfMap(apiResponse.getResponseBody()); } private void applyDateRangeToRequest(ServiceNowTableAPIRequestBuilder requestBuilder, String startDate, @@ -212,7 +222,7 @@ private String getErrorMessage(String responseBody) { */ public List> fetchTableRecordsRetryableMode(String tableName, SourceValueType valueType, String startDate, String endDate, int offset, - int limit) throws IOException { + int limit) throws ServiceNowAPIException { final List> results = new ArrayList<>(); Callable fetchRecords = () -> { results.addAll(fetchTableRecords(tableName, valueType, startDate, endDate, offset, limit)); @@ -220,7 +230,7 @@ public List> fetchTableRecordsRetryableMode(String tableName }; Retryer retryer = RetryerBuilder.newBuilder() - .retryIfExceptionOfType(RetryableException.class) + .retryIfException(this::isExceptionRetryable) .withWaitStrategy(WaitStrategies.exponentialWait(ServiceNowConstants.WAIT_TIME, TimeUnit.MILLISECONDS)) .withStopStrategy(StopStrategies.stopAfterAttempt(ServiceNowConstants.MAX_NUMBER_OF_RETRY_ATTEMPTS)) .build(); @@ -228,7 +238,9 @@ public List> fetchTableRecordsRetryableMode(String tableName try { retryer.call(fetchRecords); } catch (RetryException | ExecutionException e) { - throw new IOException(String.format("Data Recovery failed for batch %s to %s.", offset, (offset + limit)), e); + throw new ServiceNowAPIException( + String.format("Data Recovery failed for batch %s to %s.", offset, (offset + limit)), + e, null, false); } return results; @@ -262,12 +274,11 @@ public SchemaResponse parseSchemaResponse(String responseBody) { * * @param tableName ServiceNow table name for which schema is getting fetched * @return schema for given ServiceNow table - * @throws OAuthProblemException - * @throws OAuthSystemException + * @throws ServiceNowAPIException */ public Schema fetchTableSchema(String tableName) - throws OAuthProblemException, OAuthSystemException, IOException { - return fetchTableSchema(tableName, getAccessToken()); + throws ServiceNowAPIException { + return fetchTableSchema(tableName, getAccessToken()); } /** @@ -277,14 +288,15 @@ public Schema fetchTableSchema(String tableName) * @param accessToken Access Token to use * @return schema for given ServiceNow table */ - public Schema fetchTableSchema(String tableName, String accessToken) throws IOException { + public Schema fetchTableSchema(String tableName, String accessToken) + throws ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( this.conf.getRestApiEndpoint(), tableName, true) .setExcludeReferenceLink(true); RestAPIResponse apiResponse; requestBuilder.setAuthHeader(accessToken); - apiResponse = executeGet(requestBuilder.build()); + apiResponse = executeGetWithRetries(requestBuilder.build()); SchemaResponse response = parseSchemaResponse(apiResponse.getResponseBody()); List columns = new ArrayList<>(); @@ -302,11 +314,10 @@ public Schema fetchTableSchema(String tableName, String accessToken) throws IOEx * * @param tableName ServiceNow table name for which record count is fetched. * @return the table record count - * @throws OAuthProblemException - * @throws OAuthSystemException + * @throws ServiceNowAPIException */ public int getTableRecordCount(String tableName) - throws OAuthProblemException, OAuthSystemException, IOException { + throws ServiceNowAPIException { return getTableRecordCount(tableName, getAccessToken()); } @@ -316,9 +327,9 @@ public int getTableRecordCount(String tableName) * @param tableName ServiceNow table name for which record count is fetched. * @param accessToken Access Token for the call * @return the table record count - * @throws IOException + * @throws ServiceNowAPIException */ - public int getTableRecordCount(String tableName, String accessToken) throws IOException { + public int getTableRecordCount(String tableName, String accessToken) throws ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( this.conf.getRestApiEndpoint(), tableName, false) .setExcludeReferenceLink(true) @@ -327,7 +338,7 @@ public int getTableRecordCount(String tableName, String accessToken) throws IOEx RestAPIResponse apiResponse = null; requestBuilder.setResponseHeaders(ServiceNowConstants.HEADER_NAME_TOTAL_COUNT); requestBuilder.setAuthHeader(accessToken); - apiResponse = executeGet(requestBuilder.build()); + apiResponse = executeGetWithRetries(requestBuilder.build()); return getRecordCountFromHeader(apiResponse); } @@ -338,7 +349,7 @@ public int getTableRecordCount(String tableName, String accessToken) throws IOEx * @param entity Details of the Record to be created * @description This function is being used in end-to-end (e2e) tests to fetch a record from the ServiceNow Table. */ - public String createRecord(String tableName, HttpEntity entity) throws IOException { + public String createRecord(String tableName, HttpEntity entity) throws IOException, ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( this.conf.getRestApiEndpoint(), tableName, false); String systemID; @@ -352,8 +363,8 @@ public String createRecord(String tableName, HttpEntity entity) throws IOExcepti apiResponse = executePost(requestBuilder.build()); systemID = String.valueOf(getSystemId(apiResponse)); - } catch (OAuthSystemException | OAuthProblemException | UnsupportedEncodingException e) { - throw new IOException("Error in creating a new record", e); + } catch (IOException e) { + throw new ServiceNowAPIException("Error in creating a new record", e, null, false); } return systemID; } @@ -372,7 +383,7 @@ private String getSystemId(RestAPIResponse restAPIResponse) { * @param query The query */ public Map getRecordFromServiceNowTable(String tableName, String query) - throws OAuthProblemException, OAuthSystemException, IOException { + throws ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( this.conf.getRestApiEndpoint(), tableName, false) @@ -381,7 +392,7 @@ public Map getRecordFromServiceNowTable(String tableName, String RestAPIResponse restAPIResponse; String accessToken = getAccessToken(); requestBuilder.setAuthHeader(accessToken); - restAPIResponse = executeGet(requestBuilder.build()); + restAPIResponse = executeGetWithRetries(requestBuilder.build()); APIResponse apiResponse = GSON.fromJson(restAPIResponse.getResponseBody(), APIResponse.class); return apiResponse.getResult().get(0); diff --git a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java index 5f1061c..b2331ee 100644 --- a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java +++ b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java @@ -40,6 +40,7 @@ import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.Constants; import io.cdap.plugin.common.ReferenceNames; +import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIRequestBuilder; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; @@ -93,7 +94,7 @@ public BrowseDetail browse(ConnectorContext connectorContext, BrowseRequest brow try { String accessToken = serviceNowTableAPIClient.getAccessToken(); return browse(connectorContext, accessToken); - } catch (OAuthSystemException | OAuthProblemException e) { + } catch (ServiceNowAPIException e) { throw new IOException(e); } } @@ -102,7 +103,7 @@ public BrowseDetail browse(ConnectorContext connectorContext, BrowseRequest brow * Browse Details for the given AccessToken. */ public BrowseDetail browse(ConnectorContext connectorContext, - String accessToken) throws IOException { + String accessToken) throws ServiceNowAPIException { int count = 0; FailureCollector collector = connectorContext.getFailureCollector(); config.validateCredentialsFields(collector); @@ -125,14 +126,15 @@ public BrowseDetail browse(ConnectorContext connectorContext, /** * @return the list of tables. */ - private TableList listTables(String accessToken) throws IOException { + private TableList listTables(String accessToken) throws ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( config.getRestApiEndpoint(), OBJECT_TABLE_LIST, false); requestBuilder.setAuthHeader(accessToken); requestBuilder.setAcceptHeader(MediaType.APPLICATION_JSON); requestBuilder.setContentTypeHeader(MediaType.APPLICATION_JSON); ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(config); - RestAPIResponse apiResponse = serviceNowTableAPIClient.executeGet(requestBuilder.build()); + RestAPIResponse apiResponse = + serviceNowTableAPIClient.executeGetWithRetries(requestBuilder.build()); return GSON.fromJson(apiResponse.getResponseBody(), TableList.class); } @@ -164,13 +166,13 @@ public List sample(ConnectorContext connectorContext, SampleRe } try { return getTableData(table, sampleRequest.getLimit()); - } catch (OAuthProblemException | OAuthSystemException e) { - throw new IOException("Unable to fetch the data."); + } catch (OAuthProblemException | OAuthSystemException | ServiceNowAPIException e) { + throw new IOException("Unable to fetch the data.", e); } } private List getTableData(String tableName, int limit) - throws OAuthProblemException, OAuthSystemException, IOException { + throws OAuthProblemException, OAuthSystemException, ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( config.getRestApiEndpoint(), tableName, false) .setExcludeReferenceLink(true) @@ -180,7 +182,7 @@ private List getTableData(String tableName, int limit) String accessToken = serviceNowTableAPIClient.getAccessToken(); requestBuilder.setAuthHeader(accessToken); requestBuilder.setResponseHeaders(ServiceNowConstants.HEADER_NAME_TOTAL_COUNT); - RestAPIResponse apiResponse = serviceNowTableAPIClient.executeGet(requestBuilder.build()); + RestAPIResponse apiResponse = serviceNowTableAPIClient.executeGetWithRetries(requestBuilder.build()); List> result = serviceNowTableAPIClient.parseResponseToResultListOfMap (apiResponse.getResponseBody()); List recordList = new ArrayList<>(); diff --git a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java index 03581df..8cd8988 100644 --- a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java +++ b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIClient.java @@ -16,11 +16,14 @@ package io.cdap.plugin.servicenow.restapi; -import com.jcraft.jsch.IO; -import io.cdap.plugin.servicenow.apiclient.NonRetryableException; -import io.cdap.plugin.servicenow.apiclient.RetryableException; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; +import com.github.rholder.retry.Attempt; +import com.github.rholder.retry.RetryException; +import com.github.rholder.retry.Retryer; +import com.github.rholder.retry.RetryerBuilder; +import com.github.rholder.retry.StopStrategies; +import com.github.rholder.retry.WaitStrategies; +import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; +import io.cdap.plugin.servicenow.util.ServiceNowConstants; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; @@ -38,11 +41,9 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; /** * An abstract class to call Rest API. @@ -67,6 +68,60 @@ public RestAPIResponse executeGet(RestAPIRequest request) throws IOException { } } + /** + * Executes the Rest API request and returns the response with retries. + * + * @param request the Rest API request. + * @return an instance of RestAPIResponse object. + * @throws ServiceNowAPIException + */ + public RestAPIResponse executeGetWithRetries(RestAPIRequest request) + throws ServiceNowAPIException { + Callable callable = () -> executeGet(request); + return handleExecution(getRetryer(), callable); + } + + private RestAPIResponse handleExecution( + Retryer retryer, Callable callable) + throws ServiceNowAPIException { + try { + RestAPIResponse response = retryer.call(callable); + // Execution is successful + if (response.hasException()) { + // Execution is successful and returned non retryable error + throw response.getException(); + } + return response; + } catch (RetryException e) { + // Execution successful, returned retryable error and retries exhausted + Attempt apiResponseAttempt = e.getLastFailedAttempt(); + if (apiResponseAttempt.hasException()) { + // last attempt has execution failure + throw new ServiceNowAPIException(apiResponseAttempt.getExceptionCause(), null); + } else { + // last execution attempt was successful but has an error response + // if execution is successful, it's expected to have a exception in response object + RestAPIResponse response = (RestAPIResponse) apiResponseAttempt.getResult(); + throw response.getException(); + } + } catch (ExecutionException e) { + // Execution failed with error + throw new ServiceNowAPIException(e, null); + } + } + + private Retryer getRetryer() { + return RetryerBuilder.newBuilder() + .retryIfResult( + restAPIResponse -> + restAPIResponse.hasException() && restAPIResponse.getException().isErrorRetryable()) + .withWaitStrategy( + WaitStrategies.exponentialWait(ServiceNowConstants.WAIT_TIME, TimeUnit.MILLISECONDS)) + .withStopStrategy( + StopStrategies.stopAfterAttempt(ServiceNowConstants.MAX_NUMBER_OF_RETRY_ATTEMPTS)) + .build(); + } + /** * Executes the Rest API request and returns the response. * diff --git a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java index be0a0b1..44a0d0c 100644 --- a/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java +++ b/src/main/java/io/cdap/plugin/servicenow/restapi/RestAPIResponse.java @@ -18,8 +18,7 @@ import com.google.gson.Gson; import com.google.gson.JsonObject; -import io.cdap.plugin.servicenow.apiclient.NonRetryableException; -import io.cdap.plugin.servicenow.apiclient.RetryableException; +import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import org.apache.http.Header; import org.apache.http.HttpResponse; @@ -27,7 +26,6 @@ import org.apache.http.util.EntityUtils; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -36,6 +34,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import javax.annotation.Nullable; /** * Pojo class to capture the API response. @@ -45,55 +44,60 @@ public class RestAPIResponse { private static final String HTTP_ERROR_MESSAGE = "Http call to ServiceNow instance returned status code %d."; private static final String REST_ERROR_MESSAGE = "Rest Api response has errors. Error message: %s."; private static final Set SUCCESS_CODES = new HashSet<>(Collections.singletonList(HttpStatus.SC_OK)); - private static final Set RETRYABLE_CODES = new HashSet<>(Arrays.asList(429, - HttpStatus.SC_BAD_GATEWAY, - HttpStatus.SC_SERVICE_UNAVAILABLE, - HttpStatus.SC_REQUEST_TIMEOUT, - HttpStatus.SC_GATEWAY_TIMEOUT)); - private final int httpStatus; private final Map headers; private final String responseBody; + @Nullable private final ServiceNowAPIException exception; - public RestAPIResponse(int httpStatus, Map headers, String responseBody) { - this.httpStatus = httpStatus; + public RestAPIResponse( + Map headers, + @Nullable String responseBody, + @Nullable ServiceNowAPIException exception) { this.headers = headers; this.responseBody = responseBody; + this.exception = exception; } /** * Parses HttpResponse into RestAPIResponse object when no errors occur. - * Throws a {@link RetryableException} if the error is retryable. - * Throws an {@link NonRetryableException} if the error is not retryable. + * Throws a {@link ServiceNowAPIException}. * * @param httpResponse The HttpResponse object to parse * @param headerNames The list of header names to be extracted * @return An instance of RestAPIResponse object. */ - public static RestAPIResponse parse(HttpResponse httpResponse, String... headerNames) throws IOException { - validateHttpResponse(httpResponse); - List headerNameList = headerNames == null ? Collections.emptyList() : Arrays.asList(headerNames); - int httpStatus = httpResponse.getStatusLine().getStatusCode(); + public static RestAPIResponse parse(HttpResponse httpResponse, String... headerNames) { + List headerNameList = + headerNames == null ? Collections.emptyList() : Arrays.asList(headerNames); Map headers = new HashMap<>(); if (!headerNameList.isEmpty()) { - headers.putAll(Arrays.stream(httpResponse.getAllHeaders()) - .filter(o -> headerNameList.contains(o.getName())) - .collect(Collectors.toMap(Header::getName, Header::getValue))); + headers.putAll( + Arrays.stream(httpResponse.getAllHeaders()) + .filter(o -> headerNameList.contains(o.getName())) + .collect(Collectors.toMap(Header::getName, Header::getValue))); } - String responseBody = EntityUtils.toString(httpResponse.getEntity()); - validateRestApiResponse(responseBody); - return new RestAPIResponse(httpStatus, headers, responseBody); + + ServiceNowAPIException serviceNowAPIException = validateHttpResponse(httpResponse); + if (serviceNowAPIException != null) { + return new RestAPIResponse(headers, null, serviceNowAPIException); + } + + String responseBody = null; + try { + responseBody = EntityUtils.toString(httpResponse.getEntity()); + } catch (IOException e) { + return new RestAPIResponse(headers, null, new ServiceNowAPIException(e, httpResponse)); + } + serviceNowAPIException = validateRestApiResponse(httpResponse, responseBody); + return new RestAPIResponse(headers, responseBody, serviceNowAPIException); } public static RestAPIResponse parse(HttpResponse httpResponse) throws IOException { return parse(httpResponse, new String[0]); } - public int getHttpStatus() { - return httpStatus; - } - - private static void validateRestApiResponse(String responseBody) { + private static ServiceNowAPIException validateRestApiResponse( + HttpResponse response, String responseBody) { JsonObject jo = GSON.fromJson(responseBody, JsonObject.class); // check if status is "failure" String status = null; @@ -101,33 +105,37 @@ private static void validateRestApiResponse(String responseBody) { status = jo.get(ServiceNowConstants.STATUS).getAsString(); } if (!ServiceNowConstants.FAILURE.equals(status)) { - return; + return null; } // check if failure is retryable String errorMessage = jo.getAsJsonObject(ServiceNowConstants.ERROR).get(ServiceNowConstants.MESSAGE).getAsString(); - if (errorMessage.contains(ServiceNowConstants.MAXIMUM_EXECUTION_TIME_EXCEEDED)) { - throw new RetryableException(String.format(REST_ERROR_MESSAGE, errorMessage)); - } else { - throw new NonRetryableException(String.format(REST_ERROR_MESSAGE, errorMessage)); - } + return new ServiceNowAPIException(String.format(REST_ERROR_MESSAGE, errorMessage), response); } - private static void validateHttpResponse(HttpResponse response) { + private static ServiceNowAPIException validateHttpResponse(HttpResponse response) { int code = response.getStatusLine().getStatusCode(); if (SUCCESS_CODES.contains(code)) { - return; - } - if (RETRYABLE_CODES.contains(code)) { - throw new RetryableException(String.format(HTTP_ERROR_MESSAGE, code)); + return null; } - throw new NonRetryableException(String.format(HTTP_ERROR_MESSAGE, code)); + return new ServiceNowAPIException( + String.format(HTTP_ERROR_MESSAGE, code), response); } public Map getHeaders() { return headers; } + @Nullable public String getResponseBody() { return responseBody; } + + @Nullable + public ServiceNowAPIException getException() { + return exception; + } + + public boolean hasException() { + return exception != null; + } } diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java b/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java index 1662ca8..af346c4 100644 --- a/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java @@ -26,6 +26,7 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import io.cdap.cdap.api.retry.RetryableException; +import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIRequestBuilder; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; @@ -107,7 +108,8 @@ public RestRequest getRestRequest(JsonObject jsonObject) { * @param restRequestsMap The map of restRequests * @param accessToken The access token */ - public void createPostRequest(Map restRequestsMap, String accessToken) { + public void createPostRequest(Map restRequestsMap, String accessToken) + throws ServiceNowAPIException { ServiceNowBatchRequest payloadRequest = getPayloadRequest(restRequestsMap); ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( config.getConnection().getRestApiEndpoint()); @@ -164,7 +166,7 @@ public void createPostRequest(Map restRequestsMap, String a } } catch (IOException e) { LOG.error("Error while connecting to ServiceNow", e.getMessage()); - throw new RetryableException("Error while connecting to ServiceNow", e); + throw new ServiceNowAPIException("Error while connecting to ServiceNow", e, null, true); } } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java index 25a2b8b..a77ad33 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java @@ -18,6 +18,7 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.util.ServiceNowConstants; @@ -104,7 +105,7 @@ private static ServiceNowTableInfo getTableMetaData(String tableName, ServiceNow try { schema = restApi.fetchTableSchema(tableName); recordCount = restApi.getTableRecordCount(tableName); - } catch (OAuthProblemException | OAuthSystemException | IOException e) { + } catch (ServiceNowAPIException e) { throw new RuntimeException(String.format("Error in fetching table metadata due to reason: %s", e.getMessage()), e); } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java index cadefe3..0d492c6 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java @@ -19,6 +19,7 @@ import com.google.common.base.Strings; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.util.ServiceNowConstants; @@ -97,7 +98,7 @@ private static ServiceNowTableInfo getTableMetaData(String tableName, ServiceNow try { schema = restApi.fetchTableSchema(tableName); recordCount = restApi.getTableRecordCount(tableName); - } catch (OAuthProblemException | OAuthSystemException | IOException e) { + } catch (ServiceNowAPIException e) { throw new RuntimeException(e); } LOG.debug("table {}, rows = {}", tableName, recordCount); diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java index 364485a..09f87e6 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java @@ -19,6 +19,7 @@ import com.google.common.annotations.VisibleForTesting; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowRecordConverter; import io.cdap.plugin.servicenow.util.ServiceNowConstants; @@ -94,7 +95,7 @@ public StructuredRecord getCurrentValue() throws IOException { } @VisibleForTesting - void fetchData() throws IOException { + void fetchData() throws ServiceNowAPIException { // Get the table data results = restApi.fetchTableRecordsRetryableMode(tableName, multiSourcePluginConf.getValueType(), multiSourcePluginConf.getStartDate(), @@ -112,7 +113,7 @@ private void fetchSchema(ServiceNowTableAPIClientImpl restApi) { List schemaFields = new ArrayList<>(tableFields); schemaFields.add(Schema.Field.of(tableNameField, Schema.of(Schema.Type.STRING))); schema = Schema.recordOf(tableName, schemaFields); - } catch (OAuthProblemException | OAuthSystemException | IOException e) { + } catch (ServiceNowAPIException e) { throw new RuntimeException(e); } } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java index 16459a6..511d9cc 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java @@ -18,6 +18,7 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowRecordConverter; import io.cdap.plugin.servicenow.util.ServiceNowTableInfo; @@ -102,7 +103,7 @@ public StructuredRecord getCurrentValue() throws IOException { return recordBuilder.build(); } - private void fetchData() throws IOException { + private void fetchData() throws ServiceNowAPIException { // Get the table data results = restApi.fetchTableRecordsRetryableMode(tableName, pluginConf.getValueType(), pluginConf.getStartDate(), pluginConf.getEndDate(), split.getOffset(), diff --git a/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java b/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java new file mode 100644 index 0000000..99640a5 --- /dev/null +++ b/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java @@ -0,0 +1,82 @@ +package io.cdap.plugin.servicenow.apiclient; + +import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; +import io.cdap.plugin.servicenow.util.SourceValueType; + +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +public class ServiceNowTableAPIClientImplTest { + + @Rule + public ExpectedException exceptionRule = ExpectedException.none(); + @Test + public void testFetchTableRecordsRetryableMode_RetriesAndSucceeds() throws ServiceNowAPIException { + ServiceNowConnectorConfig mockConfig = Mockito.mock(ServiceNowConnectorConfig.class); + ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig); + ServiceNowTableAPIClientImpl implSpy = Mockito.spy(impl); + List> mockResults = new ArrayList<>(); + mockResults.add(new HashMap() {{ + put("keyTest", "valueTest"); + }}); + HttpResponse mockResponse = Mockito.mock(HttpResponse.class); + Mockito.when(mockResponse.getStatusLine()).thenReturn(Mockito.mock(StatusLine.class)); + Mockito.when(mockResponse.getStatusLine().getStatusCode()).thenReturn(HttpStatus.SC_REQUEST_TIMEOUT); + Mockito.doThrow(new ServiceNowAPIException("Retryable Error", mockResponse)) + .doReturn(mockResults) + .when(implSpy).fetchTableRecords( + Mockito.anyString(), + Mockito.any(), + Mockito.anyString(), + Mockito.anyString(), + Mockito.anyInt(), + Mockito.anyInt()); + List> receivedResults = + implSpy.fetchTableRecordsRetryableMode( + "test", SourceValueType.SHOW_DISPLAY_VALUE, "", "", 0, 0); + Mockito.verify(implSpy, Mockito.times(2)).fetchTableRecords( + Mockito.anyString(), + Mockito.any(), + Mockito.anyString(), + Mockito.anyString(), + Mockito.anyInt(), + Mockito.anyInt()); + Assert.assertEquals(receivedResults, mockResults); + } + + @Test + public void testFetchTableRecordsRetryableMode_nonRetryable() + throws ServiceNowAPIException { + ServiceNowConnectorConfig mockConfig = Mockito.mock(ServiceNowConnectorConfig.class); + ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig); + ServiceNowTableAPIClientImpl implSpy = Mockito.spy(impl); + HttpResponse mockResponse = Mockito.mock(HttpResponse.class); + Mockito.when(mockResponse.getStatusLine()).thenReturn(Mockito.mock(StatusLine.class)); + Mockito.when(mockResponse.getStatusLine().getStatusCode()).thenReturn(HttpStatus.SC_INTERNAL_SERVER_ERROR); + Mockito.doThrow( + new ServiceNowAPIException("Non-retryable Error", mockResponse)) + .doReturn(new ArrayList<>()) + .when(implSpy).fetchTableRecords( + Mockito.anyString(), + Mockito.any(), + Mockito.anyString(), + Mockito.anyString(), + Mockito.anyInt(), + Mockito.anyInt()); + exceptionRule.expect(ServiceNowAPIException.class); + exceptionRule.expectMessage("Data Recovery failed for batch 0 to 0."); + implSpy.fetchTableRecordsRetryableMode( + "test", SourceValueType.SHOW_DISPLAY_VALUE, "", "", 0, 0); + } +} diff --git a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java index bfa2ec7..1d2c373 100644 --- a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java @@ -133,8 +133,8 @@ public void testGenerateSpec() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). diff --git a/src/test/java/io/cdap/plugin/servicenow/restapi/RestAPIClientTest.java b/src/test/java/io/cdap/plugin/servicenow/restapi/RestAPIClientTest.java index 9c72328..da15c5b 100644 --- a/src/test/java/io/cdap/plugin/servicenow/restapi/RestAPIClientTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/restapi/RestAPIClientTest.java @@ -1,7 +1,5 @@ package io.cdap.plugin.servicenow.restapi; -import io.cdap.plugin.servicenow.apiclient.NonRetryableException; -import io.cdap.plugin.servicenow.apiclient.RetryableException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIRequestBuilder; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; @@ -12,6 +10,7 @@ import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.message.BasicStatusLine; import org.apache.http.util.EntityUtils; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mockito; @@ -31,7 +30,7 @@ }) public class RestAPIClientTest { - @Test(expected = RetryableException.class) + @Test public void testExecuteGet_throwRetryableException() throws IOException { CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class); StatusLine statusLine = Mockito.mock(BasicStatusLine.class); @@ -50,11 +49,13 @@ public void testExecuteGet_throwRetryableException() throws IOException { ServiceNowConnectorConfig config = Mockito.mock(ServiceNowConnectorConfig.class); ServiceNowTableAPIClientImpl client = new ServiceNowTableAPIClientImpl(config); - client.executeGet(request); + RestAPIResponse actualResponse = client.executeGet(request); + Assert.assertNotNull(actualResponse.getException()); + Assert.assertTrue(actualResponse.getException().isErrorRetryable()); } - @Test(expected = NonRetryableException.class) - public void testExecuteGet_throwIOException() throws IOException { + @Test + public void testExecuteGet_throwNonRetryableException() throws IOException { CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class); StatusLine statusLine = Mockito.mock(BasicStatusLine.class); Mockito.when(statusLine.getStatusCode()).thenReturn(HttpStatus.SC_INTERNAL_SERVER_ERROR); @@ -72,7 +73,9 @@ public void testExecuteGet_throwIOException() throws IOException { ServiceNowConnectorConfig config = Mockito.mock(ServiceNowConnectorConfig.class); ServiceNowTableAPIClientImpl client = new ServiceNowTableAPIClientImpl(config); - client.executeGet(request); + RestAPIResponse actualResponse = client.executeGet(request); + Assert.assertNotNull(actualResponse.getException()); + Assert.assertFalse(actualResponse.getException().isErrorRetryable()); } @Test diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java index 397de04..1a0ce79 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java @@ -17,6 +17,7 @@ import com.google.gson.JsonObject; import io.cdap.plugin.servicenow.ServiceNowBaseConfig; +import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.restapi.RestAPIClient; @@ -97,9 +98,9 @@ public void testWriteWithUnSuccessfulApiResponse() throws Exception { List> result = new ArrayList<>(); map.put("key", "value"); result.add(map); - int httpStatus = HttpStatus.SC_INTERNAL_SERVER_ERROR; Map headers = new HashMap<>(); - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); + RestAPIResponse restAPIResponse = new RestAPIResponse( + headers, responseBody, null); Mockito.when(restApi.executePost(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); @@ -118,7 +119,8 @@ public void testWriteWithUnSuccessfulApiResponse() throws Exception { Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse); ServiceNowRecordWriter serviceNowRecordWriter = new ServiceNowRecordWriter(serviceNowSinkConfig); serviceNowRecordWriter.write(null, jsonObject); - Assert.assertEquals(500, restAPIResponse.getHttpStatus()); +// Assert.assertNotNull(restAPIResponse.getException()); +// Assert.assertFalse(restAPIResponse.getException().isErrorRetryable()); } @Test @@ -143,9 +145,8 @@ public void testWriteWithSuccessFulApiResponse() throws Exception { List> result = new ArrayList<>(); map.put("key", "value"); result.add(map); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); Mockito.when(restApi.executePost(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); @@ -168,7 +169,7 @@ public void testWriteWithSuccessFulApiResponse() throws Exception { thenReturn(response); ServiceNowRecordWriter serviceNowRecordWriter = new ServiceNowRecordWriter(serviceNowSinkConfig); serviceNowRecordWriter.write(null, jsonObject); - Assert.assertEquals(200, restAPIResponse.getHttpStatus()); + Assert.assertNull(restAPIResponse.getException()); } @Test @@ -193,9 +194,8 @@ public void testWriteWithUnservicedRequests() throws Exception { List> result = new ArrayList<>(); map.put("key", "value"); result.add(map); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); Mockito.when(restApi.executePost(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); @@ -218,6 +218,6 @@ public void testWriteWithUnservicedRequests() throws Exception { thenReturn(response); ServiceNowRecordWriter serviceNowRecordWriter = new ServiceNowRecordWriter(serviceNowSinkConfig); serviceNowRecordWriter.write(null, jsonObject); - Assert.assertEquals(200, restAPIResponse.getHttpStatus()); + Assert.assertNull(restAPIResponse.getException()); } } diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java index 70e359d..42841db 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java @@ -21,6 +21,7 @@ import io.cdap.cdap.etl.api.validation.ValidationException; import io.cdap.cdap.etl.api.validation.ValidationFailure; import io.cdap.cdap.etl.mock.validation.MockFailureCollector; +import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.restapi.RestAPIClient; @@ -29,6 +30,7 @@ import io.cdap.plugin.servicenow.sink.model.SchemaResponse; import io.cdap.plugin.servicenow.sink.model.ServiceNowSchemaField; import io.cdap.plugin.servicenow.util.ServiceNowConstants; +import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.StatusLine; import org.apache.http.client.methods.CloseableHttpResponse; @@ -285,6 +287,7 @@ public void testValidateSchema() throws Exception { int httpStatus = HttpStatus.SC_NOT_FOUND; Map headers = new HashMap<>(); String responseBody = "{\n" + + " \"status_code\":201 " + " \"result\": [\n" + " {\n" + " \"calendar_integration\": \"1\",\n" + @@ -299,7 +302,11 @@ public void testValidateSchema() throws Exception { List schemaFields = new ArrayList<>(); schemaFields.add(schemaField); SchemaResponse schemaResponse = new SchemaResponse(schemaFields); - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); + HttpResponse mockResponse = Mockito.mock(HttpResponse.class); + Mockito.when(mockResponse.getStatusLine()).thenReturn(Mockito.mock(StatusLine.class)); + Mockito.when(mockResponse.getStatusLine().getStatusCode()).thenReturn(httpStatus); + RestAPIResponse restAPIResponse = new RestAPIResponse( + headers, responseBody, new ServiceNowAPIException("", mockResponse)); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); @@ -317,7 +324,7 @@ public void testValidateSchema() throws Exception { CloseableHttpResponse httpResponse = Mockito.mock(CloseableHttpResponse.class); Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse); PowerMockito.when(RestAPIResponse.parse(httpResponse, null)).thenReturn(response); - Mockito.when(restApi.executeGet(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); + Mockito.when(restApi.executeGetWithRetries(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); Mockito.when(restApi.fetchTableSchema(Mockito.anyString(), Mockito.any(FailureCollector.class))).thenReturn(schema); Mockito.when(restApi.parseSchemaResponse(restAPIResponse.getResponseBody())) .thenReturn(schemaResponse); @@ -348,7 +355,6 @@ public void testValidateSchemaWithOperation() throws Exception { Map map = new HashMap<>(); map.put("key", "value"); result.add(map); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" + @@ -360,7 +366,7 @@ public void testValidateSchemaWithOperation() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); @@ -381,7 +387,7 @@ public void testValidateSchemaWithOperation() throws Exception { Mockito.when(httpResponse.getStatusLine()).thenReturn(statusLine); Mockito.when(httpClient.execute(Mockito.any())).thenReturn(httpResponse); PowerMockito.when(RestAPIResponse.parse(httpResponse, null)).thenReturn(response); - Mockito.when(restApi.executeGet(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); + Mockito.when(restApi.executeGetWithRetries(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); Mockito.when(restApi.fetchTableSchema("tableName", collector)). thenReturn(schema); config.validateSchema(schema, collector); diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java index f64d1db..1a3f294 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java @@ -89,17 +89,16 @@ public void testConfigurePipeline() throws Exception { PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); List> result = new ArrayList<>(); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": []\n" + "}"; MockFailureCollector collector = new MockFailureCollector(); - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); serviceNowSink.configurePipeline(mockPipelineConfigurer); - Assert.assertEquals(200, restAPIResponse.getHttpStatus()); + Assert.assertNull(restAPIResponse.getException()); Assert.assertEquals(0, collector.getValidationFailures().size()); } @@ -117,7 +116,6 @@ public void testPrepareRun() throws Exception { Map map = new HashMap<>(); map.put("key", "value"); result.add(map); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" + @@ -134,8 +132,8 @@ public void testPrepareRun() throws Exception { Schema.Field.of("price", Schema.of(Schema.Type.DOUBLE))); Emitter> emitter = Mockito.mock(Emitter.class); Mockito.when(context.getInputSchema()).thenReturn(schema); - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java index 657d128..f6e2fea 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java @@ -75,7 +75,6 @@ public void testFetchTableInfo() throws Exception { Map map = new HashMap<>(); map.put("key", "value"); result.add(map); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" + @@ -139,8 +138,8 @@ public void testFetchTableInfo() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). @@ -175,7 +174,6 @@ public void testFetchTableInfoReportingMode() throws Exception { Map map = new HashMap<>(); map.put("key", "value"); result.add(map); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" + @@ -239,8 +237,8 @@ public void testFetchTableInfoReportingMode() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). @@ -275,13 +273,12 @@ public void testFetchTableInfoWithEmptyTableName() throws Exception { Map map = new HashMap<>(); map.put("key", "value"); result.add(map); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.mockStatic(ServiceNowInputFormat.class); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java index 7614e2e..975d367 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java @@ -20,6 +20,7 @@ import io.cdap.cdap.api.data.format.UnexpectedFormatException; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.plugin.PluginProperties; +import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableDataResponse; import io.cdap.plugin.servicenow.connector.ServiceNowRecordConverter; @@ -120,7 +121,7 @@ public void testConvertToBooleanValueForInvalidFieldValue() { } @Test - public void testFetchData() throws IOException { + public void testFetchData() throws ServiceNowAPIException, IOException { String tableName = serviceNowMultiSourceConfig.getTableNames(); ServiceNowInputSplit split = new ServiceNowInputSplit(tableName, 1); @@ -143,7 +144,8 @@ public void testFetchData() throws IOException { Mockito.when(restApi.fetchTableSchema(tableName)) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowMultiRecordReader.initialize(split, null); - } catch (RuntimeException | OAuthProblemException | OAuthSystemException e) { + } catch (RuntimeException + | ServiceNowAPIException e) { Assert.assertTrue(e instanceof RuntimeException); } Mockito.doNothing().when(serviceNowMultiRecordReader).fetchData(); @@ -153,22 +155,24 @@ public void testFetchData() throws IOException { Assert.assertTrue(serviceNowMultiRecordReader.nextKeyValue()); } - @Test(expected = IOException.class) - public void testFetchDataOnInvalidTable() throws IOException, OAuthProblemException, OAuthSystemException { - serviceNowMultiSourceConfig = ServiceNowSourceConfigHelper.newConfigBuilder() - .setReferenceName("referenceName") - .setRestApiEndpoint(REST_API_ENDPOINT) - .setUser(USER) - .setPassword(PASSWORD) - .setClientId(CLIENT_ID) - .setClientSecret(CLIENT_SECRET) - .setTableNames("") - .setValueType("Actual") - .setStartDate("2021-01-01") - .setEndDate("2022-02-18") - .setPageSize(10) - .setTableNameField("tablename") - .buildMultiSource(); + @Test(expected = ServiceNowAPIException.class) + public void testFetchDataOnInvalidTable() + throws IOException, OAuthProblemException, OAuthSystemException, ServiceNowAPIException { + serviceNowMultiSourceConfig = + ServiceNowSourceConfigHelper.newConfigBuilder() + .setReferenceName("referenceName") + .setRestApiEndpoint(REST_API_ENDPOINT) + .setUser(USER) + .setPassword(PASSWORD) + .setClientId(CLIENT_ID) + .setClientSecret(CLIENT_SECRET) + .setTableNames("") + .setValueType("Actual") + .setStartDate("2021-01-01") + .setEndDate("2022-02-18") + .setPageSize(10) + .setTableNameField("tablename") + .buildMultiSource(); String tableName = serviceNowMultiSourceConfig.getTableNames(); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); @@ -195,7 +199,7 @@ public void testFetchDataOnInvalidTable() throws IOException, OAuthProblemExcept Mockito.when(restApi.fetchTableSchema(tableName)) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowMultiRecordReader.initialize(split, null); - } catch (RuntimeException | OAuthProblemException | OAuthSystemException e) { + } catch (RuntimeException e) { Assert.assertTrue(e instanceof RuntimeException); } serviceNowMultiRecordReader.fetchData(); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java index 67fb025..9022f3a 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java @@ -110,7 +110,6 @@ public void testValidate() throws Exception { Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); Map map = new HashMap<>(); List> result = new ArrayList<>(); @@ -178,8 +177,8 @@ public void testValidate() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); serviceNowMultiSourceConfig.validate(mockFailureCollector); Assert.assertEquals(0, mockFailureCollector.getValidationFailures().size()); @@ -208,13 +207,12 @@ public void testValidateWhenTableIsEmpty() throws Exception { PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); serviceNowMultiSourceConfig.validate(mockFailureCollector); Assert.assertEquals(1, mockFailureCollector.getValidationFailures().size()); Assert.assertEquals("Table: sys_user is empty.", mockFailureCollector.getValidationFailures().get(0).getMessage()); @@ -241,7 +239,6 @@ public void testValidateReferenceName() throws Exception { Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); Map map = new HashMap<>(); List> result = new ArrayList<>(); @@ -309,8 +306,8 @@ public void testValidateReferenceName() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); try { serviceNowMultiSourceConfig.validate(mockFailureCollector); @@ -344,7 +341,6 @@ public void testValidateWhenTableFieldNameIsEmpty() throws Exception { Mockito.when(restApi.getAccessToken()).thenReturn("token"); PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); Map map = new HashMap<>(); List> result = new ArrayList<>(); @@ -412,8 +408,8 @@ public void testValidateWhenTableFieldNameIsEmpty() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); serviceNowMultiSourceConfig.validate(mockFailureCollector); Assert.assertEquals(1, mockFailureCollector.getValidationFailures().size()); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java index bb096ed..a87af2f 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java @@ -94,7 +94,6 @@ public void testConfigurePipeline() throws Exception { List> result = new ArrayList<>(); map.put("key", "value"); result.add(map); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" + @@ -158,8 +157,8 @@ public void testConfigurePipeline() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); serviceNowMultiSource.configurePipeline(mockPipelineConfigurer); Assert.assertNull(mockPipelineConfigurer.getOutputSchema()); @@ -175,13 +174,12 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); List> result = new ArrayList<>(); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); try { serviceNowMultiSource.configurePipeline(mockPipelineConfigurer); @@ -220,7 +218,6 @@ public void testPrepareRun() throws Exception { Map map = new HashMap<>(); map.put("key", "value"); result.add(map); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" + @@ -286,8 +283,8 @@ public void testPrepareRun() throws Exception { "}"; PowerMockito.mockStatic(ServiceNowMultiInputFormat.class); Mockito.when(ServiceNowMultiInputFormat.setInput(Mockito.any(), Mockito.any())).thenReturn((tableInfo)); - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java index 582c292..7a11cce 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java @@ -21,6 +21,7 @@ import io.cdap.cdap.etl.api.validation.ValidationException; import io.cdap.cdap.etl.api.validation.ValidationFailure; import io.cdap.cdap.etl.mock.validation.MockFailureCollector; +import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; @@ -28,7 +29,9 @@ import io.cdap.plugin.servicenow.util.SourceApplication; import io.cdap.plugin.servicenow.util.SourceQueryMode; import io.cdap.plugin.servicenow.util.SourceValueType; +import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -618,13 +621,12 @@ public void testValidateWhenTableIsEmpty() throws Exception { PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); config.validate(mockFailureCollector); Assert.assertEquals(1, mockFailureCollector.getValidationFailures().size()); Assert.assertEquals("Table: sys_user is empty.", mockFailureCollector.getValidationFailures().get(0).getMessage()); @@ -649,7 +651,11 @@ public void testValidateWhenTableNameIsInvalid() throws Exception { PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); String errorMessage = "Http call returned 400 response code"; - Mockito.when(restApi.executeGet(Mockito.any())).thenThrow(new IOException(errorMessage)); + HttpResponse mockResponse = Mockito.mock(HttpResponse.class); + Mockito.when(mockResponse.getStatusLine()).thenReturn(Mockito.mock(StatusLine.class)); + Mockito.when(mockResponse.getStatusLine().getStatusCode()).thenReturn(HttpStatus.SC_BAD_REQUEST); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())) + .thenThrow(new ServiceNowAPIException(errorMessage, mockResponse)); config.validate(mockFailureCollector); Assert.assertEquals(1, mockFailureCollector.getValidationFailures().size()); String expectedErrorMessage = "ServiceNow API returned an unexpected result or the specified " + diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java index 6511895..0d7a8dd 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java @@ -106,7 +106,6 @@ public void testConfigurePipeline() throws Exception { List> result = new ArrayList<>(); map.put("key", "value"); result.add(map); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" + @@ -173,8 +172,8 @@ public void testConfigurePipeline() throws Exception { PowerMockito.mockStatic(ServiceNowInputFormat.class); Mockito.when(ServiceNowInputFormat.fetchTableInfo(Mockito.any(), Mockito.any(), Mockito.anyString(), Mockito.any())).thenReturn(tableInfo); - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class). @@ -208,13 +207,12 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); List> result = new ArrayList<>(); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": []\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - Mockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); try { serviceNowSource.configurePipeline(mockPipelineConfigurer); @@ -239,7 +237,6 @@ public void testPrepareRun() throws Exception { Map map = new HashMap<>(); map.put("key", "value"); result.add(map); - int httpStatus = HttpStatus.SC_OK; Map headers = new HashMap<>(); String responseBody = "{\n" + " \"result\": [\n" + @@ -303,8 +300,8 @@ public void testPrepareRun() throws Exception { " }\n" + " ]\n" + "}"; - RestAPIResponse restAPIResponse = new RestAPIResponse(httpStatus, headers, responseBody); - PowerMockito.when(restApi.executeGet(Mockito.any())).thenReturn(restAPIResponse); + RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); + PowerMockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); PowerMockito.whenNew(OAuthClient.class).