Merge pull request #212 from socrata/rjm/increase-timeout-for-scanshape

EN-55620: Increase timeout for scanShape calls in the GIS job type

src/main/java/com/socrata/datasync/HttpUtility.java

@@ -40,6 +40,7 @@ public class HttpUtility {
 
     private CloseableHttpClient httpClient = null;
     private RequestConfig proxyConfig = null;
+    private RequestConfig noProxyConfig = null;
     private String authHeader;
     private String appToken;
     private boolean authRequired = false;
@@ -73,12 +74,18 @@ public HttpUtility(UserPreferences userPrefs, boolean useAuth, int maxRetries, d
             appToken = userPrefs.getConnectionInfo().getToken();
         }
         authRequired = useAuth;
+
+        noProxyConfig = RequestConfig.custom().
+            setConnectTimeout(15000). // 15s
+            setSocketTimeout(60000). // 1m
+            build();
+
         if(userPrefs != null) {
             String proxyHost = userPrefs.getProxyHost();
             String proxyPort = userPrefs.getProxyPort();
             if (canUse(proxyHost) && canUse(proxyPort)) {
                 HttpHost proxy = new HttpHost(proxyHost, Integer.valueOf(proxyPort));
-                proxyConfig = RequestConfig.custom().setProxy(proxy).build();
+                proxyConfig = RequestConfig.copy(noProxyConfig).setProxy(proxy).build();
                 if (canUse(userPrefs.getProxyUsername()) && canUse(userPrefs.getProxyPassword())) {
                     CredentialsProvider credsProvider = new BasicCredentialsProvider();
                     credsProvider.setCredentials(
@@ -89,11 +96,6 @@ public HttpUtility(UserPreferences userPrefs, boolean useAuth, int maxRetries, d
             }
         }
 
-        RequestConfig requestConfig = RequestConfig.custom().
-            setConnectTimeout(15000). // 15s
-            setSocketTimeout(60000). // 1m
-            build();
-
         SSLContext sslContext;
         try {
             sslContext = SSLContexts.custom().useTLS().build();
@@ -114,7 +116,7 @@ public HttpUtility(UserPreferences userPrefs, boolean useAuth, int maxRetries, d
             setSSLSocketFactory(factory).
             setRetryHandler(datasyncDefaultHandler).
             setKeepAliveStrategy(datasyncDefaultKeepAliveStrategy).
-            setDefaultRequestConfig(requestConfig).
+            setDefaultRequestConfig(noProxyConfig).
             build();
     }
 
@@ -155,20 +157,46 @@ private HttpGet buildHttpGet(URI uri, String contentType){
      * @return the unprocessed results of the post
      */
     public CloseableHttpResponse post(URI uri, HttpEntity entity) throws IOException {
+        return doPost(uri, entity, null);
+    }
+
+    public CloseableHttpResponse post(URI uri, HttpEntity entity, int timeoutMS) throws IOException {
+        return doPost(uri, entity, timeoutMS);
+    }
+
+    private CloseableHttpResponse doPost(URI uri, HttpEntity entity, Integer timeoutMS) throws IOException {
         HttpPost httpPost = new HttpPost(uri);
         httpPost.setHeader(HttpHeaders.USER_AGENT, userAgent);
         httpPost.setHeader(entity.getContentType());
         httpPost.addHeader(datasyncVersionHeader, VersionProvider.getThisVersion());
         httpPost.setEntity(entity);
-        if (proxyConfig != null)
-            httpPost.setConfig(proxyConfig);
+
+        RequestConfig baseRequestConfig;
+
+        if (proxyConfig == null) {
+            baseRequestConfig = noProxyConfig;
+        } else {
+            baseRequestConfig = proxyConfig;
+        }
+        httpPost.setConfig(addTimeout(RequestConfig.copy(baseRequestConfig), timeoutMS).build());
+
         if (authRequired) {
             httpPost.setHeader(HttpHeaders.AUTHORIZATION, authHeader);
             httpPost.setHeader(appHeader, appToken);
         }
         return httpClient.execute(httpPost);
     }
 
+    private static RequestConfig.Builder addTimeout(RequestConfig.Builder base, Integer timeoutMS) {
+        if(timeoutMS == null) {
+            return base;
+        } else {
+            return base.
+                setConnectTimeout(timeoutMS).
+                setSocketTimeout(timeoutMS);
+        }
+    }
+
     public void close() throws IOException {
         httpClient.close();
     }

src/main/java/com/socrata/datasync/publishers/GISPublisher.java

@@ -27,6 +27,7 @@
 
 public class GISPublisher {
 
+    private static final int scanTimeoutMS = 2 * 60 * 1000; // 2 minutes
     private static final Logger logging = Logger.getLogger(GISJob.class.getName());
     private static String ticket = "";
 
@@ -37,7 +38,7 @@ public static JobStatus replaceGeo(File file,
                                        UserPreferences userPrefs) {
         try {
             URI scan_url = makeUri(connectionInfo.getUrl(), "scan", "");
-            Blueprint blueprint = postRawFile(scan_url, file, userPrefs);
+            Blueprint blueprint = postRawFile(scan_url, file, userPrefs, scanTimeoutMS);
             return replaceGeoFile(blueprint, file, userPrefs, connectionInfo, datasetID, layerMap);
         } catch (IOException e) {
             String message = e.getMessage();
@@ -205,15 +206,16 @@ private static String[] getStatus(URI uri,
 
     private static Blueprint postRawFile(URI uri,
                                          File file,
-                                         UserPreferences userPrefs) throws IOException {
+                                         UserPreferences userPrefs,
+                                         int timeoutMS) throws IOException {
         HttpUtility httpUtility = new HttpUtility(userPrefs, true, 3, 2);
 
         System.out.println("Posting file...");
         HttpEntity httpEntity = MultipartEntityBuilder.create()
             .addBinaryBody(file.getName(), file, ContentType.APPLICATION_OCTET_STREAM,file.getName())
             .build();
 
-        HttpResponse response = httpUtility.post(uri, httpEntity);
+        HttpResponse response = httpUtility.post(uri, httpEntity, timeoutMS);
         HttpEntity resEntity = response.getEntity();
         String result = EntityUtils.toString(resEntity);
         logging.log(Level.FINE, result);