Skip to content

Commit

Permalink
EN-507 - Fix networking issues with DataSync 1.6 (#129)
Browse files Browse the repository at this point in the history 
EN-507 - Fix networking issues with DataSync 1.6

There are a number of customers who are reporting networking
issues with DataSync 1.6.  In particular, when you click "Map
fields," the client does a bad SSL handshake and the client's firewall
shuts down the connection.  It does not repro when using
DataSync from the command line.

The root cause of this appears to be soda-java's use of the Jersey
client. Since this only repros when using Soda-java, I'm simply changing
this flow to use the HttpUtility which we know already works.  FWIW,
I did try and upgrade soda-java to Jersey 2.22 and use the same
HTTPClient.  However, it continued to hit the same renegotiation issues.

QA: Verified with Peter Moore that it fixes customer issue.  Regression tests
  • Loading branch information
@spaceballone
spaceballone committed May 11, 2016
1 parent 70ff2ee commit 465a1fa
Show file tree
Hide file tree
Showing 7 changed files with 70 additions and 65 deletions.
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ mvn clean compile -Dmaven.test.skip=true assembly:single
This puts the JAR file into the "target" directory inside the repo. So to open DataSync, simply:
```
cd target
java -jar DataSync-1.7-jar-with-dependencies.jar
java -jar DataSync-1.7.1-jar-with-dependencies.jar
```

### Java SDK
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>DataSync</groupId>
<artifactId>DataSync</artifactId>
<version>1.7</version>
<version>1.7.1</version>
<developers>
<developer>
<name>Ayn Leslie-Cook</name>
Expand Down
43 changes: 38 additions & 5 deletions src/main/java/com/socrata/datasync/DatasetUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,17 @@
import com.socrata.model.importer.Dataset;
import org.apache.http.HttpException;
import org.apache.http.HttpStatus;
import org.apache.http.HttpEntity;
import org.apache.http.util.EntityUtils;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.utils.URIBuilder;
import org.codehaus.jackson.map.DeserializationConfig;
import org.codehaus.jackson.map.ObjectMapper;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
Expand All @@ -27,26 +31,55 @@ public class DatasetUtils {
private static ObjectMapper mapper = new ObjectMapper().enable(DeserializationConfig.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY);

public static Dataset getDatasetInfo(UserPreferences userPrefs, String viewId) throws URISyntaxException, IOException, HttpException {
HttpUtility http = new HttpUtility(userPrefs, true);
String[] schemaAndDomain = userPrefs.getDomain().trim().split("//");
String justDomain = schemaAndDomain[schemaAndDomain.length -1];
String justDomain = getDomainWithoutScheme(userPrefs);
URI absolutePath = new URIBuilder()
.setScheme("https")
.setHost(justDomain)
.setPath("/api/views/" + viewId)
.build();

CloseableHttpResponse resp = get(userPrefs, absolutePath, "application/json");

return mapper.readValue(resp.getEntity().getContent(), Dataset.class);
}

public static String getDatasetSample(UserPreferences userPrefs, String viewId, int rowsToSample) throws URISyntaxException, IOException, HttpException {
String justDomain = getDomainWithoutScheme(userPrefs);
URI absolutePath = new URIBuilder()
.setScheme("https")
.setHost(justDomain)
.setPath("/resource/" + viewId + ".csv")
.addParameter("$limit",""+rowsToSample)
.build();

CloseableHttpResponse resp = get(userPrefs, absolutePath, "application/csv");

HttpEntity entity = resp.getEntity();

return EntityUtils.toString(entity,"UTF-8");
}

private static String getDomainWithoutScheme(UserPreferences userPrefs){
String[] schemaAndDomain = userPrefs.getDomain().trim().split("//");
String justDomain = schemaAndDomain[schemaAndDomain.length -1];
return justDomain;
}

private static CloseableHttpResponse get(UserPreferences userPrefs, URI absolutePath, String contentType) throws URISyntaxException, IOException, HttpException {
HttpUtility http = new HttpUtility(userPrefs, true);
System.out.println("Path: " + absolutePath.toString());
StatusLine statusLine;
int status;
int retriesAvailable = 3;
int retries = 0;
do {
try (CloseableHttpResponse resp = http.get(absolutePath, "application/json")) {
try (CloseableHttpResponse resp = http.get(absolutePath, contentType)) {
statusLine = resp.getStatusLine();
status = statusLine.getStatusCode();
if (status != HttpStatus.SC_OK) {
retries += 1;
} else {
return mapper.readValue(resp.getEntity().getContent(), Dataset.class);
return resp;
}
}
} while (status != HttpStatus.SC_OK && retries < retriesAvailable);
Expand Down
36 changes: 11 additions & 25 deletions src/main/java/com/socrata/datasync/model/DatasetModel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,23 +3,20 @@
import au.com.bytecode.opencsv.CSVReader;
import com.socrata.api.Soda2Consumer;
import com.socrata.datasync.DatasetUtils;
import com.socrata.datasync.config.userpreferences.UserPreferences;
import com.socrata.exceptions.LongRunningQueryException;
import com.socrata.exceptions.SodaError;
import com.socrata.model.importer.Column;
import com.socrata.model.importer.Dataset;
import com.socrata.model.soql.SoqlQuery;
import com.socrata.api.HttpLowLevel;
import com.socrata.api.Soda2Consumer;
import com.socrata.api.Soda2Producer;
import com.socrata.api.SodaDdl;
import com.socrata.builders.SoqlQueryBuilder;
import java.io.StringReader;
import com.sun.jersey.api.client.ClientResponse;

import javax.print.URIException;
import javax.swing.*;
import javax.swing.table.AbstractTableModel;
import java.io.IOException;
import java.io.StringReader;
import java.net.URISyntaxException;
import org.apache.http.HttpException;
import java.util.ArrayList;
import java.util.Vector;

Expand All @@ -38,37 +35,26 @@ public class DatasetModel extends AbstractTableModel {

private Vector data = new Vector();

public DatasetModel(
String domain,
String username,
String password,
String appToken,
String fourbyfour) throws LongRunningQueryException, InterruptedException, SodaError, IOException{
public DatasetModel(UserPreferences prefs,
String fourbyfour) throws LongRunningQueryException, InterruptedException, HttpException, IOException, URISyntaxException{

this.domain = domain;
SodaDdl ddl = SodaDdl.newDdl(domain,username,password,appToken);
Soda2Consumer consumer = Soda2Consumer.newConsumer(domain,
username,
password,
appToken);
this.domain = prefs.getDomain();

initializeDataset(ddl,consumer, fourbyfour);
initializeDataset(prefs, fourbyfour);
}

//Used to pull the charset for this domain
public String getDomain(){
return domain;
}

private boolean initializeDataset(SodaDdl ddl, Soda2Consumer consumer, String fourbyfour) throws LongRunningQueryException, InterruptedException, SodaError, IOException{
datasetInfo = (Dataset) ddl.loadDatasetInfo(fourbyfour);
private boolean initializeDataset(UserPreferences prefs, String fourbyfour) throws LongRunningQueryException, InterruptedException, HttpException, IOException, URISyntaxException{
datasetInfo = DatasetUtils.getDatasetInfo(prefs,fourbyfour);

columns = (ArrayList) datasetInfo.getColumns();

SoqlQueryBuilder builder = new SoqlQueryBuilder(SoqlQuery.SELECT_ALL).setLimit(rowsToSample);
ClientResponse response = consumer.query(fourbyfour, HttpLowLevel.CSV_TYPE,builder.build());
String csv = DatasetUtils.getDatasetSample(prefs,fourbyfour,rowsToSample);

String csv = response.getEntity(String.class);
CSVReader reader = new CSVReader(new StringReader(csv));

String[] lines = reader.readNext();
Expand Down
33 changes: 9 additions & 24 deletions src/main/java/com/socrata/datasync/ui/IntegrationJobTab.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.socrata.datasync.ui;

import com.socrata.api.SodaDdl;
import com.socrata.datasync.*;
import com.socrata.datasync.DatasetUtils;
import com.socrata.datasync.PublishMethod;
Expand Down Expand Up @@ -275,16 +274,11 @@ private void loadJobDataIntoUIFields(IntegrationJob job) {
}


private void updateControlFileModel(ControlFile controlFile, String fourbyfour) throws LongRunningQueryException, InterruptedException, SodaError, IOException{//final String pathToControlFile, final String controlFileContent) throws IOException{
private void updateControlFileModel(ControlFile controlFile, String fourbyfour) throws LongRunningQueryException, InterruptedException, HttpException, IOException, URISyntaxException{

UserPreferences userPrefs = new UserPreferencesJava();

datasetModel = new DatasetModel(userPrefs.getDomain(),
userPrefs.getUsername(),
userPrefs.getPassword(),
userPrefs.getAPIKey(),
fourbyfour);

datasetModel = new DatasetModel(userPrefs, fourbyfour);

controlFileModel = new ControlFileModel(controlFile,datasetModel);

Expand Down Expand Up @@ -493,7 +487,7 @@ public void actionPerformed(ActionEvent evnt) {
try {
if (controlFileModel == null) {
ControlFile controlFile = generateControlFile(
getSodaDdl(),
new UserPreferencesJava(),
fileToPublishTextField.getText(),
(PublishMethod) publishMethodComboBox.getSelectedItem(),
datasetIDTextField.getText(),
Expand Down Expand Up @@ -533,17 +527,17 @@ private boolean fileToPublishIsSelected() {
* @throws com.socrata.exceptions.SodaError
* @throws InterruptedException
*/
private String generateControlFileContent(SodaDdl ddl, String fileToPublish, PublishMethod publishMethod,
String datasetId, boolean containsHeaderRow) throws SodaError, InterruptedException, IOException {
ControlFile control = generateControlFile(ddl,fileToPublish,publishMethod,datasetId,containsHeaderRow);
private String generateControlFileContent(UserPreferences prefs, String fileToPublish, PublishMethod publishMethod,
String datasetId, boolean containsHeaderRow) throws HttpException, URISyntaxException, InterruptedException, IOException {
ControlFile control = generateControlFile(prefs,fileToPublish,publishMethod,datasetId,containsHeaderRow);
ObjectMapper mapper = new ObjectMapper().configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
return mapper.writeValueAsString(control);
}

private ControlFile generateControlFile(SodaDdl ddl, String fileToPublish, PublishMethod publishMethod,
String datasetId, boolean containsHeaderRow) throws SodaError, InterruptedException, IOException {
private ControlFile generateControlFile(UserPreferences prefs, String fileToPublish, PublishMethod publishMethod,
String datasetId, boolean containsHeaderRow) throws HttpException, URISyntaxException, InterruptedException, IOException {

Dataset datasetInfo = (Dataset) ddl.loadDatasetInfo(datasetId);
Dataset datasetInfo = DatasetUtils.getDatasetInfo(prefs, datasetId);
boolean useGeocoding = DatasetUtils.hasLocationColumn(datasetInfo);

String[] columns = null;
Expand All @@ -563,13 +557,4 @@ private boolean datasetIdValid() {
String datasetId = datasetIDTextField.getText();
return Utils.uidIsValid(datasetId);
}

private SodaDdl getSodaDdl() {
UserPreferences userPrefs = new UserPreferencesJava();
return SodaDdl.newDdl(
userPrefs.getDomain(),
userPrefs.getUsername(),
userPrefs.getPassword(),
userPrefs.getAPIKey());
}
}
2 changes: 1 addition & 1 deletion src/test/java/com/socrata/datasync/VersionProviderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,6 @@ public void testIsLatestVersion() throws URISyntaxException {

@Test
public void testGetThisVersion() {
TestCase.assertEquals("1.7", VersionProvider.getThisVersion());
TestCase.assertEquals("1.7.1", VersionProvider.getThisVersion());
}
}
17 changes: 9 additions & 8 deletions src/test/java/com/socrata/datasync/model/ControlFileModelTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import com.socrata.datasync.config.userpreferences.UserPreferences;
import com.socrata.datasync.config.userpreferences.UserPreferencesFile;
import com.socrata.exceptions.LongRunningQueryException;
import com.socrata.exceptions.SodaError;
import org.apache.http.HttpException;
import junit.framework.TestCase;
import org.codehaus.jackson.map.DeserializationConfig;
import org.codehaus.jackson.map.ObjectMapper;
Expand All @@ -15,6 +15,7 @@

import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;

/**
* Series of tests to validate that the control file model produces a correct control file.
Expand All @@ -26,7 +27,7 @@ public class ControlFileModelTest extends TestBase {
ObjectMapper mapper = new ObjectMapper().enable(DeserializationConfig.Feature.ACCEPT_SINGLE_VALUE_AS_ARRAY);

@Test
public void testColumnUpdates() throws IOException, LongRunningQueryException, InterruptedException, SodaError {
public void testColumnUpdates() throws IOException, LongRunningQueryException, InterruptedException, HttpException, URISyntaxException {
ControlFile cf = getTestControlFile();
ControlFileModel model = getTestModel(cf);

Expand All @@ -37,7 +38,7 @@ public void testColumnUpdates() throws IOException, LongRunningQueryException, I

//Can I get out the same control file that I put in?
@Test
public void testSerialization() throws IOException, LongRunningQueryException, InterruptedException, SodaError {
public void testSerialization() throws IOException, LongRunningQueryException, InterruptedException, HttpException, URISyntaxException {

ControlFile cf = getTestControlFile();
ObjectMapper mapper = new ObjectMapper().configure(SerializationConfig.Feature.INDENT_OUTPUT, true);
Expand All @@ -48,7 +49,7 @@ public void testSerialization() throws IOException, LongRunningQueryException, I
}

@Test
public void testIgnoredColumns() throws IOException, LongRunningQueryException, InterruptedException, SodaError {
public void testIgnoredColumns() throws IOException, LongRunningQueryException, InterruptedException, HttpException, URISyntaxException {
ControlFile cf = getTestControlFile();
ControlFileModel model = getTestModel(cf);
int columnLengthBeforeUpdate = model.getColumnCount();
Expand All @@ -64,7 +65,7 @@ public void testIgnoredColumns() throws IOException, LongRunningQueryException,
}

@Test
public void testHasHeaderRow() throws IOException, LongRunningQueryException, InterruptedException, SodaError {
public void testHasHeaderRow() throws IOException, LongRunningQueryException, InterruptedException, HttpException, URISyntaxException {
//Assert that we are skipping the first row when has header row == 1
ControlFile cf = getTestControlFile();
ControlFileModel model = getTestModel(cf);
Expand All @@ -81,7 +82,7 @@ public void testHasHeaderRow() throws IOException, LongRunningQueryException, In


@Test
public void testSyntheticColumns() throws IOException, LongRunningQueryException, InterruptedException, SodaError {
public void testSyntheticColumns() throws IOException, LongRunningQueryException, InterruptedException, HttpException, URISyntaxException {
ControlFile cf = getTestControlFile();
ControlFileModel model = getTestModel(cf);

Expand Down Expand Up @@ -109,11 +110,11 @@ private ControlFile getTestControlFile() throws IOException {
return cf;
}

private ControlFileModel getTestModel(ControlFile cf) throws IOException, LongRunningQueryException, InterruptedException, SodaError {
private ControlFileModel getTestModel(ControlFile cf) throws IOException, LongRunningQueryException, InterruptedException, HttpException, URISyntaxException {
File configFile = new File(PATH_TO_CONFIG_FILE);
ObjectMapper mapper = new ObjectMapper();
UserPreferences userPrefs = mapper.readValue(configFile, UserPreferencesFile.class);
DatasetModel datasetModel = new DatasetModel(userPrefs.getDomain(),userPrefs.getUsername(),userPrefs.getPassword(),userPrefs.getAPIKey(),TestBase.UNITTEST_DATASET_ID);
DatasetModel datasetModel = new DatasetModel(userPrefs,TestBase.UNITTEST_DATASET_ID);
ControlFileModel model = new ControlFileModel(cf,datasetModel);
return model;
}
Expand Down

0 comments on commit 465a1fa

Please sign in to comment.