diff --git a/java/pom.xml b/java/pom.xml
index 6a982ca..73bab1c 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -6,7 +6,7 @@
4.0.0
com.anaplan.client
anaplan-connect
- 1.4.1
+ 1.4.2
Anaplan Connect
@@ -14,30 +14,30 @@
LICENSE.md
MMM dd, yyyy @ KK:mm:ss a (z)
1.8
- 3.7.0
- 9.6.0
+ 3.8.0
+ 10.0.0
github
- 4.11
+ 4.12
1.2.3
- 1.11
+ 1.12
2.6
- 4.4.10
- 4.5.6
- 4.5.6
- 4.5.6
- 23.6-jre
- 4.1
- 1.4.196
+ 4.4.11
+ 4.5.7
+ 4.5.7
+ 4.5.7
+ 27.0.1-jre
+ 4.5
+ 1.4.197
1.3.17
- 9.5.1
+ 10.0.0
2.9.8
2.8.2
0.12
- 3.0.0
+ 3.0.1
1.10.19
5.1.6
1.16.18
- 1.60
+ 1.61
0.3.9
diff --git a/java/src/main/java/com/anaplan/client/CellWriter.java b/java/src/main/java/com/anaplan/client/CellWriter.java
index 95f48f8..efb0d7b 100644
--- a/java/src/main/java/com/anaplan/client/CellWriter.java
+++ b/java/src/main/java/com/anaplan/client/CellWriter.java
@@ -17,6 +17,8 @@
import com.anaplan.client.ex.AnaplanAPIException;
import java.io.IOException;
+import java.io.InputStream;
+import java.sql.SQLException;
/**
* Abstract sink for tabulated cell data.
@@ -38,7 +40,16 @@ public interface CellWriter {
*
* @param row An array of string cell values, one per column
*/
- void writeDataRow(Object[] row) throws AnaplanAPIException, IOException;
+ void writeDataRow(Object[] row) throws AnaplanAPIException, IOException, SQLException;
+
+ /**
+ * Write a data row.
+ * This should be called after writeHeaderRow, for each line of data.
+ *
+ * @param separator An array of string cell values, one per column
+ */
+
+ int writeDataRow(String exportId,int maxRetryCount,int retryTimeout, InputStream inputStream,int chunks,String chunkId, int[] mapcols, int columnCount, String separator) throws AnaplanAPIException, IOException, SQLException;
/**
* Complete the transfer. Any remaining data is transferred,
diff --git a/java/src/main/java/com/anaplan/client/Constants.java b/java/src/main/java/com/anaplan/client/Constants.java
index a11b2ec..33b76cd 100644
--- a/java/src/main/java/com/anaplan/client/Constants.java
+++ b/java/src/main/java/com/anaplan/client/Constants.java
@@ -9,14 +9,15 @@ public class Constants {
public static final int AC_MAJOR = 1;
public static final int AC_MINOR = 4;
- public static final int AC_REVISION = 1 ;
+ public static final int AC_REVISION = 2 ;
+ public static final String AC_Release = "-Snapshot";
public static final boolean AUTH_CLIENT_CACHE_ENABLED = false;
public static final Integer AUTH_TTL_SECONDS = 30;
public static final String X_ACONNECT_HEADER_KEY = "X-AConnect-Client";
- public static final String X_ACONNECT_HEADER_VALUE = "Anaplan_Connect_1.4.1";
+ public static final String X_ACONNECT_HEADER_VALUE = "Anaplan_Connect_1.4.2";
public static final String X_ACONNECT_HEADER = X_ACONNECT_HEADER_KEY + ":" + X_ACONNECT_HEADER_VALUE;
public static final String CORS_HEADER_KEY = "Origin";
@@ -30,4 +31,6 @@ public class Constants {
public static final int MIN_HTTP_CONNECTION_TIMEOUT_SECS = 3;
public static final int MAX_HTTP_CONNECTION_TIMEOUT_SECS = 60;
+
+ public static final double DEFAULT_BACKOFF_MULTIPLIER = 1.5;
}
diff --git a/java/src/main/java/com/anaplan/client/Program.java b/java/src/main/java/com/anaplan/client/Program.java
index f8dc22b..e667ca1 100644
--- a/java/src/main/java/com/anaplan/client/Program.java
+++ b/java/src/main/java/com/anaplan/client/Program.java
@@ -16,15 +16,20 @@
import com.anaplan.client.auth.Credentials;
import com.anaplan.client.auth.KeyStoreManager;
+import com.anaplan.client.dto.ChunkData;
import com.anaplan.client.dto.ExportMetadata;
import com.anaplan.client.dto.ModelData;
import com.anaplan.client.ex.AnaplanAPIException;
import com.anaplan.client.ex.BadSystemPropertyError;
+import com.anaplan.client.ex.NoChunkError;
import com.anaplan.client.ex.PrivateKeyException;
import com.anaplan.client.jdbc.JDBCCellReader;
+import com.anaplan.client.jdbc.JDBCCellWriter;
import com.anaplan.client.jdbc.JDBCConfig;
import com.anaplan.client.logging.LogUtils;
import com.anaplan.client.transport.ConnectionProperties;
+import com.anaplan.client.transport.retryer.AnaplanJdbcRetryer;
+import com.anaplan.client.transport.retryer.FeignApiRetryer;
import com.google.common.base.Strings;
import com.opencsv.CSVParser;
import org.apache.commons.ssl.PKCS8Key;
@@ -51,9 +56,7 @@
import java.security.cert.X509Certificate;
import java.security.interfaces.RSAPrivateKey;
import java.security.spec.PKCS8EncodedKeySpec;
-import java.util.Collection;
-import java.util.Properties;
-import java.util.Scanner;
+import java.util.*;
/**
* A command-line interface to the Anaplan Connect API library. Running the
@@ -102,6 +105,7 @@ public abstract class Program {
private static int maxRetryCount = Constants.MIN_RETRY_COUNT;
private static int retryTimeout = Constants.MIN_RETRY_TIMEOUT_SECS;
private static int httpConnectionTimeout = Constants.MIN_HTTP_CONNECTION_TIMEOUT_SECS;
+ private ConnectionProperties properties;
private static final Logger LOG = LoggerFactory.getLogger(Program.class);
@@ -143,28 +147,6 @@ public static void main(String... args) {
}
} else if (arg == "-q" || arg == "-quiet") {
quiet = true;
- } else if (arg == "-MO" || arg == "-modules") {
- somethingDone = true;
- Model model = getModel(workspaceId, modelId);
- if (model != null) {
- for (Module module : model.getModules()) {
- LOG.info(Utils.formatTSV(
- module.getId(),
- module.getCode(),
- module.getName()));
- }
- }
- } else if (arg == "-VI" || arg == "-views") {
- somethingDone = true;
- Module module = getModule(workspaceId, modelId, moduleId);
- if (module != null) {
- for (View view : module.getViews()) {
- LOG.info(Utils.formatTSV(
- view.getId(),
- view.getCode(),
- view.getName()));
- }
- }
} else if (arg == "-F" || arg == "-files") {
somethingDone = true;
Model model = getModel(workspaceId, modelId);
@@ -396,7 +378,7 @@ public static void main(String... args) {
String certificatePath = args[argi++];
setCertificatePath(certificatePath);
} else if (arg == "-pkey" || arg == "-privatekey") {
- if (keyStorePath !=null){
+ if (keyStorePath != null) {
throw new IllegalArgumentException("expected either the privatekey or the keystore arguments");
}
String auth = args[argi++];
@@ -409,7 +391,7 @@ public static void main(String... args) {
setPassphrase("?");
}
} else if (arg == "-k" || arg == "-keystore") {
- if (passphrase !=null || privateKeyPath != null ){
+ if (passphrase != null || privateKeyPath != null) {
throw new IllegalArgumentException("expected either the privatekey or keystore arguments");
}
String keyStorePath = args[argi++];
@@ -546,10 +528,9 @@ public static void main(String... args) {
} else if (arg.equals("-jdbcproperties")) {
String propertiesFilePath = args[argi++];
JDBCConfig jdbcConfig = loadJdbcProperties(propertiesFilePath);
-
- ServerFile serverFile = getServerFile(workspaceId, modelId,
- fileId, true);
- if (serverFile != null) {
+ if (fileId != null) {
+ ServerFile serverFile = getServerFile(workspaceId, modelId,
+ fileId, true);
CellWriter cellWriter = null;
CellReader cellReader = null;
try {
@@ -575,6 +556,81 @@ public static void main(String... args) {
if (cellWriter != null)
cellWriter.abort();
}
+ } else if (exportId != null) {
+ ServerFile serverFile = getServerFile(workspaceId, modelId,
+ exportId, true);
+ if (serverFile != null) {
+ CellWriter cellWriter = null;
+ somethingDone = true;
+ Export export = getExport(workspaceId, modelId, exportId);
+ ExportMetadata emd = export.getExportMetadata();
+ InputStream inputStream = null;
+ int columnCount = emd.getColumnCount();
+ int transferredrows = 0;
+ int[] mapcols = new int[columnCount];
+ String separator = emd.getSeparator();
+ //build map for metadata for exports
+ HashMap headerName = new HashMap();
+ for (int i = 0; i < emd.getHeaderNames().length; i++) {
+ headerName.put(emd.getHeaderNames()[i], i);
+ }
+ for (int k = 0; k < maxRetryCount; k++) {
+ try {
+ List chunkList = serverFile.getChunks();
+ //jdbc params exists
+ if (jdbcConfig.getJdbcParams() != null && jdbcConfig.getJdbcParams().length > 0
+ && !jdbcConfig.getJdbcParams()[0].equals("")) {
+ mapcols = new int[jdbcConfig.getJdbcParams().length];
+ //extract matching anaplan columns
+ for (int i = 0; i < jdbcConfig.getJdbcParams().length; i++) {
+ String paramName = ((String) jdbcConfig.getJdbcParams()[i]).trim();
+ if (headerName.containsKey(paramName)) {
+ mapcols[i] = headerName.get(paramName);
+ } else {
+ LOG.debug("{} from JDBC properties file is not a valid column in Anaplan", jdbcConfig.getJdbcParams()[i]);
+ throw new AnaplanAPIException("Please make sure column names in jdbcproperties file match with the exported columns on Anaplan");
+ }
+ }
+ }
+ //Retry Fix
+ cellWriter = new JDBCCellWriter(jdbcConfig);
+ for (ChunkData chunk : chunkList) {
+ byte[] chunkContent = serverFile.getChunkContent(chunk.getId());
+ if (chunkContent == null) throw new NoChunkError(chunk.getId());
+ inputStream = new ByteArrayInputStream(chunkContent);
+ transferredrows = cellWriter.writeDataRow(exportId,maxRetryCount,retryTimeout,inputStream, chunkList.size(), chunk.getId(), mapcols, columnCount, separator);
+ }
+ if (transferredrows != 0) {
+ LOG.info("Transferred {} records to {}", transferredrows, jdbcConfig.getJdbcConnectionUrl());
+ } else if (transferredrows == 0) {
+ LOG.info("No records were transferred to {}", jdbcConfig.getJdbcConnectionUrl());
+ }
+ k = maxRetryCount;
+ } catch (AnaplanAPIException ape){
+ LOG.error(ape.getMessage());
+ k=maxRetryCount;
+ } catch (Exception e) {
+ AnaplanJdbcRetryer anaplanJdbcRetryer = new AnaplanJdbcRetryer((long) (retryTimeout * 1000),
+ (long) Constants.MAX_RETRY_TIMEOUT_SECS * 1000,
+ FeignApiRetryer.DEFAULT_BACKOFF_MULTIPLIER);
+ Long interval = anaplanJdbcRetryer.nextMaxInterval(k);
+ try {
+ LOG.debug("Could not connect to the database! Will retry in {} seconds ", interval/1000);
+ // do not retry if we get any other error
+ Thread.sleep(interval);
+ } catch (InterruptedException e1) {
+ // we still want to retry, even though sleep was interrupted
+ LOG.debug("Sleep was interrupted.");
+ }
+ } finally {
+ if (inputStream != null)
+ inputStream.close();
+ if (cellWriter!=null)
+ cellWriter.close();
+ }
+ }
+ }
+
}
} else {
displayHelp();
@@ -1278,9 +1334,9 @@ private static RSAPrivateKey getPrivateKey() throws GeneralSecurityException {
File privateKeyFile = new File(privateKeyPath);
if (privateKeyFile.isFile()) {
//load privateKey from file
- return loadPrivateKeyFromFile(privateKeyPath,passphrase);
+ return loadPrivateKeyFromFile(privateKeyPath, passphrase);
} else {
- throw new RuntimeException("The specified privateKey path '" + privateKeyPath + "' is invalid");
+ throw new RuntimeException("The specified privateKey path '" + privateKeyPath + "' is invalid");
}
} else if (keyStorePath != null && keyStorePrivateKeyAlias != null) {
return new KeyStoreManager().getKeyStorePrivateKey(keyStorePath, getKeyStorePassword(), keyStorePrivateKeyAlias);
@@ -1289,7 +1345,7 @@ private static RSAPrivateKey getPrivateKey() throws GeneralSecurityException {
}
}
- /**
+ /**
* Returns the certificate path set using setCertificatePath()
*
* @return the certificate path
@@ -1483,7 +1539,8 @@ private static String readFileContents(File file) throws FileNotFoundException {
* @throws CertificateException
* @throws FileNotFoundException
*/
- private static X509Certificate loadCertificateFromFile(File certificateFile) throws CertificateException, FileNotFoundException {
+ private static X509Certificate loadCertificateFromFile(File certificateFile) throws
+ CertificateException, FileNotFoundException {
// loading certificate chain
CertificateFactory certificateFactory = CertificateFactory.getInstance("X.509");
InputStream certificateStream = new FileInputStream(certificateFile);
@@ -1504,8 +1561,8 @@ private static X509Certificate loadCertificateFromFile(File certificateFile) thr
* @return a RSAPrivateKey
*/
public static RSAPrivateKey loadPrivateKeyFromFile(String privateKeyPath, String passphrase) {
- byte [] privateKeyDer;
- byte [] privateKeyDecrypted;
+ byte[] privateKeyDer;
+ byte[] privateKeyDecrypted;
// Read PEM file
try (PEMParser pemParser = new PEMParser(new FileReader(privateKeyPath))) {
// Convert PEM object to DER
@@ -1518,8 +1575,8 @@ public static RSAPrivateKey loadPrivateKeyFromFile(String privateKeyPath, String
PrivateKey privateKey = (pkcs8Key.isRSA()) ? KeyFactory.getInstance("RSA").generatePrivate(pkcs8EncodedKeySpec) :
KeyFactory.getInstance("DSA").generatePrivate(pkcs8EncodedKeySpec);
return (RSAPrivateKey) privateKey;
- } catch(Exception e){
- throw new PrivateKeyException(privateKeyPath + ", " +e);
+ } catch (Exception e) {
+ throw new PrivateKeyException(privateKeyPath + ", " + e);
}
}
@@ -1545,10 +1602,12 @@ private static JDBCConfig loadJdbcProperties(String jdbcPropertiesPath) {
jdbcConfig.setJdbcConnectionUrl(jdbcProps.getProperty("jdbc.connect.url"));
jdbcConfig.setJdbcUsername(jdbcProps.getProperty("jdbc.username"));
jdbcConfig.setJdbcPassword(jdbcProps.getProperty("jdbc.password"));
- try {
- jdbcConfig.setJdbcFetchSize(Integer.parseInt(jdbcProps.getProperty("jdbc.fetch.size")));
- } catch (NumberFormatException e) {
- throw new RuntimeException("Invalid JDBC Fetch-size provided in properties.");
+ if (fileId != null) {
+ try {
+ jdbcConfig.setJdbcFetchSize(Integer.parseInt(jdbcProps.getProperty("jdbc.fetch.size")));
+ } catch (NumberFormatException e) {
+ throw new RuntimeException("Invalid JDBC Fetch-size provided in properties.");
+ }
}
jdbcConfig.setStoredProcedure(Boolean.valueOf(jdbcProps.getProperty("jdbc.isStoredProcedure", "false")));
jdbcConfig.setJdbcQuery(jdbcProps.getProperty("jdbc.query"));
@@ -1612,10 +1671,6 @@ private static void displayHelp() {
+ "(-w|-workspace) (|): select a workspace by id/name\n"
+ "(-M|-models): list available models in selected workspace\n"
+ "(-m|-model) (|): select a model by id/name\n"
- + "(-MO|-modules): list available modules in selected model\n"
- + "(-mo|-module): (|): select a module by id/name\n"
- + "(-VI|-views): list available views in selected module\n"
- + "(-vi|-view): (|): select a view by id/name\n"
+ "(-F|-files): list available server files in selected model\n"
+ "(-f|-file) (|): select a server file by id/name\n"
+ "(-ch|-chunksize): upload chunk-size number, defaults to 1048576.\n"
@@ -1659,7 +1714,7 @@ private static void displayHelp() {
private static void displayVersion() {
LOG.debug(Strings.repeat("=", 70));
- LOG.debug("Anaplan Connect {}.{}.{}", Constants.AC_MAJOR,Constants.AC_MINOR,Constants.AC_REVISION);
+ LOG.debug("Anaplan Connect {}.{}.{}", Constants.AC_MAJOR, Constants.AC_MINOR, Constants.AC_REVISION);
LOG.debug("{} ({})/ ({})/", System.getProperty("java.vm.name"), System.getProperty("java.vendor"),
System.getProperty("java.vm.version"), System.getProperty("java.version"));
LOG.debug("({}{})/{}", System.getProperty("os.name"), System.getProperty("os.arch"), System.getProperty("os.version"));
diff --git a/java/src/main/java/com/anaplan/client/ServerFile.java b/java/src/main/java/com/anaplan/client/ServerFile.java
index a41d1c1..cf9be04 100644
--- a/java/src/main/java/com/anaplan/client/ServerFile.java
+++ b/java/src/main/java/com/anaplan/client/ServerFile.java
@@ -26,19 +26,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FilterOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.LineNumberReader;
-import java.io.OutputStream;
-import java.io.RandomAccessFile;
-import java.io.SequenceInputStream;
+import java.io.*;
import java.nio.charset.StandardCharsets;
+import java.sql.SQLException;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.List;
@@ -166,7 +156,8 @@ public InputStream getDownloadStream() {
@Override
public boolean hasMoreElements() {
- return index < chunkList.size();
+ int chunkListSize = chunkList == null ? 0 : chunkList.size();
+ return index < chunkListSize;
}
@Override
@@ -245,6 +236,7 @@ public void upLoad(File source, boolean deleteExisting, int chunkSize) throws IO
long length = source.length();
data.setChunkCount((int) ((length - 1) / chunkSize) + 1);
ServerFileResponse response = getApi().upsertFileDataSource(getWorkspace().getId(), getModel().getId(), getId(), data);
+ System.setProperty("file.encoding", data.getEncoding());
if (response == null || response.getItem() == null) {
throw new CreateImportDatasourceError(getName());
}
@@ -271,8 +263,24 @@ public void upLoad(File source, boolean deleteExisting, int chunkSize) throws IO
buffer = new byte[size];
}
sourceFile.readFully(buffer, 0, size);
- totalReadSoFar += size;
- getApi().uploadChunkCompressed(getWorkspace().getId(), getModel().getId(), getId(), chunk.getId(), buffer);
+ //reading the last index of the separator
+ int separatorLastIndex = lastIndexOf(buffer,data.getSeparator());
+ //calculating the size of byte array to load the bytes until the last index of separator
+ int finalSize = separatorLastIndex+1;
+ //creating the buffer to load the byte array until last separator
+ byte[] finalBuffer = new byte[finalSize];
+ //copying the data from existing byte array to new byte array until last separator
+ System.arraycopy(buffer,0,finalBuffer,0,finalSize);
+ //calculating the total read size from the file
+ totalReadSoFar += finalSize;
+ //checking if there is another chunk to decide if to upload the newly created buffer or existing buffer.
+ //existing buffer will be uploaded in case of last chunk
+ if(chunkIterator.hasNext()) {
+ getApi().uploadChunkCompressed(getWorkspace().getId(), getModel().getId(), getId(), chunk.getId(), finalBuffer);
+ }else{
+ getApi().uploadChunkCompressed(getWorkspace().getId(), getModel().getId(), getId(), chunk.getId(), buffer);
+ }
+ sourceFile.seek(totalReadSoFar);
LOG.debug("Uploaded chunk: {} (size={}MB)", chunk.getId(), chunkSize / 1000000);
}
} finally {
@@ -286,6 +294,27 @@ public void upLoad(File source, boolean deleteExisting, int chunkSize) throws IO
}
}
+ /**
+ * returns the last index of single byte separator from a byte array
+ * @param outerArray
+ * @param separator
+ * @return last index of a single byte separator
+ */
+ public int lastIndexOf(byte[] outerArray, String separator) {
+ byte[] smallerArray = separator.getBytes();
+ for(int i = outerArray.length - smallerArray.length; i > 0; --i) {
+ boolean found = true;
+ for(int j = 0; j < smallerArray.length; ++j) {
+ if (outerArray[i+j] != smallerArray[j]) {
+ found = false;
+ break;
+ }
+ }
+ if (found) return i;
+ }
+ return -1;
+ }
+
/**
* Finalizes the upload-stream and updates it's metadata.
*/
@@ -417,6 +446,12 @@ public void writeDataRow(Object[] row) throws AnaplanAPIException,
output.write(buf.append('\n').toString().getBytes("UTF-8"));
}
+ @Override
+ public int writeDataRow(String exportId,int maxRetryCount,int retryTimeout,InputStream inputStream,int noOfChunks, String chunkId, int[] mapcols, int columnCount, String separator) throws AnaplanAPIException, IOException, SQLException {
+ //dummy value as the implementation is done in JdbcCellWriter
+ return 1;
+ }
+
@Override
public void close() throws IOException {
if (output != null) {
diff --git a/java/src/main/java/com/anaplan/client/auth/AbstractAuthenticator.java b/java/src/main/java/com/anaplan/client/auth/AbstractAuthenticator.java
index 5dc03bb..f69adb9 100644
--- a/java/src/main/java/com/anaplan/client/auth/AbstractAuthenticator.java
+++ b/java/src/main/java/com/anaplan/client/auth/AbstractAuthenticator.java
@@ -6,7 +6,7 @@
import com.anaplan.client.ex.AnaplanAPIException;
import com.anaplan.client.transport.AnaplanApiProvider;
import com.anaplan.client.transport.ConnectionProperties;
-import com.anaplan.client.transport.FeignApiRetryer;
+import com.anaplan.client.transport.retryer.FeignApiRetryer;
import com.anaplan.client.transport.interceptors.AConnectHeaderInjector;
import feign.Feign;
import feign.FeignException;
@@ -24,7 +24,8 @@
public abstract class AbstractAuthenticator extends AnaplanApiProvider implements Authenticator {
private static final Logger LOG = LoggerFactory.getLogger(AbstractAuthenticator.class.getName());
- private static final int TOKEN_EXPIRATION_BACKOFF_SECONDS = 60000;
+ private static final int TOKEN_EXPIRATION_REFRESH_WINDOW = 5 * 60 * 1000;
+ private static final int TOKEN_EXPIRED_WINDOW = 60 * 1000;
private AnaplanAuthenticationAPI authClient;
private byte[] authToken;
private Long authTokenExpiresAt;
@@ -38,15 +39,15 @@ public abstract class AbstractAuthenticator extends AnaplanApiProvider implement
/**
* Fetches auth token from Anaplan Auth Service, checks to see if its expired
* and accordingly fetches a fresh new token or refreshes the existing token, exactly
- * 1 minute before it expires, as defined by TOKEN_EXPIRATION_BACKOFF_SECONDS.
+ * 1 minute before it expires, as defined by TOKEN_EXPIRED_WINDOW.
*
* @return AuthenticationResp
*/
@Override
public String getAuthToken() {
- if (authToken == null || authTokenExpiresAt == null) {
+ if (authToken == null || authTokenExpiresAt == null || System.currentTimeMillis() - authTokenExpiresAt > TOKEN_EXPIRED_WINDOW) {
authToken = authenticate();
- } else if (authTokenExpiresAt - TOKEN_EXPIRATION_BACKOFF_SECONDS < System.currentTimeMillis()) {
+ } else if (authTokenExpiresAt - System.currentTimeMillis() < TOKEN_EXPIRATION_REFRESH_WINDOW) {
authToken = refreshToken();
}
return new String(authToken);
diff --git a/java/src/main/java/com/anaplan/client/ex/AnaplanRetryableException.java b/java/src/main/java/com/anaplan/client/ex/AnaplanRetryableException.java
new file mode 100644
index 0000000..939f740
--- /dev/null
+++ b/java/src/main/java/com/anaplan/client/ex/AnaplanRetryableException.java
@@ -0,0 +1,18 @@
+package com.anaplan.client.ex;
+
+public class AnaplanRetryableException extends Exception {
+ /**
+ * Create an exception with the specified message.
+ */
+ public AnaplanRetryableException(String message) {
+ super(message);
+ }
+
+ /**
+ * Create an exception with the specified message and cause.
+ */
+ public AnaplanRetryableException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
+
diff --git a/java/src/main/java/com/anaplan/client/jdbc/JDBCCellWriter.java b/java/src/main/java/com/anaplan/client/jdbc/JDBCCellWriter.java
new file mode 100644
index 0000000..8505331
--- /dev/null
+++ b/java/src/main/java/com/anaplan/client/jdbc/JDBCCellWriter.java
@@ -0,0 +1,408 @@
+package com.anaplan.client.jdbc;
+
+/**
+ * NOTE: There is no decisive way to figure out if the provided query is a
+ * Stored-Procedure/Function call, or a regular SELECT query.
+ */
+
+import com.anaplan.client.CellWriter;
+import com.anaplan.client.Constants;
+import com.anaplan.client.ex.AnaplanAPIException;
+import com.anaplan.client.ex.AnaplanRetryableException;
+import com.anaplan.client.ex.TooLongQueryError;
+import com.anaplan.client.transport.ConnectionProperties;
+import com.anaplan.client.transport.retryer.AnaplanJdbcRetryer;
+import com.anaplan.client.transport.retryer.FeignApiRetryer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.LineNumberReader;
+import java.security.InvalidParameterException;
+import java.sql.*;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An implementation of CellWriter that connects to a JDBC data source.
+ * A query is executed and the results are written to the database one record at a time.
+ *
+ * @since 1.4.2
+ */
+
+public class JDBCCellWriter implements CellWriter {
+ private static final Logger LOG = LoggerFactory.getLogger(JDBCCellWriter.class);
+ private static final int MAX_ALLOWED_SQL_CHARACTERS = 65535;
+ private static final int MAX_ALLOWED_CONNECTION_STRING_LENGTH = 1500;
+ private Connection connection;
+ private ConnectionProperties properties;
+ private boolean autoCommit;
+ private Statement statement;
+ private ResultSet resultSet;
+ private JDBCConfig jdbcConfig;
+ private String lastRow;
+ private int batch_no;
+ private int datarowstransferred = 0;
+ private int batch_records = 0;
+ private int batch_size = 1000;
+ private int update = 0;
+ private int not_update = 0;
+ private PreparedStatement preparedStatement = null;
+ private List tempRowList = new ArrayList<>(); //to save rows so that we can reuse in case of retry.
+
+ public JDBCCellWriter(JDBCConfig jdbcConfig) {
+ this.jdbcConfig = jdbcConfig;
+ String rawJdbcQuery = jdbcConfig.getJdbcQuery();
+ this.jdbcConfig.setJdbcQuery(sanitizeQuery(rawJdbcQuery));
+ }
+
+ /**
+ * Checks if the provided SQL query is sanitary:
+ * - not greater than MAX_ALLOWED_SQL_CHARACTERS =
+ *
+ * @param query
+ * @return
+ */
+ private String sanitizeQuery(String query) {
+ if (query.length() >= MAX_ALLOWED_SQL_CHARACTERS) {
+ throw new TooLongQueryError(query.length());
+ }
+ return query;
+ }
+
+ @Override
+ public void writeHeaderRow(Object[] row) throws AnaplanAPIException, IOException {
+ }
+
+ @Override
+ public void writeDataRow(Object[] row) throws AnaplanAPIException, IOException, SQLException {
+ }
+
+ /**
+ * Write Anaplan exported data to the configurable DB
+ *
+ * @param exportId
+ * @param maxRetryCount
+ * @param retryTimeout
+ * @param inputStream
+ * @param noOfChunks
+ * @param chunkId
+ * @param mapcols
+ * @param columnCount
+ * @param separator An array of string cell values, one per column
+ * @return
+ * @throws AnaplanAPIException
+ * @throws SQLException
+ */
+
+ @Override
+ public int writeDataRow(String exportId, int maxRetryCount, int retryTimeout, InputStream inputStream, int noOfChunks, String chunkId, int[] mapcols, int columnCount, String separator)
+ throws AnaplanAPIException, SQLException {
+ if (jdbcConfig.getJdbcConnectionUrl().length() > MAX_ALLOWED_CONNECTION_STRING_LENGTH) {
+ throw new InvalidParameterException("JDBC connection string cannot be more than " + MAX_ALLOWED_CONNECTION_STRING_LENGTH + " characters in length!");
+ }
+ try {
+ LineNumberReader lnr = new LineNumberReader(
+ new InputStreamReader(inputStream));
+ String line;
+ List rowBatch = new ArrayList<>();
+ while (null != (line = lnr.readLine())) {
+ //ignore the header
+ if (lnr.getLineNumber() == 1 && chunkId.equals("0")) {
+ LOG.info("Export {} to database started successfully", exportId);
+ }else {
+ if (lnr.getLineNumber() == 1 && !(chunkId.equals("0"))) {
+ line = lastRow.concat(line);
+ }
+ String[] row = line.split(separator);
+ rowBatch.add(row);
+ lastRow = line;
+ if (++batch_records % batch_size == 0) {
+ ++batch_no;
+ //for this batch, code should have dummy values to bypass the check for chunkId and no of chunks
+ // chunkId is being sent as 1 and no of chunks as 2 to bypass the check
+ batchExecution(rowBatch, columnCount, mapcols,"1",2, maxRetryCount, retryTimeout);
+ rowBatch = new ArrayList<>(); //reset temoRowList
+ batch_records = 0;
+ }
+ }
+ }
+ //transfer the last batch when all of the lines from inputstream have been read
+ ++batch_no;
+ batchExecution(rowBatch, columnCount, mapcols,chunkId,noOfChunks, maxRetryCount, retryTimeout);
+ batch_records = 0;
+ //batch update exceptions captured to determine the committed and failed records
+ }catch (Exception e) {
+ LOG.debug("Error observed : {}", e.getStackTrace());
+ throw new AnaplanAPIException(e.getMessage());
+ } finally {
+ if (preparedStatement != null) {
+ if (!preparedStatement.isClosed())
+ preparedStatement.close();
+ }
+ }
+ return datarowstransferred;
+ }
+
+ /**
+ * execute the batch and get the update count of records
+ *
+ * @param maxRetryCount
+ * @param retryTimeout
+ */
+ private void batchExecution(List rowBatch, int columnCount, int[] mapcols, String chunkId, int noOfChunks,
+ int maxRetryCount, int retryTimeout) throws SQLException {
+ int k = 0; //retry count
+ int[] count = new int[batch_size];
+ boolean retry = false;
+ int notAvailable = 0;
+ int rowBatchSize = rowBatch.size();
+ do {
+ k++;
+ try {
+ if (connection == null || connection.isClosed()) {
+ getJdbcConnection(maxRetryCount, retryTimeout);
+ }
+ if (preparedStatement == null || preparedStatement.isClosed()) {
+ preparedStatement = connection.prepareStatement(jdbcConfig.getJdbcQuery(),
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ }
+ int chunkno = Integer.parseInt(chunkId);
+ if (chunkno != noOfChunks-1) {
+ rowBatchSize = rowBatch.size()-1;
+ }
+ for (int i = 0; i < rowBatchSize; i++) {
+ psLineEndWithSeparator(mapcols, columnCount, rowBatch.get(i));
+ preparedStatement.addBatch();
+ }
+ // ++batch_no;
+ executeBatch(rowBatchSize, maxRetryCount, retryTimeout);
+ batch_records = 0;
+ k = 0;
+ retry = false;
+ datarowstransferred = datarowstransferred + update;
+ update = 0;
+ not_update = 0;
+ } catch (AnaplanRetryableException ae) {
+ retry = true;
+ }
+ } while (k < maxRetryCount && retry);
+ // not successful
+ if (retry) {
+ throw new AnaplanAPIException("Could not connect to the database after " + maxRetryCount + " retries");
+ }
+ }
+
+
+ /**
+ * @param mapcols
+ * @param columnCount
+ * @param row
+ * @throws SQLException
+ */
+ //during code refactoring, usage of this method was removed. this will be removed once we QA the database exports.
+ private void psLineNotEndWithSeparator(int[] mapcols, int columnCount, String[] row) throws SQLException {
+ if (jdbcConfig.getJdbcParams() != null && jdbcConfig.getJdbcParams().length > 0
+ && !jdbcConfig.getJdbcParams()[0].equals("") && mapcols.length != 0) {
+ for (int i = 0; i < mapcols.length; i++) {
+ preparedStatement.setString(i + 1, String.valueOf(row[mapcols[i]]) != "" ? String.valueOf(row[mapcols[i]]) : "");
+ }
+ } else {
+ for (int i = 0; i < columnCount; i++) {
+ preparedStatement.setString(i + 1, String.valueOf(row[i]) != "" ? String.valueOf(row[i]) : "");
+ }
+ }
+ }
+
+ /**
+ * prepare the batch when line ends with a separator
+ *
+ * @param mapcols
+ * @param columnCount
+ * @param row
+ * @throws SQLException
+ */
+ private void psLineEndWithSeparator(int[] mapcols, int columnCount, String[] row) throws SQLException {
+ //handling the last column if the value is null in anaplan
+ if (jdbcConfig.getJdbcParams() != null && jdbcConfig.getJdbcParams().length > 0
+ && !jdbcConfig.getJdbcParams()[0].equals("") && mapcols.length != 0) {
+ for (int i = 0; i < mapcols.length; i++) {
+ if (i == mapcols.length - 1 && row.length= 0) {
+ update++;
+ } else if (updateCounts[i] == Statement.EXECUTE_FAILED) {
+ not_update++;
+ }
+ }
+ }
+ retry = true;
+ // batch_records = 0;
+ } catch (SQLException e) {
+ throw new AnaplanRetryableException(e.getMessage());
+ }
+ } while (k < maxRetryCount && retry);
+ return count;
+ }
+
+ /**
+ * Connects to DB with 5 retries
+ */
+ private void getJdbcConnection(int maxRetryCount, int retryTimeout) {
+ int k = 0; //retry count
+ boolean retry = false;
+ do {
+ k++;
+ try {
+ if (connection == null || connection.isClosed()) {
+ connection = DriverManager.getConnection(jdbcConfig.getJdbcConnectionUrl(),
+ jdbcConfig.getJdbcUsername(), jdbcConfig.getJdbcPassword());
+ connection.setAutoCommit(false);
+ LOG.info("Created JDBC connection to: {}", connection.getMetaData().getURL());
+ retry = false;
+ }
+ } catch (SQLException e) {
+ sleepNoNetwork(maxRetryCount, retryTimeout, k);
+ retry = true;
+ }
+ } while (k < maxRetryCount && retry);
+ // if not successful after configured no of max retries
+ if (retry) {
+ throw new AnaplanAPIException("Could not connect to the database after " + maxRetryCount + " retries");
+ }
+ }
+
+ /**
+ * common sleep process for database retries
+ *
+ * @param maxRetryCount
+ * @param k
+ */
+ private void sleepNoNetwork(int maxRetryCount, int retryTimeout, int k) {
+ Long interval=(long)retryTimeout*1000;
+ AnaplanJdbcRetryer anaplanJdbcRetryer = new AnaplanJdbcRetryer((long) (retryTimeout * 1000),
+ (long) Constants.MAX_RETRY_TIMEOUT_SECS * 1000,
+ FeignApiRetryer.DEFAULT_BACKOFF_MULTIPLIER);
+ if (k>1) {
+ interval = anaplanJdbcRetryer.nextMaxInterval(k-1);
+ }
+ if(k this.maxPeriod ? this.maxPeriod : interval;
+ }
+
+}
diff --git a/java/src/main/java/com/anaplan/client/transport/retryer/AnaplanRetryer.java b/java/src/main/java/com/anaplan/client/transport/retryer/AnaplanRetryer.java
new file mode 100644
index 0000000..675a066
--- /dev/null
+++ b/java/src/main/java/com/anaplan/client/transport/retryer/AnaplanRetryer.java
@@ -0,0 +1,18 @@
+package com.anaplan.client.transport.retryer;
+
+/**
+ * Anaplan retryer that emulates Spring's BackoffExponentialPolicy for setting intervals/periods
+ * between attempts.
+ */
+
+import com.anaplan.client.Constants;
+
+public interface AnaplanRetryer {
+ long DEFAULT_PERIOD = Constants.MIN_RETRY_TIMEOUT_SECS * 1000L;
+ long DEFAULT_MAX_PERIOD = Constants.MAX_RETRY_TIMEOUT_SECS * 1000L;
+ int DEFAULT_MAX_ATTEMPTS = Constants.MIN_RETRY_COUNT;
+ double DEFAULT_BACKOFF_MULTIPLIER = Constants.DEFAULT_BACKOFF_MULTIPLIER;
+
+ long nextMaxInterval(int noOfAttempts);
+
+}
diff --git a/java/src/main/java/com/anaplan/client/transport/FeignApiRetryer.java b/java/src/main/java/com/anaplan/client/transport/retryer/FeignApiRetryer.java
similarity index 94%
rename from java/src/main/java/com/anaplan/client/transport/FeignApiRetryer.java
rename to java/src/main/java/com/anaplan/client/transport/retryer/FeignApiRetryer.java
index 8ac43b8..03ef920 100644
--- a/java/src/main/java/com/anaplan/client/transport/FeignApiRetryer.java
+++ b/java/src/main/java/com/anaplan/client/transport/retryer/FeignApiRetryer.java
@@ -1,4 +1,4 @@
-package com.anaplan.client.transport;
+package com.anaplan.client.transport.retryer;
import com.anaplan.client.Constants;
import feign.RetryableException;
@@ -14,10 +14,10 @@ public class FeignApiRetryer extends Retryer.Default {
private static final Logger LOG = LoggerFactory.getLogger(FeignApiRetryer.class);
- public static final long DEFAULT_PERIOD = Constants.MIN_RETRY_COUNT * 1000L;
+ public static final long DEFAULT_PERIOD = Constants.MIN_RETRY_TIMEOUT_SECS * 1000L;
public static final long DEFAULT_MAX_PERIOD = Constants.MAX_RETRY_TIMEOUT_SECS * 1000L;
public static final int DEFAULT_MAX_ATTEMPTS = Constants.MIN_RETRY_COUNT;
- public static final double DEFAULT_BACKOFF_MULTIPLIER = 1.5;
+ public static final double DEFAULT_BACKOFF_MULTIPLIER = Constants.DEFAULT_BACKOFF_MULTIPLIER;
private Long period;
private Long maxPeriod;
diff --git a/java/src/main/java/com/anaplan/client/transport/serialization/ByteArraySerializer.java b/java/src/main/java/com/anaplan/client/transport/serialization/ByteArraySerializer.java
index 833e9ba..6a13863 100644
--- a/java/src/main/java/com/anaplan/client/transport/serialization/ByteArraySerializer.java
+++ b/java/src/main/java/com/anaplan/client/transport/serialization/ByteArraySerializer.java
@@ -5,7 +5,7 @@
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
+import java.nio.charset.Charset;
/**
* This helps to serialize the raw bytes of file-chunk into an UTF-8 encoded string for upload.
@@ -26,6 +26,6 @@ public ByteArraySerializer(Class t) {
*/
@Override
public void serialize(byte[] bytes, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException {
- jsonGenerator.writeRawValue(new String(bytes, StandardCharsets.UTF_8));
+ jsonGenerator.writeRawValue(new String(bytes, Charset.forName(System.getProperty("file.encoding"))));
}
}
diff --git a/java/src/main/resources/logback.xml b/java/src/main/resources/logback.xml
index b9bf660..c86c9b2 100644
--- a/java/src/main/resources/logback.xml
+++ b/java/src/main/resources/logback.xml
@@ -5,9 +5,9 @@
+ value="%d{yyyy-MM-dd HH:mm:ss} %(%-5level) %([%-20.20logger{20}:%(%-5line)]) %(%X{process_id}) |-- %m %ex{5}%n" />
+ value="%d{yyyy-MM-dd HH:mm:ss} %(%-5level) %(%X{process_id}) |-- %m %ex{5}%n" />
diff --git a/java/src/test/java/com/anaplan/client/JDBCCellReaderTest.java b/java/src/test/java/com/anaplan/client/JDBCCellReaderTest.java
index 48b8f0e..38a5bea 100644
--- a/java/src/test/java/com/anaplan/client/JDBCCellReaderTest.java
+++ b/java/src/test/java/com/anaplan/client/JDBCCellReaderTest.java
@@ -20,11 +20,11 @@
public class JDBCCellReaderTest extends BaseTest {
- private static final String testJdbcQueryProperties = "/test-jdbc-query.properties";
+ private static final String testJdbcQueryProperties = "/test-jdbc-query-imports.properties";
private JDBCCellReader jdbcCellReader;
private JDBCConfig jdbcConfig;
- private JDBCConfig loadConfig() throws IOException {
+ public JDBCConfig loadConfig() throws IOException {
Properties jdbcProperties = new Properties();
try {
jdbcProperties.load(JDBCCellReader.class.getResourceAsStream(testJdbcQueryProperties));
diff --git a/java/src/test/java/com/anaplan/client/JDBCCellWriterTest.java b/java/src/test/java/com/anaplan/client/JDBCCellWriterTest.java
new file mode 100644
index 0000000..9d28319
--- /dev/null
+++ b/java/src/test/java/com/anaplan/client/JDBCCellWriterTest.java
@@ -0,0 +1,102 @@
+package com.anaplan.client;
+
+
+import com.anaplan.client.ex.AnaplanAPIException;
+import com.anaplan.client.jdbc.JDBCCellWriter;
+import com.anaplan.client.jdbc.JDBCConfig;
+import com.opencsv.CSVParser;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import static junit.framework.TestCase.assertTrue;
+import static org.junit.Assert.assertEquals;
+
+public class JDBCCellWriterTest extends BaseTest {
+
+ private static final String testJdbcQueryProperties = "/test-jdbc-query-exports.properties";
+ private JDBCCellWriter jdbcCellWriter;
+ private JDBCConfig jdbcConfig;
+
+
+ private JDBCConfig loadConfig() throws IOException {
+ Properties jdbcProperties = new Properties();
+ try {
+ jdbcProperties.load(JDBCCellWriter.class.getResourceAsStream(testJdbcQueryProperties));
+ } catch (IOException e) {
+ throw new RuntimeException("Cannot find test properties!", e);
+ }
+ JDBCConfig jdbcConfig = new JDBCConfig();
+ jdbcConfig.setJdbcConnectionUrl(jdbcProperties.getProperty("jdbc.connect.url"));
+ jdbcConfig.setJdbcQuery(jdbcProperties.getProperty("jdbc.query"));
+ jdbcConfig.setJdbcParams(new CSVParser().parseLine(jdbcProperties.getProperty("jdbc.params")));
+
+ return jdbcConfig;
+ }
+
+ private Method getPrivateMethod(String methodName, Class... argClasses) throws NoSuchMethodException {
+ Method method = JDBCCellWriter.class.getDeclaredMethod(methodName, argClasses);
+ method.setAccessible(true);
+ return method;
+ }
+
+ @Before
+ public void setUp() throws SQLException, IOException {
+ jdbcConfig = loadConfig();
+ jdbcCellWriter = new JDBCCellWriter(jdbcConfig);
+ }
+
+ @After
+ public void tearDown() {
+ jdbcCellWriter.close();
+ }
+
+ @Test
+ public void testSqlInsertQuerySingleRow() throws AnaplanAPIException,IOException,SQLException {
+ int[] mapcols = {0,0,0};
+ InputStream inputStream = new FileInputStream("src/test/resources/files/chunk_1row.txt");
+ assertEquals(1,jdbcCellWriter.writeDataRow("exportId",5,10,inputStream,1,"0",mapcols,3,","));
+ }
+
+ @Test
+ public void testSqlInsertQueryFourRows() throws AnaplanAPIException,IOException,SQLException {
+ int[] mapcols = {0,0,0};
+ InputStream inputStream = new FileInputStream("src/test/resources/files/chunk_2rows.txt");
+ assertEquals(4,jdbcCellWriter.writeDataRow("exportId",5,10,inputStream,1,"0",mapcols,3,","));
+ }
+
+ @Test
+ public void testCorruptedRecordsException() throws AnaplanAPIException,IOException,SQLException {
+ int[] mapcols = {0,0,0,11111};
+ InputStream inputStream = new FileInputStream("src/test/resources/files/chunk_3rows.txt");
+ assertEquals(4,jdbcCellWriter.writeDataRow("exportId",5,10,inputStream,1,"0",mapcols,3,","));
+ }
+
+ @Test
+ public void testGoodSanitizeQuery() throws Exception {
+ String query = jdbcConfig.getJdbcQuery();
+ Method sanitizeQueryMethod = getPrivateMethod("sanitizeQuery", String.class);
+ assertEquals(query, sanitizeQueryMethod.invoke(jdbcCellWriter, query));
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void testBadSanitizeQuery() throws Throwable {
+ String veryLongQuery = new String(new char[2000]).replace("\0", jdbcConfig.getJdbcQuery() + ";");
+ assertTrue(veryLongQuery.length() >= 65535);
+ Method sanitizeQueryMethod = getPrivateMethod("sanitizeQuery", String.class);
+ try {
+ sanitizeQueryMethod.invoke(jdbcCellWriter, veryLongQuery);
+ } catch (InvocationTargetException e) {
+ throw e.getCause();
+ }
+ }
+
+}
diff --git a/java/src/test/java/com/anaplan/client/ServerFileTest.java b/java/src/test/java/com/anaplan/client/ServerFileTest.java
index 155fe76..c1646a3 100644
--- a/java/src/test/java/com/anaplan/client/ServerFileTest.java
+++ b/java/src/test/java/com/anaplan/client/ServerFileTest.java
@@ -14,22 +14,12 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.*;
import java.nio.charset.Charset;
-import static junit.framework.TestCase.assertEquals;
-import static junit.framework.TestCase.assertNotNull;
-import static junit.framework.TestCase.fail;
+import static junit.framework.TestCase.*;
import static org.junit.Assert.assertThat;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.anyString;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.*;
public class ServerFileTest extends BaseTest {
@@ -158,6 +148,36 @@ public void testDownloadStream() throws Exception {
assertFilesEquals(file0v3, check0v3File);
}
+ /**
+ * testing last index Of comma separator
+ * @throws Exception
+ */
+ @Test
+ public void testlastIndexOfComma() throws Exception {
+ int size =100;
+ byte[] buffer = new byte[size];
+ File sourceFile = new File("src/test/resources/files/indexOfCommaSeparator.txt");
+ RandomAccessFile raf = new RandomAccessFile(sourceFile, "r");
+ raf.readFully(buffer, 0, size);
+ int indexOfLastSeparator = mockServerFile.lastIndexOf(buffer,",");
+ assertEquals(90,indexOfLastSeparator);
+ }
+
+ /**
+ * testing last index Of tab separator
+ * @throws Exception
+ */
+ @Test
+ public void testlastIndexOftab() throws Exception {
+ int size =100;
+ byte[] buffer = new byte[size];
+ File sourceFile = new File("src/test/resources/files/indexOfTabSeparator.txt");
+ RandomAccessFile raf = new RandomAccessFile(sourceFile, "r");
+ raf.readFully(buffer, 0, size);
+ int indexOfLastSeparator = mockServerFile.lastIndexOf(buffer,"\t");
+ assertEquals(90,indexOfLastSeparator);
+ }
+
@Test
public void testCreateServerFile() throws Exception {
String newFileName = "NewFile1";
diff --git a/java/src/test/java/com/anaplan/client/auth/AuthRetryTest.java b/java/src/test/java/com/anaplan/client/auth/AuthRetryTest.java
index 2aaa0a6..390129b 100644
--- a/java/src/test/java/com/anaplan/client/auth/AuthRetryTest.java
+++ b/java/src/test/java/com/anaplan/client/auth/AuthRetryTest.java
@@ -37,6 +37,7 @@ public class AuthRetryTest extends BaseTest {
private final String mockAuthServiceUrl = "http://mock-auth.anaplan.com";
private final String mockUsername = "someusername";
private final String mockPassword = "asdasdasdsa";
+ private final Long AuthTokenExpiresAt = System.currentTimeMillis()+300001;
private AnaplanAuthenticationAPI mockAuthApi;
private MockRetryBasicAuthenticator basicAuth;
@@ -84,7 +85,7 @@ public void testRefreshAuthTokenRetryAndFail() throws IOException {
doThrow(new UnknownHostException())
.when(mockClient).execute(Mockito.any(Request.class), Mockito.any(Options.class));
basicAuth.setAuthToken("asdasdsa");
- basicAuth.setAuthTokenExpiresAt(123123123L);
+ basicAuth.setAuthTokenExpiresAt(AuthTokenExpiresAt);
testRetry("null executing POST http://mock-auth.anaplan.com/token/refresh");
}
}
diff --git a/java/src/test/java/com/anaplan/client/auth/BasicAuthenticatorTest.java b/java/src/test/java/com/anaplan/client/auth/BasicAuthenticatorTest.java
index b23f392..7956d17 100644
--- a/java/src/test/java/com/anaplan/client/auth/BasicAuthenticatorTest.java
+++ b/java/src/test/java/com/anaplan/client/auth/BasicAuthenticatorTest.java
@@ -28,7 +28,7 @@
public class BasicAuthenticatorTest extends BaseTest {
private final String mockAuthServiceUrl = "http://mock-auth.anaplan.com";
- private final Long mockAuthTokenExpiresAt = 1496486205L;
+ private final Long mockAuthTokenExpiresAt = System.currentTimeMillis()+300001;
private final String mockAuthToken = "authentication-token";
private final String mockRefreshToken = "refresh-token-value";
private AbstractAuthenticator basicAuth;
diff --git a/java/src/test/resources/files/chunk_1row.txt b/java/src/test/resources/files/chunk_1row.txt
new file mode 100644
index 0000000..3c9e0b1
--- /dev/null
+++ b/java/src/test/resources/files/chunk_1row.txt
@@ -0,0 +1,2 @@
+id,no,col
+id10,10,colB-Value
\ No newline at end of file
diff --git a/java/src/test/resources/files/chunk_2rows.txt b/java/src/test/resources/files/chunk_2rows.txt
new file mode 100644
index 0000000..5edcaaa
--- /dev/null
+++ b/java/src/test/resources/files/chunk_2rows.txt
@@ -0,0 +1,5 @@
+id,no,col
+id10,10,
+id11,11,colB-Value
+id12,,colB-Value
+id13,13,
\ No newline at end of file
diff --git a/java/src/test/resources/files/chunk_3rows.txt b/java/src/test/resources/files/chunk_3rows.txt
new file mode 100644
index 0000000..ac77cc0
--- /dev/null
+++ b/java/src/test/resources/files/chunk_3rows.txt
@@ -0,0 +1,5 @@
+id,no,col
+id10,10,mmmmm
+id11,11,colB-Value,s,d,f,g,g
+id12,,colB-Value,0nkm,lllk
+id13,13,mmmmm
\ No newline at end of file
diff --git a/java/src/test/resources/files/indexOfCommaSeparator.txt b/java/src/test/resources/files/indexOfCommaSeparator.txt
new file mode 100644
index 0000000..34ae0f4
--- /dev/null
+++ b/java/src/test/resources/files/indexOfCommaSeparator.txt
@@ -0,0 +1,25 @@
+,Parent,Code
+c0,工場生産明細,c0
+c1,工場生産明細,c1
+c2,工場生産明細,c2
+c3,工場生産明細,c3
+c4,工場生産明細,c4
+c5,工場生産明細,c5
+c6,工場生産明細,c6
+c7,工場生産明細,c7
+c8,工場生産明細,c8
+c9,工場生産明細,c9
+c10,工場生産明細,c10
+c11,工場生産明細,c11
+c12,工場生産明細,c12
+c13,工場生産明細,c13
+c14,工場生産明細,c14
+c15,工場生産明細,c15
+c16,工場生産明細,c16
+c17,工場生産明細,c17
+c18,工場生産明細,c18
+c19,工場生産明細,c19
+c20,工場生産明細,c20
+c21,工場生産明細,c21
+c22,工場生産明細,c22
+c23,工場生産明細,c23
\ No newline at end of file
diff --git a/java/src/test/resources/files/indexOfTabSeparator.txt b/java/src/test/resources/files/indexOfTabSeparator.txt
new file mode 100644
index 0000000..3d9348d
--- /dev/null
+++ b/java/src/test/resources/files/indexOfTabSeparator.txt
@@ -0,0 +1,32 @@
+ Parent Code
+c0 工場生産明細 c0
+c1 工場生産明細 c1
+c2 工場生産明細 c2
+c3 工場生産明細 c3
+c4 工場生産明細 c4
+c5 工場生産明細 c5
+c6 工場生産明細 c6
+c7 工場生産明細 c7
+c8 工場生産明細 c8
+c9 工場生産明細 c9
+c10 工場生産明細 c10
+c11 工場生産明細 c11
+c12 工場生産明細 c12
+c13 工場生産明細 c13
+c14 工場生産明細 c14
+c15 工場生産明細 c15
+c16 工場生産明細 c16
+c17 工場生産明細 c17
+c18 工場生産明細 c18
+c19 工場生産明細 c19
+c20 工場生産明細 c20
+c21 工場生産明細 c21
+c22 工場生産明細 c22
+c23 工場生産明細 c23
+c24 工場生産明細 c24
+c25 工場生産明細 c25
+c26 工場生産明細 c26
+c27 工場生産明細 c27
+c28 工場生産明細 c28
+c29 工場生産明細 c29
+c30 工場生産明細 c30
\ No newline at end of file
diff --git a/java/src/test/resources/h2_init.sql b/java/src/test/resources/h2_init.sql
index 1832c4c..d21a64d 100644
--- a/java/src/test/resources/h2_init.sql
+++ b/java/src/test/resources/h2_init.sql
@@ -6,6 +6,12 @@ create memory table TestUserTable (
colC varchar(255) not null
);
+create memory table TestExportTable (
+ colA varchar(255) not null PRIMARY KEY,
+ colB varchar(255),
+ colC varchar(255)
+);
+
insert into TestUserTable values ('id1', 1, 'colB-Value1');
insert into TestUserTable values ('id2', 2, 'colB-Value2');
insert into TestUserTable values ('id3', 3, 'colB-Value3');
diff --git a/java/src/test/resources/test-jdbc-query-exports.properties b/java/src/test/resources/test-jdbc-query-exports.properties
new file mode 100644
index 0000000..a36eb98
--- /dev/null
+++ b/java/src/test/resources/test-jdbc-query-exports.properties
@@ -0,0 +1,4 @@
+jdbc.connect.url=jdbc:h2:mem:testh2;MODE=MySQL;INIT=runscript from 'src/test/resources/h2_init.sql'
+jdbc.username=
+jdbc.password=
+jdbc.query=insert into TestExportTable values (?, ?, ?)
diff --git a/java/src/test/resources/test-jdbc-query.properties b/java/src/test/resources/test-jdbc-query-imports.properties
similarity index 100%
rename from java/src/test/resources/test-jdbc-query.properties
rename to java/src/test/resources/test-jdbc-query-imports.properties