diff --git a/.idea/misc.xml b/.idea/misc.xml index cd0eadb1..2139cc15 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,5 +1,8 @@ + + + diff --git a/build.gradle b/build.gradle index 172775b2..eda0c8b0 100644 --- a/build.gradle +++ b/build.gradle @@ -13,6 +13,7 @@ compileTestJava { } repositories { + // maven { url uri('/home/gspowley/.m2/repository/')} mavenLocal() mavenCentral() maven { url "https://artifactory.broadinstitute.org/artifactory/libs-snapshot/" } @@ -23,8 +24,7 @@ dependencies { compile 'org.broadinstitute:gatk-native-bindings:0.0.3' compile 'org.apache.logging.log4j:log4j-api:2.5' compile 'org.apache.logging.log4j:log4j-core:2.5' - compile 'com.github.samtools:htsjdk:2.7.0-18-g4d0070b-SNAPSHOT' - + compile 'com.github.samtools:htsjdk:2.9.0' testCompile 'org.testng:testng:6.9.9' } diff --git a/src/main/java/com/intel/gkl/compression/IntelDeflater.java b/src/main/java/com/intel/gkl/compression/IntelDeflater.java index ed06a963..4953e547 100644 --- a/src/main/java/com/intel/gkl/compression/IntelDeflater.java +++ b/src/main/java/com/intel/gkl/compression/IntelDeflater.java @@ -79,6 +79,7 @@ public synchronized boolean load(File tempDir) { private int level; private int strategy; private boolean nowrap; + /** * Creates a new compressor using the specified compression level. @@ -119,6 +120,7 @@ public void reset() { resetNative(nowrap); inputBuffer = null; inputBufferLength = 0; + endOfStream = false; finished = false; } @@ -139,8 +141,11 @@ public void setInput(byte[] b, int off, int len) throws NullPointerException { if(len <= 0) { throw new NullPointerException("Input buffer length is zero."); } + inputBuffer = b; inputBufferLength = len; + + } /** @@ -166,6 +171,7 @@ public void finish() { */ @Override public int deflate(byte[] b, int off, int len ) { + return deflateNative(b, len); } diff --git a/src/main/java/com/intel/gkl/compression/IntelInflater.java b/src/main/java/com/intel/gkl/compression/IntelInflater.java index d3c3b284..f19c3929 100644 --- a/src/main/java/com/intel/gkl/compression/IntelInflater.java +++ b/src/main/java/com/intel/gkl/compression/IntelInflater.java @@ -69,12 +69,13 @@ public synchronized boolean load(File tempDir) { private long lz_stream; private byte[] inputBuffer; private int inputBufferLength; + private int inputBufferOffset; private boolean finished; private boolean nowrap; private static native void initNative(); private native void resetNative(boolean nowrap); - private native int inflateNative( byte[] b, int len); + private native int inflateNative( byte[] b, int off, int len); private native void endNative(); /** @@ -122,6 +123,7 @@ public void setInput(byte[] b, int off, int len) throws NullPointerException { throw new NullPointerException("Input buffer length is zero."); } inputBuffer = b; + inputBufferOffset = off; inputBufferLength = len; } @@ -139,7 +141,7 @@ public void setInput(byte[] b, int off, int len) throws NullPointerException { */ @Override public int inflate (byte[] b, int off, int len ) { - return inflateNative(b, len); + return inflateNative(b, off, len); } /** @@ -150,7 +152,7 @@ public int inflate (byte[] b, int off, int len ) { */ @Override public int inflate (byte[] b ) { - return inflateNative( b, 0); + return inflateNative( b, 0, 0); } /** diff --git a/src/main/java/com/intel/gkl/compression/IntelInflaterFactory.java b/src/main/java/com/intel/gkl/compression/IntelInflaterFactory.java new file mode 100644 index 00000000..9aaaade0 --- /dev/null +++ b/src/main/java/com/intel/gkl/compression/IntelInflaterFactory.java @@ -0,0 +1,37 @@ +package com.intel.gkl.compression; + +import htsjdk.samtools.util.zip.InflaterFactory; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.File; +import java.util.zip.Inflater; + +/** + * Created by pnvaidya on 2/1/17. + */ +public class IntelInflaterFactory extends InflaterFactory { + private final static Logger logger = LogManager.getLogger(IntelDeflaterFactory.class); + private boolean intelInflaterSupported; + + public IntelInflaterFactory(File tmpDir) { + intelInflaterSupported = new IntelInflater().load(tmpDir); + } + + public IntelInflaterFactory() { + this(null); + } + + public Inflater makeInflater(final boolean nowrap) { + if (intelInflaterSupported && nowrap) { + return new IntelInflater(nowrap); + } + logger.warn("IntelInflater is not supported, using Java.util.zip.Inflater"); + return new Inflater(nowrap); + } + + public boolean usingIntelInflater() { + return intelInflaterSupported; + } +} diff --git a/src/main/native/compression/IntelDeflater.cc b/src/main/native/compression/IntelDeflater.cc index 76b0d5e6..676a3194 100644 --- a/src/main/native/compression/IntelDeflater.cc +++ b/src/main/native/compression/IntelDeflater.cc @@ -54,8 +54,8 @@ static jfieldID FID_inputBuffer; static jfieldID FID_inputBufferLength; static jfieldID FID_endOfStream; static jfieldID FID_finished; -static jfieldID FID_finish; static jfieldID FID_level; +//static jfieldID FID_inputBufferOffset; @@ -69,8 +69,8 @@ JNIEXPORT void JNICALL Java_com_intel_gkl_compression_IntelDeflater_initNative FID_inputBufferLength = env->GetFieldID(cls, "inputBufferLength", "I"); FID_endOfStream = env->GetFieldID(cls, "endOfStream", "Z"); FID_finished = env->GetFieldID(cls, "finished", "Z"); - FID_finish = env->GetFieldID(cls, "finish", "Z"); FID_level = env->GetFieldID(cls,"level","I"); +// FID_inputBufferOffset = env->GetFieldID(cls, "inputBufferOffset", "I"); } @@ -185,9 +185,10 @@ JNIEXPORT jint JNICALL Java_com_intel_gkl_compression_IntelDeflater_deflateNativ jbyteArray inputBuffer = (jbyteArray)env->GetObjectField(obj, FID_inputBuffer); jint inputBufferLength = env->GetIntField(obj, FID_inputBufferLength); + //jint inputBufferOffset = env->GetIntField(obj, FID_inputBufferOffset); jboolean endOfStream = env->GetBooleanField(obj, FID_endOfStream); jint level = env->GetIntField(obj, FID_level); - jboolean finish = env->GetBooleanField(obj, FID_finish); + if(level == 1) { @@ -198,11 +199,11 @@ JNIEXPORT jint JNICALL Java_com_intel_gkl_compression_IntelDeflater_deflateNativ jbyte* next_in = (jbyte*)env->GetPrimitiveArrayCritical(inputBuffer, 0); jbyte* next_out = (jbyte*)env->GetPrimitiveArrayCritical(outputBuffer, 0); - lz_stream->next_in = (UINT8*)next_in; + lz_stream->next_in = (UINT8*)(next_in); lz_stream->avail_in = inputBufferLength; lz_stream->end_of_stream = endOfStream; - lz_stream->next_out = (UINT8*)next_out; - lz_stream->avail_out = outputBufferLength; + lz_stream->next_out = (UINT8*) (next_out); + lz_stream->avail_out = outputBufferLength ; int bytes_in = inputBufferLength; @@ -316,4 +317,5 @@ Java_com_intel_gkl_compression_IntelDeflater_endNative(JNIEnv *env, jobject obj) if (level != 1) { deflateEnd(lz_stream); } + else free(lz_stream); } diff --git a/src/main/native/compression/IntelInflater.cc b/src/main/native/compression/IntelInflater.cc index d7a4bfef..c01cf0ab 100644 --- a/src/main/native/compression/IntelInflater.cc +++ b/src/main/native/compression/IntelInflater.cc @@ -51,6 +51,7 @@ static jfieldID FID_inf_lz_stream; static jfieldID FID_inf_inputBuffer; static jfieldID FID_inf_inputBufferLength; static jfieldID FID_inf_finished; +static jfieldID FID_inf_inputBufferOffset; @@ -63,6 +64,7 @@ JNIEXPORT void JNICALL Java_com_intel_gkl_compression_IntelInflater_initNative FID_inf_inputBuffer = env->GetFieldID(cls, "inputBuffer", "[B"); FID_inf_inputBufferLength = env->GetFieldID(cls, "inputBufferLength", "I"); FID_inf_finished = env->GetFieldID(cls, "finished", "Z"); + FID_inf_inputBufferOffset = env->GetFieldID(cls, "inputBufferOffset", "I"); } @@ -81,9 +83,7 @@ JNIEXPORT void JNICALL Java_com_intel_gkl_compression_IntelInflater_resetNative } env->SetLongField(obj, FID_inf_lz_stream, (jlong)lz_stream); - - - fprintf(stdout,"%c",nowrap); + } isal_inflate_init(lz_stream); lz_stream->avail_in = 0; @@ -91,13 +91,11 @@ JNIEXPORT void JNICALL Java_com_intel_gkl_compression_IntelInflater_resetNative lz_stream->avail_out = 0; lz_stream->next_out = Z_NULL; - } - } JNIEXPORT jint JNICALL Java_com_intel_gkl_compression_IntelInflater_inflateNative -(JNIEnv * env, jobject obj, jbyteArray outputBuffer, jint outputBufferLength) +(JNIEnv * env, jobject obj, jbyteArray outputBuffer, jint outputBufferOffset, jint outputBufferLength) { @@ -106,13 +104,16 @@ JNIEXPORT jint JNICALL Java_com_intel_gkl_compression_IntelInflater_inflateNativ jbyteArray inputBuffer = (jbyteArray)env->GetObjectField(obj, FID_inf_inputBuffer); jint inputBufferLength = env->GetIntField(obj, FID_inf_inputBufferLength); + jint inputBufferOffset = env->GetIntField(obj, FID_inf_inputBufferOffset); + jbyte* next_in = (jbyte*)env->GetPrimitiveArrayCritical(inputBuffer, 0); jbyte* next_out = (jbyte*)env->GetPrimitiveArrayCritical(outputBuffer, 0); - lz_stream->next_in = (Bytef *) next_in; + + lz_stream->next_in = (Bytef *) (next_in + inputBufferOffset); lz_stream->avail_in = (uInt) inputBufferLength; - lz_stream->next_out = (Bytef *) next_out; + lz_stream->next_out = (Bytef *) (next_out + outputBufferOffset); lz_stream->avail_out = (uInt) outputBufferLength; int bytes_in = inputBufferLength; diff --git a/src/main/native/compression/IntelInflater.h b/src/main/native/compression/IntelInflater.h index cfa2c6ae..9aa2e677 100644 --- a/src/main/native/compression/IntelInflater.h +++ b/src/main/native/compression/IntelInflater.h @@ -29,7 +29,7 @@ JNIEXPORT void JNICALL Java_com_intel_gkl_compression_IntelInflater_resetNative * Signature: ([BI)I */ JNIEXPORT jint JNICALL Java_com_intel_gkl_compression_IntelInflater_inflateNative - (JNIEnv *, jobject, jbyteArray, jint); + (JNIEnv *, jobject, jbyteArray, jint, jint); /* * Class: com_intel_gkl_compression_IntelInflater diff --git a/src/test/java/com/intel/gkl/compression/DeflaterIntegrationTest.java b/src/test/java/com/intel/gkl/compression/DeflaterIntegrationTest.java index a12cf7a9..2cd6f3dc 100644 --- a/src/test/java/com/intel/gkl/compression/DeflaterIntegrationTest.java +++ b/src/test/java/com/intel/gkl/compression/DeflaterIntegrationTest.java @@ -59,6 +59,7 @@ public Deflater makeDeflater(final int compressionLevel, final boolean nowrap) { long totalRecords = 0; try (final SamReader reader = readerFactory.open(inputFile)) { final SAMFileHeader header = reader.getFileHeader(); + final SAMFileWriterFactory writerFactory = new SAMFileWriterFactory(); writerFactory.setCompressionLevel(compressionLevel); writerFactory.setDeflaterFactory(deflaterFactory); diff --git a/src/test/java/com/intel/gkl/compression/InflaterUnitTest.java b/src/test/java/com/intel/gkl/compression/InflaterUnitTest.java new file mode 100644 index 00000000..bdfbe24d --- /dev/null +++ b/src/test/java/com/intel/gkl/compression/InflaterUnitTest.java @@ -0,0 +1,73 @@ +package com.intel.gkl.compression; + +import com.intel.gkl.IntelGKLUtils; +import htsjdk.samtools.util.BlockCompressedInputStream; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.io.File; +import java.io.IOException; +import java.util.zip.Deflater; +import java.util.zip.Inflater; + +/** + * Created by pnvaidya on 2/1/17. + */ +public class InflaterUnitTest { + + private final static Logger log = LogManager.getLogger(InflaterUnitTest.class); + private final static String INPUT_FILE = IntelGKLUtils.pathToTestResource("HiSeq.1mb.1RG.2k_lines.bam"); + + @Test(enabled = true) + public void inflaterUnitTest() throws IOException { + + final File inputFile = new File(INPUT_FILE); + + long inputBytes = inputFile.length(); + int compressedBytes = 0; + final byte[] inputbuffer = new byte[(int)inputBytes]; + final byte[] outputbuffer = new byte[(int)inputBytes]; + final byte[] finalbuffer = new byte[(int)inputBytes]; + + long totalTime = 0; + + for (int i = 1; i < 10; i++) { + + final IntelInflaterFactory intelInflaterFactory = new IntelInflaterFactory(); + final Inflater inflater = intelInflaterFactory.makeInflater(true); + + final IntelDeflaterFactory intelDeflaterFactory = new IntelDeflaterFactory(); + final Deflater deflater = intelDeflaterFactory.makeDeflater(i, true); + + + final BlockCompressedInputStream inputStream = new BlockCompressedInputStream(inputFile); + + inputBytes = inputStream.read(inputbuffer, 0, inputbuffer.length); + deflater.reset(); + deflater.setInput(inputbuffer, 0, inputbuffer.length); + deflater.finish(); + compressedBytes = deflater.deflate(outputbuffer, 0, outputbuffer.length); + + + try { + inflater.reset(); + inflater.setInput(outputbuffer, 0, compressedBytes); + final long start = System.currentTimeMillis(); + inflater.inflate(finalbuffer, 0, inputbuffer.length); + totalTime = System.currentTimeMillis() - start; + System.out.printf("Level: %d, time: %d\n", i, totalTime); + + + } catch (java.util.zip.DataFormatException e) { + e.printStackTrace(); + } + Assert.assertEquals(inputbuffer, finalbuffer); + inflater.end(); + deflater.end(); + } + + + } +}