Merge pull request #45 from Intel-HLS/pv_inflaterfactory

Pv inflaterfactory

build.gradle

@@ -13,6 +13,7 @@ compileTestJava {
 }
 
 repositories {
+   // maven { url uri('/home/gspowley/.m2/repository/')}
     mavenLocal()
     mavenCentral()
     maven { url "https://artifactory.broadinstitute.org/artifactory/libs-snapshot/" }
@@ -23,8 +24,7 @@ dependencies {
     compile 'org.broadinstitute:gatk-native-bindings:0.0.3'
     compile 'org.apache.logging.log4j:log4j-api:2.5'
     compile 'org.apache.logging.log4j:log4j-core:2.5'
-    compile 'com.github.samtools:htsjdk:2.7.0-18-g4d0070b-SNAPSHOT'
-
+    compile 'com.github.samtools:htsjdk:2.9.0'
     testCompile 'org.testng:testng:6.9.9'
 }
 

src/main/java/com/intel/gkl/compression/IntelDeflater.java

@@ -79,6 +79,7 @@ public synchronized boolean load(File tempDir) {
     private int level;
     private int strategy;
     private boolean nowrap;
+
 
      /**
      * Creates a new compressor using the specified compression level.
@@ -119,6 +120,7 @@ public void reset() {
         resetNative(nowrap);
         inputBuffer = null;
         inputBufferLength = 0;
+
         endOfStream = false;
         finished = false;
     }
@@ -139,8 +141,11 @@ public void setInput(byte[] b, int off, int len) throws NullPointerException {
         if(len <= 0) {
             throw new NullPointerException("Input buffer length is zero.");
         }
+
         inputBuffer = b;
         inputBufferLength = len;
+
+
     }
 
     /**
@@ -166,6 +171,7 @@ public void finish() {
      */
     @Override
     public int deflate(byte[] b, int off, int len ) {
+
         return deflateNative(b, len);
     }
 

src/main/java/com/intel/gkl/compression/IntelInflater.java

@@ -69,12 +69,13 @@ public synchronized boolean load(File tempDir) {
     private long lz_stream;
     private byte[] inputBuffer;
     private int inputBufferLength;
+    private int inputBufferOffset;
     private boolean finished;
     private boolean nowrap;
 
     private static native void initNative();
     private native void resetNative(boolean nowrap);
-    private native int inflateNative( byte[] b, int len);
+    private native int inflateNative( byte[] b, int off, int len);
     private native void endNative();
 
     /**
@@ -122,6 +123,7 @@ public void setInput(byte[] b, int off, int len) throws NullPointerException {
             throw new NullPointerException("Input buffer length is zero.");
         }
         inputBuffer = b;
+        inputBufferOffset = off;
         inputBufferLength = len;
     }
 
@@ -139,7 +141,7 @@ public void setInput(byte[] b, int off, int len) throws NullPointerException {
      */
     @Override
     public int inflate (byte[] b, int off, int len ) {
-        return inflateNative(b, len);
+        return inflateNative(b, off, len);
     }
 
     /**
@@ -150,7 +152,7 @@ public int inflate (byte[] b, int off, int len ) {
      */
     @Override
     public int inflate (byte[] b ) {
-        return inflateNative( b, 0);
+        return inflateNative( b, 0, 0);
     }
 
     /**

src/main/java/com/intel/gkl/compression/IntelInflaterFactory.java

@@ -0,0 +1,37 @@
+package com.intel.gkl.compression;
+
+import htsjdk.samtools.util.zip.InflaterFactory;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.File;
+import java.util.zip.Inflater;
+
+/**
+ * Created by pnvaidya on 2/1/17.
+ */
+public class IntelInflaterFactory extends InflaterFactory {
+    private final static Logger logger = LogManager.getLogger(IntelDeflaterFactory.class);
+    private boolean intelInflaterSupported;
+
+    public IntelInflaterFactory(File tmpDir) {
+        intelInflaterSupported = new IntelInflater().load(tmpDir);
+    }
+
+    public IntelInflaterFactory() {
+        this(null);
+    }
+
+    public Inflater makeInflater(final boolean nowrap) {
+        if (intelInflaterSupported && nowrap) {
+            return new IntelInflater(nowrap);
+        }
+        logger.warn("IntelInflater is not supported, using Java.util.zip.Inflater");
+        return new Inflater(nowrap);
+    }
+
+    public boolean usingIntelInflater() {
+        return intelInflaterSupported;
+    }
+}

src/main/native/compression/IntelDeflater.cc

@@ -54,8 +54,8 @@ static jfieldID FID_inputBuffer;
 static jfieldID FID_inputBufferLength;
 static jfieldID FID_endOfStream;
 static jfieldID FID_finished;
-static jfieldID FID_finish;
 static jfieldID FID_level;
+//static jfieldID FID_inputBufferOffset;
 
 
 
@@ -69,8 +69,8 @@ JNIEXPORT void JNICALL Java_com_intel_gkl_compression_IntelDeflater_initNative
   FID_inputBufferLength = env->GetFieldID(cls, "inputBufferLength", "I");
   FID_endOfStream = env->GetFieldID(cls, "endOfStream", "Z");
   FID_finished = env->GetFieldID(cls, "finished", "Z");
-  FID_finish = env->GetFieldID(cls, "finish", "Z");
   FID_level = env->GetFieldID(cls,"level","I");
+//  FID_inputBufferOffset = env->GetFieldID(cls, "inputBufferOffset", "I");
 
 }
 
@@ -185,9 +185,10 @@ JNIEXPORT jint JNICALL Java_com_intel_gkl_compression_IntelDeflater_deflateNativ
 
   jbyteArray inputBuffer = (jbyteArray)env->GetObjectField(obj, FID_inputBuffer);
   jint inputBufferLength = env->GetIntField(obj, FID_inputBufferLength);
+  //jint inputBufferOffset = env->GetIntField(obj, FID_inputBufferOffset);
   jboolean endOfStream = env->GetBooleanField(obj, FID_endOfStream);
   jint level = env->GetIntField(obj, FID_level);
-  jboolean finish = env->GetBooleanField(obj, FID_finish);
+
 
   if(level == 1) {
 
@@ -198,11 +199,11 @@ JNIEXPORT jint JNICALL Java_com_intel_gkl_compression_IntelDeflater_deflateNativ
     jbyte* next_in = (jbyte*)env->GetPrimitiveArrayCritical(inputBuffer, 0);
     jbyte* next_out = (jbyte*)env->GetPrimitiveArrayCritical(outputBuffer, 0);
 
-    lz_stream->next_in = (UINT8*)next_in;
+    lz_stream->next_in = (UINT8*)(next_in);
     lz_stream->avail_in = inputBufferLength;
     lz_stream->end_of_stream = endOfStream;
-    lz_stream->next_out = (UINT8*)next_out;
-    lz_stream->avail_out = outputBufferLength;
+    lz_stream->next_out = (UINT8*) (next_out);
+    lz_stream->avail_out = outputBufferLength ;
 
     int bytes_in = inputBufferLength;
 
@@ -316,4 +317,5 @@ Java_com_intel_gkl_compression_IntelDeflater_endNative(JNIEnv *env, jobject obj)
   if (level != 1) {
     deflateEnd(lz_stream);
   }
+  else free(lz_stream);
 }

src/main/native/compression/IntelInflater.cc

@@ -51,6 +51,7 @@ static jfieldID FID_inf_lz_stream;
 static jfieldID FID_inf_inputBuffer;
 static jfieldID FID_inf_inputBufferLength;
 static jfieldID FID_inf_finished;
+static jfieldID FID_inf_inputBufferOffset;
 
 
 
@@ -63,6 +64,7 @@ JNIEXPORT void JNICALL Java_com_intel_gkl_compression_IntelInflater_initNative
   FID_inf_inputBuffer = env->GetFieldID(cls, "inputBuffer", "[B");
   FID_inf_inputBufferLength = env->GetFieldID(cls, "inputBufferLength", "I");
   FID_inf_finished = env->GetFieldID(cls, "finished", "Z");
+  FID_inf_inputBufferOffset = env->GetFieldID(cls, "inputBufferOffset", "I");
 
 }
 
@@ -81,23 +83,19 @@ JNIEXPORT void JNICALL Java_com_intel_gkl_compression_IntelInflater_resetNative
       }
       env->SetLongField(obj, FID_inf_lz_stream, (jlong)lz_stream);
 
-
-
-      fprintf(stdout,"%c",nowrap);
+    }
       isal_inflate_init(lz_stream);
 
       lz_stream->avail_in = 0;
       lz_stream->next_in = Z_NULL;
       lz_stream->avail_out = 0;
       lz_stream->next_out = Z_NULL;
 
-     }
-
 }
 
 
 JNIEXPORT jint JNICALL Java_com_intel_gkl_compression_IntelInflater_inflateNative
-(JNIEnv * env, jobject obj, jbyteArray outputBuffer, jint outputBufferLength)
+(JNIEnv * env, jobject obj, jbyteArray outputBuffer, jint outputBufferOffset, jint outputBufferLength)
 {
 
 
@@ -106,13 +104,16 @@ JNIEXPORT jint JNICALL Java_com_intel_gkl_compression_IntelInflater_inflateNativ
 
     jbyteArray inputBuffer = (jbyteArray)env->GetObjectField(obj, FID_inf_inputBuffer);
       jint inputBufferLength = env->GetIntField(obj, FID_inf_inputBufferLength);
+      jint inputBufferOffset = env->GetIntField(obj, FID_inf_inputBufferOffset);
+
 
         jbyte* next_in = (jbyte*)env->GetPrimitiveArrayCritical(inputBuffer, 0);
         jbyte* next_out = (jbyte*)env->GetPrimitiveArrayCritical(outputBuffer, 0);
 
-        lz_stream->next_in = (Bytef *) next_in;
+
+        lz_stream->next_in = (Bytef *) (next_in + inputBufferOffset);
         lz_stream->avail_in = (uInt) inputBufferLength;
-        lz_stream->next_out = (Bytef *) next_out;
+        lz_stream->next_out = (Bytef *) (next_out + outputBufferOffset);
         lz_stream->avail_out = (uInt) outputBufferLength;
 
         int bytes_in = inputBufferLength;

src/test/java/com/intel/gkl/compression/DeflaterIntegrationTest.java

@@ -59,6 +59,7 @@ public Deflater makeDeflater(final int compressionLevel, final boolean nowrap) {
                 long totalRecords = 0;
                 try (final SamReader reader = readerFactory.open(inputFile)) {
                     final SAMFileHeader header = reader.getFileHeader();
+
                     final SAMFileWriterFactory writerFactory = new SAMFileWriterFactory();
                     writerFactory.setCompressionLevel(compressionLevel);
                     writerFactory.setDeflaterFactory(deflaterFactory);

src/test/java/com/intel/gkl/compression/InflaterUnitTest.java

@@ -0,0 +1,73 @@
+package com.intel.gkl.compression;
+
+import com.intel.gkl.IntelGKLUtils;
+import htsjdk.samtools.util.BlockCompressedInputStream;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.zip.Deflater;
+import java.util.zip.Inflater;
+
+/**
+ * Created by pnvaidya on 2/1/17.
+ */
+public class InflaterUnitTest {
+
+    private final static Logger log = LogManager.getLogger(InflaterUnitTest.class);
+    private final static String INPUT_FILE = IntelGKLUtils.pathToTestResource("HiSeq.1mb.1RG.2k_lines.bam");
+
+    @Test(enabled = true)
+    public void inflaterUnitTest() throws IOException {
+
+        final File inputFile = new File(INPUT_FILE);
+
+        long inputBytes = inputFile.length();
+        int compressedBytes = 0;
+        final byte[] inputbuffer = new byte[(int)inputBytes];
+        final byte[] outputbuffer = new byte[(int)inputBytes];
+        final byte[] finalbuffer = new byte[(int)inputBytes];
+
+        long totalTime = 0;
+
+        for (int i = 1; i < 10; i++) {
+
+            final IntelInflaterFactory intelInflaterFactory = new IntelInflaterFactory();
+            final Inflater inflater = intelInflaterFactory.makeInflater(true);
+
+            final IntelDeflaterFactory intelDeflaterFactory = new IntelDeflaterFactory();
+            final Deflater deflater = intelDeflaterFactory.makeDeflater(i, true);
+
+
+            final BlockCompressedInputStream inputStream = new BlockCompressedInputStream(inputFile);
+
+            inputBytes = inputStream.read(inputbuffer, 0, inputbuffer.length);
+            deflater.reset();
+            deflater.setInput(inputbuffer, 0, inputbuffer.length);
+            deflater.finish();
+            compressedBytes = deflater.deflate(outputbuffer, 0, outputbuffer.length);
+
+
+            try {
+                inflater.reset();
+                inflater.setInput(outputbuffer, 0, compressedBytes);
+                final long start = System.currentTimeMillis();
+                inflater.inflate(finalbuffer, 0, inputbuffer.length);
+                totalTime = System.currentTimeMillis() - start;
+                System.out.printf("Level: %d, time: %d\n", i, totalTime);
+
+
+            } catch (java.util.zip.DataFormatException e) {
+                e.printStackTrace();
+            }
+            Assert.assertEquals(inputbuffer, finalbuffer);
+            inflater.end();
+            deflater.end();
+        }
+
+
+    }
+}