From e90e0318bd711ad3b6b21c372b868463790fbe72 Mon Sep 17 00:00:00 2001 From: rharkes Date: Wed, 11 Sep 2024 00:20:31 +0200 Subject: [PATCH 1/2] Add LZ4 compression/decompression --- src/main/java/ome/codecs/LZ4Codec.java | 116 +++++++++++++++++++++ src/test/java/ome/codecs/LZ4CodecTest.java | 49 +++++++++ 2 files changed, 165 insertions(+) create mode 100644 src/main/java/ome/codecs/LZ4Codec.java create mode 100644 src/test/java/ome/codecs/LZ4CodecTest.java diff --git a/src/main/java/ome/codecs/LZ4Codec.java b/src/main/java/ome/codecs/LZ4Codec.java new file mode 100644 index 00000000..27351aa1 --- /dev/null +++ b/src/main/java/ome/codecs/LZ4Codec.java @@ -0,0 +1,116 @@ +/* + * #%L + * Image encoding and decoding routines. + * %% + * Copyright (C) 2005 - 2022 Open Microscopy Environment: + * - Board of Regents of the University of Wisconsin-Madison + * - Glencoe Software, Inc. + * - University of Dundee + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +package ome.codecs; + +import loci.common.RandomAccessInputStream; +import java.io.IOException; +import java.util.Arrays; + +import io.airlift.compress.MalformedInputException; +import io.airlift.compress.lz4.Lz4Decompressor; +import io.airlift.compress.lz4.Lz4Compressor; + + +/** + * This class implements LZ4 decompression. + * + * @author Rolf Harkes r.harkes at nki.nl + */ +public class LZ4Codec extends BaseCodec { + + /* @see BaseCodec#compress(byte[], CodecOptions) */ + @Override + public byte[] compress(byte[] data, CodecOptions options) + throws CodecException + { + if (data == null || data.length == 0) + throw new IllegalArgumentException("No data to compress"); + Lz4Compressor compressor = new Lz4Compressor(); + byte[] output = new byte[compressor.maxCompressedLength(data.length)]; + int len = compressor.compress(data,0, data.length, output, 0, output.length); + return Arrays.copyOfRange(output, 0, len); + } + + /* @see BaseCodec#decompress(RandomAccessInputStream, CodecOptions) */ + @Override + public byte[] decompress(RandomAccessInputStream in, CodecOptions options) + throws CodecException, IOException + { + long byteCount = in.length() - in.getFilePointer(); + if (byteCount > Integer.MAX_VALUE || byteCount < 0) { + throw new CodecException("Integer overflow detected when calculating file byteCount."); + } + byte[] data = new byte[(int) byteCount]; + in.readFully(data); + return decompress(data, options); + } + + /* @see BaseCodec#decompress(byte[]) */ + @Override + public byte[] decompress(byte[] data) + throws CodecException + { + return decompress(data, 0, data.length, data.length*255); + } + + /* @see BaseCodec#decompress(byte[], CodecOptions) */ + @Override + public byte[] decompress(byte[] data, CodecOptions options) + throws CodecException + { + return decompress(data, 0, data.length, options.maxBytes); + } + + /** + * Decompresses a block of data of specified length from an initial offset. + * + * @param data The data to be decompressed. + * @param inputOffset The position of the input data at which to begin decompression. + * @param length The length of input data to be decompressed. + * @return The decompressed data. + * @throws CodecException If data is not valid. + */ + public byte[] decompress(byte[] data, int inputOffset, int length, int outputLength) + throws CodecException + { + Lz4Decompressor decompressor = new Lz4Decompressor(); + byte[] output = new byte[outputLength]; + try { + decompressor.decompress(data, inputOffset, length, output, 0, outputLength); + } + catch(MalformedInputException e) { + throw new CodecException(e); + } + return output; + } +} diff --git a/src/test/java/ome/codecs/LZ4CodecTest.java b/src/test/java/ome/codecs/LZ4CodecTest.java new file mode 100644 index 00000000..8df942da --- /dev/null +++ b/src/test/java/ome/codecs/LZ4CodecTest.java @@ -0,0 +1,49 @@ +package ome.codecs; + +import org.testng.annotations.Test; + +import static org.testng.AssertJUnit.assertEquals; + +public class LZ4CodecTest { + + LZ4Codec codec = new LZ4Codec(); + + @Test + public void testCompressShortUniqueSequence() throws Exception { + byte[] expected = {-96,1,2,3,4,5,6,7,8,9,10}; + byte[] in = {1,2,3,4,5,6,7,8,9,10}; + CodecOptions options = new CodecOptions(); + byte[] comp = codec.compress(in, options); + assertEquals(expected, comp); + } + + @Test + public void testDecompressShortUniqueSequence() throws Exception { + byte[] in = {-96,1,2,3,4,5,6,7,8,9,10}; + byte[] expected = {1,2,3,4,5,6,7,8,9,10}; + CodecOptions options = new CodecOptions(); + options.maxBytes=expected.length; + byte[] comp = codec.decompress(in, options); + assertEquals(expected, comp); + } + + @Test + public void testCompressLongSeq() throws Exception { + byte[] expected = {21,0,1,0,21,1,1,0,21,2,1,0,21,3,1,0,21,4,1,0,21,5,1,0,21,6,1,0,21,7,1,0,21,8,1,0,21,9,1,0,21,10,1,0,21,11,1,0,21,12,1,0,21,13,1,0,21,14,1,0,21,15,1,0,21,16,1,0,21,17,1,0,21,18,1,0,21,19,1,0,21,20,1,0,21,21,1,0,21,22,1,0,21,23,1,0,21,24,1,0,21,25,1,0,21,26,1,0,21,27,1,0,21,28,1,0,21,29,1,0,21,30,1,0,21,31,1,0,21,32,1,0,21,33,1,0,21,34,1,0,21,35,1,0,21,36,1,0,21,37,1,0,21,38,1,0,21,39,1,0,21,40,1,0,21,41,1,0,21,42,1,0,21,43,1,0,21,44,1,0,21,45,1,0,21,46,1,0,21,47,1,0,21,48,1,0,21,49,1,0,21,50,1,0,21,51,1,0,21,52,1,0,21,53,1,0,21,54,1,0,21,55,1,0,21,56,1,0,21,57,1,0,21,58,1,0,21,59,1,0,21,60,1,0,21,61,1,0,21,62,1,0,21,63,1,0,21,64,1,0,21,65,1,0,21,66,1,0,21,67,1,0,21,68,1,0,21,69,1,0,21,70,1,0,21,71,1,0,21,72,1,0,21,73,1,0,21,74,1,0,21,75,1,0,21,76,1,0,21,77,1,0,21,78,1,0,21,79,1,0,21,80,1,0,21,81,1,0,21,82,1,0,21,83,1,0,21,84,1,0,21,85,1,0,21,86,1,0,21,87,1,0,21,88,1,0,21,89,1,0,21,90,1,0,21,91,1,0,21,92,1,0,21,93,1,0,21,94,1,0,21,95,1,0,21,96,1,0,21,97,1,0,21,98,1,0,-96,99,99,99,99,99,99,99,99,99,99}; + byte[] in = {0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,17,17,18,18,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,22,22,22,22,22,22,22,22,22,22,23,23,23,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,25,25,25,25,25,25,25,25,25,25,26,26,26,26,26,26,26,26,26,26,27,27,27,27,27,27,27,27,27,27,28,28,28,28,28,28,28,28,28,28,29,29,29,29,29,29,29,29,29,29,30,30,30,30,30,30,30,30,30,30,31,31,31,31,31,31,31,31,31,31,32,32,32,32,32,32,32,32,32,32,33,33,33,33,33,33,33,33,33,33,34,34,34,34,34,34,34,34,34,34,35,35,35,35,35,35,35,35,35,35,36,36,36,36,36,36,36,36,36,36,37,37,37,37,37,37,37,37,37,37,38,38,38,38,38,38,38,38,38,38,39,39,39,39,39,39,39,39,39,39,40,40,40,40,40,40,40,40,40,40,41,41,41,41,41,41,41,41,41,41,42,42,42,42,42,42,42,42,42,42,43,43,43,43,43,43,43,43,43,43,44,44,44,44,44,44,44,44,44,44,45,45,45,45,45,45,45,45,45,45,46,46,46,46,46,46,46,46,46,46,47,47,47,47,47,47,47,47,47,47,48,48,48,48,48,48,48,48,48,48,49,49,49,49,49,49,49,49,49,49,50,50,50,50,50,50,50,50,50,50,51,51,51,51,51,51,51,51,51,51,52,52,52,52,52,52,52,52,52,52,53,53,53,53,53,53,53,53,53,53,54,54,54,54,54,54,54,54,54,54,55,55,55,55,55,55,55,55,55,55,56,56,56,56,56,56,56,56,56,56,57,57,57,57,57,57,57,57,57,57,58,58,58,58,58,58,58,58,58,58,59,59,59,59,59,59,59,59,59,59,60,60,60,60,60,60,60,60,60,60,61,61,61,61,61,61,61,61,61,61,62,62,62,62,62,62,62,62,62,62,63,63,63,63,63,63,63,63,63,63,64,64,64,64,64,64,64,64,64,64,65,65,65,65,65,65,65,65,65,65,66,66,66,66,66,66,66,66,66,66,67,67,67,67,67,67,67,67,67,67,68,68,68,68,68,68,68,68,68,68,69,69,69,69,69,69,69,69,69,69,70,70,70,70,70,70,70,70,70,70,71,71,71,71,71,71,71,71,71,71,72,72,72,72,72,72,72,72,72,72,73,73,73,73,73,73,73,73,73,73,74,74,74,74,74,74,74,74,74,74,75,75,75,75,75,75,75,75,75,75,76,76,76,76,76,76,76,76,76,76,77,77,77,77,77,77,77,77,77,77,78,78,78,78,78,78,78,78,78,78,79,79,79,79,79,79,79,79,79,79,80,80,80,80,80,80,80,80,80,80,81,81,81,81,81,81,81,81,81,81,82,82,82,82,82,82,82,82,82,82,83,83,83,83,83,83,83,83,83,83,84,84,84,84,84,84,84,84,84,84,85,85,85,85,85,85,85,85,85,85,86,86,86,86,86,86,86,86,86,86,87,87,87,87,87,87,87,87,87,87,88,88,88,88,88,88,88,88,88,88,89,89,89,89,89,89,89,89,89,89,90,90,90,90,90,90,90,90,90,90,91,91,91,91,91,91,91,91,91,91,92,92,92,92,92,92,92,92,92,92,93,93,93,93,93,93,93,93,93,93,94,94,94,94,94,94,94,94,94,94,95,95,95,95,95,95,95,95,95,95,96,96,96,96,96,96,96,96,96,96,97,97,97,97,97,97,97,97,97,97,98,98,98,98,98,98,98,98,98,98,99,99,99,99,99,99,99,99,99,99}; + CodecOptions options = new CodecOptions(); + byte[] comp = codec.compress(in, options); + assertEquals(expected, comp); + } + + @Test + public void testDecompressLongSeq() throws Exception { + byte[] in = {21,0,1,0,21,1,1,0,21,2,1,0,21,3,1,0,21,4,1,0,21,5,1,0,21,6,1,0,21,7,1,0,21,8,1,0,21,9,1,0,21,10,1,0,21,11,1,0,21,12,1,0,21,13,1,0,21,14,1,0,21,15,1,0,21,16,1,0,21,17,1,0,21,18,1,0,21,19,1,0,21,20,1,0,21,21,1,0,21,22,1,0,21,23,1,0,21,24,1,0,21,25,1,0,21,26,1,0,21,27,1,0,21,28,1,0,21,29,1,0,21,30,1,0,21,31,1,0,21,32,1,0,21,33,1,0,21,34,1,0,21,35,1,0,21,36,1,0,21,37,1,0,21,38,1,0,21,39,1,0,21,40,1,0,21,41,1,0,21,42,1,0,21,43,1,0,21,44,1,0,21,45,1,0,21,46,1,0,21,47,1,0,21,48,1,0,21,49,1,0,21,50,1,0,21,51,1,0,21,52,1,0,21,53,1,0,21,54,1,0,21,55,1,0,21,56,1,0,21,57,1,0,21,58,1,0,21,59,1,0,21,60,1,0,21,61,1,0,21,62,1,0,21,63,1,0,21,64,1,0,21,65,1,0,21,66,1,0,21,67,1,0,21,68,1,0,21,69,1,0,21,70,1,0,21,71,1,0,21,72,1,0,21,73,1,0,21,74,1,0,21,75,1,0,21,76,1,0,21,77,1,0,21,78,1,0,21,79,1,0,21,80,1,0,21,81,1,0,21,82,1,0,21,83,1,0,21,84,1,0,21,85,1,0,21,86,1,0,21,87,1,0,21,88,1,0,21,89,1,0,21,90,1,0,21,91,1,0,21,92,1,0,21,93,1,0,21,94,1,0,21,95,1,0,21,96,1,0,21,97,1,0,21,98,1,0,-96,99,99,99,99,99,99,99,99,99,99}; + byte[] expected = {0,0,0,0,0,0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,9,9,9,9,9,9,9,9,9,9,10,10,10,10,10,10,10,10,10,10,11,11,11,11,11,11,11,11,11,11,12,12,12,12,12,12,12,12,12,12,13,13,13,13,13,13,13,13,13,13,14,14,14,14,14,14,14,14,14,14,15,15,15,15,15,15,15,15,15,15,16,16,16,16,16,16,16,16,16,16,17,17,17,17,17,17,17,17,17,17,18,18,18,18,18,18,18,18,18,18,19,19,19,19,19,19,19,19,19,19,20,20,20,20,20,20,20,20,20,20,21,21,21,21,21,21,21,21,21,21,22,22,22,22,22,22,22,22,22,22,23,23,23,23,23,23,23,23,23,23,24,24,24,24,24,24,24,24,24,24,25,25,25,25,25,25,25,25,25,25,26,26,26,26,26,26,26,26,26,26,27,27,27,27,27,27,27,27,27,27,28,28,28,28,28,28,28,28,28,28,29,29,29,29,29,29,29,29,29,29,30,30,30,30,30,30,30,30,30,30,31,31,31,31,31,31,31,31,31,31,32,32,32,32,32,32,32,32,32,32,33,33,33,33,33,33,33,33,33,33,34,34,34,34,34,34,34,34,34,34,35,35,35,35,35,35,35,35,35,35,36,36,36,36,36,36,36,36,36,36,37,37,37,37,37,37,37,37,37,37,38,38,38,38,38,38,38,38,38,38,39,39,39,39,39,39,39,39,39,39,40,40,40,40,40,40,40,40,40,40,41,41,41,41,41,41,41,41,41,41,42,42,42,42,42,42,42,42,42,42,43,43,43,43,43,43,43,43,43,43,44,44,44,44,44,44,44,44,44,44,45,45,45,45,45,45,45,45,45,45,46,46,46,46,46,46,46,46,46,46,47,47,47,47,47,47,47,47,47,47,48,48,48,48,48,48,48,48,48,48,49,49,49,49,49,49,49,49,49,49,50,50,50,50,50,50,50,50,50,50,51,51,51,51,51,51,51,51,51,51,52,52,52,52,52,52,52,52,52,52,53,53,53,53,53,53,53,53,53,53,54,54,54,54,54,54,54,54,54,54,55,55,55,55,55,55,55,55,55,55,56,56,56,56,56,56,56,56,56,56,57,57,57,57,57,57,57,57,57,57,58,58,58,58,58,58,58,58,58,58,59,59,59,59,59,59,59,59,59,59,60,60,60,60,60,60,60,60,60,60,61,61,61,61,61,61,61,61,61,61,62,62,62,62,62,62,62,62,62,62,63,63,63,63,63,63,63,63,63,63,64,64,64,64,64,64,64,64,64,64,65,65,65,65,65,65,65,65,65,65,66,66,66,66,66,66,66,66,66,66,67,67,67,67,67,67,67,67,67,67,68,68,68,68,68,68,68,68,68,68,69,69,69,69,69,69,69,69,69,69,70,70,70,70,70,70,70,70,70,70,71,71,71,71,71,71,71,71,71,71,72,72,72,72,72,72,72,72,72,72,73,73,73,73,73,73,73,73,73,73,74,74,74,74,74,74,74,74,74,74,75,75,75,75,75,75,75,75,75,75,76,76,76,76,76,76,76,76,76,76,77,77,77,77,77,77,77,77,77,77,78,78,78,78,78,78,78,78,78,78,79,79,79,79,79,79,79,79,79,79,80,80,80,80,80,80,80,80,80,80,81,81,81,81,81,81,81,81,81,81,82,82,82,82,82,82,82,82,82,82,83,83,83,83,83,83,83,83,83,83,84,84,84,84,84,84,84,84,84,84,85,85,85,85,85,85,85,85,85,85,86,86,86,86,86,86,86,86,86,86,87,87,87,87,87,87,87,87,87,87,88,88,88,88,88,88,88,88,88,88,89,89,89,89,89,89,89,89,89,89,90,90,90,90,90,90,90,90,90,90,91,91,91,91,91,91,91,91,91,91,92,92,92,92,92,92,92,92,92,92,93,93,93,93,93,93,93,93,93,93,94,94,94,94,94,94,94,94,94,94,95,95,95,95,95,95,95,95,95,95,96,96,96,96,96,96,96,96,96,96,97,97,97,97,97,97,97,97,97,97,98,98,98,98,98,98,98,98,98,98,99,99,99,99,99,99,99,99,99,99}; + CodecOptions options = new CodecOptions(); + options.maxBytes=expected.length; + byte[] comp = codec.decompress(in, options); + assertEquals(expected, comp); + } +} + From 375e6455309d013993c1eaca9f7b8a737752388e Mon Sep 17 00:00:00 2001 From: "r.harkes" Date: Thu, 12 Sep 2024 12:15:11 +0200 Subject: [PATCH 2/2] add the copyright statement --- src/test/java/ome/codecs/LZ4CodecTest.java | 32 ++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/test/java/ome/codecs/LZ4CodecTest.java b/src/test/java/ome/codecs/LZ4CodecTest.java index 8df942da..1d5e4a01 100644 --- a/src/test/java/ome/codecs/LZ4CodecTest.java +++ b/src/test/java/ome/codecs/LZ4CodecTest.java @@ -1,3 +1,35 @@ +/* + * #%L + * Image encoding and decoding routines. + * %% + * Copyright (C) 2005 - 2017 Open Microscopy Environment: + * - Board of Regents of the University of Wisconsin-Madison + * - Glencoe Software, Inc. + * - University of Dundee + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + package ome.codecs; import org.testng.annotations.Test;