Merge branch 'master' into nz-pledgeinsize

.gitignore

@@ -2,13 +2,13 @@
 *.jl.*.cov
 *.jl.mem
 /Manifest.toml
-/deps/build.log
-/deps/deps.jl
-/deps/usr/
 src/libzstd/LibTemplate.jl
 src/libzstd/ctypes.jl
 /github/
 settings.json
+**/*.json
+**/*.json.tmp
+benchmark/Manifest.toml
 dictionary
 github_users_sample_set.tar
 github_users_sample_set.tar.zst

README.md

@@ -1,7 +1,7 @@
 CodecZstd.jl
 ============
 
-[![codecov.io][codecov-img]][codecov-url]
+[![codecov](https://codecov.io/gh/JuliaIO/CodecZstd.jl/graph/badge.svg?token=CgSrhnEKdy)](https://codecov.io/gh/JuliaIO/CodecZstd.jl)
 
 ## Installation
 
@@ -46,11 +46,4 @@ This package exports following codecs and streams:
 
 Version 0.8.3 also introduced the virtual codec `ZstdFrameCompressor` which stores the decompressed content size in the frame header. Currently, `ZstdFrameCompressor` is an alternate constructor for `ZstdCompressor`, but that is an implementation detail which should not be relied upon.
 
-See docstrings and [TranscodingStreams.jl](https://github.com/bicycle1885/TranscodingStreams.jl) for details.
-
-[travisci-img]: https://travis-ci.org/bicycle1885/CodecZstd.jl.svg?branch=master
-[travisci-url]: https://travis-ci.org/bicycle1885/CodecZstd.jl
-[appveyor-img]: https://ci.appveyor.com/api/projects/status/u58v32yenqf19x2a?svg=true
-[appveyor-url]: https://ci.appveyor.com/project/bicycle1885/codeczstd-jl
-[codecov-img]: http://codecov.io/github/bicycle1885/CodecZstd.jl/coverage.svg?branch=master
-[codecov-url]: http://codecov.io/github/bicycle1885/CodecZstd.jl?branch=master
+See docstrings and [TranscodingStreams.jl](https://github.com/JuliaIO/TranscodingStreams.jl) for details.

benchmark/Project.toml

@@ -0,0 +1,6 @@
+[deps]
+BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+CodecZstd = "6b39b394-51ab-5f42-8807-6242bab2b4c2"
+PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"

benchmark/README.md

@@ -0,0 +1,11 @@
+# CodecZstd.jl benchmarks
+
+This directory contains benchmarks for CodecZstd. To run all the
+benchmarks, launch `julia --project=benchmark` and enter:
+
+``` julia
+using PkgBenchmark
+import CodecZstd
+
+benchmarkpkg(CodecZstd)
+```

benchmark/benchmarks.jl

@@ -0,0 +1,32 @@
+using BenchmarkTools
+using Random
+using TranscodingStreams
+using CodecZstd
+
+const SUITE = BenchmarkGroup()
+cbench = SUITE["compression"] = BenchmarkGroup()
+dbench = SUITE["decompression"] = BenchmarkGroup()
+
+ccodec = ZstdCompressor()
+dcodec = ZstdDecompressor()
+TranscodingStreams.initialize(ccodec)
+TranscodingStreams.initialize(dcodec)
+
+for N in [100, 100000, 1000000]
+    u1 = rand(Xoshiro(1234), UInt8, N)
+    c1 = transcode(ZstdCompressor, u1)
+    cbench["uncompressible"][N] = @benchmarkable transcode($ccodec, $u1)
+    dbench["uncompressible"][N] = @benchmarkable transcode($dcodec, $c1)
+
+    u2 = rand(Xoshiro(1234), 0x00:0x01, N)
+    c2 = transcode(ZstdCompressor, u2)
+    cbench["compressible-bytes"][N] = @benchmarkable transcode($ccodec, $u2)
+    dbench["compressible-bytes"][N] = @benchmarkable transcode($dcodec, $c2)
+
+    f = round.(randn(Xoshiro(1234), N); base=2, digits=7)
+    # byte shuffle
+    u3 = vec(permutedims(reinterpret(reshape, UInt8, f),(2,1)))
+    c3 = transcode(ZstdCompressor, u3)
+    cbench["byteshuffle"][N] = @benchmarkable transcode($ccodec, $u3)
+    dbench["byteshuffle"][N] = @benchmarkable transcode($dcodec, $c3)
+end

src/compression.jl

@@ -78,16 +78,6 @@ end
 # Methods
 # -------
 
-function TranscodingStreams.initialize(codec::ZstdCompressor)
-    code = initialize!(codec.cstream, codec.level)
-    if iserror(code)
-        zstderror(codec.cstream, code)
-    end
-    reset!(codec.cstream.ibuffer)
-    reset!(codec.cstream.obuffer)
-    return
-end
-
 function TranscodingStreams.finalize(codec::ZstdCompressor)
     if codec.cstream.ptr != C_NULL
         code = free!(codec.cstream)
@@ -96,12 +86,21 @@ function TranscodingStreams.finalize(codec::ZstdCompressor)
         end
         codec.cstream.ptr = C_NULL
     end
-    reset!(codec.cstream.ibuffer)
-    reset!(codec.cstream.obuffer)
     return
 end
 
-function TranscodingStreams.startproc(codec::ZstdCompressor, mode::Symbol, error::Error)::Symbol
+function TranscodingStreams.startproc(codec::ZstdCompressor, mode::Symbol, error::Error)
+    if codec.cstream.ptr == C_NULL
+        codec.cstream.ptr = LibZstd.ZSTD_createCStream()
+        if codec.cstream.ptr == C_NULL
+            throw(OutOfMemoryError())
+        end
+        i_code = initialize!(codec.cstream, codec.level)
+        if iserror(i_code)
+            error[] = ErrorException("zstd initialization error")
+            return :error
+        end
+    end
     code = reset!(codec.cstream)
     if iserror(code)
         error[] = ErrorException("zstd error resetting compression context")
@@ -129,6 +128,9 @@ if isdefined(TranscodingStreams, :pledgeinsize)
 end
 
 function TranscodingStreams.process(codec::ZstdCompressor, input::Memory, output::Memory, error::Error)
+    if codec.cstream.ptr == C_NULL
+        error("startproc must be called before process")
+    end
     cstream = codec.cstream
     ibuffer_starting_pos = UInt(0)
     if codec.endOp == LibZstd.ZSTD_e_end &&

src/decompression.jl

@@ -33,16 +33,6 @@ end
 # Methods
 # -------
 
-function TranscodingStreams.initialize(codec::ZstdDecompressor)
-    code = initialize!(codec.dstream)
-    if iserror(code)
-        zstderror(codec.dstream, code)
-    end
-    reset!(codec.dstream.ibuffer)
-    reset!(codec.dstream.obuffer)
-    return
-end
-
 function TranscodingStreams.finalize(codec::ZstdDecompressor)
     if codec.dstream.ptr != C_NULL
         code = free!(codec.dstream)
@@ -51,12 +41,21 @@ function TranscodingStreams.finalize(codec::ZstdDecompressor)
         end
         codec.dstream.ptr = C_NULL
     end
-    reset!(codec.dstream.ibuffer)
-    reset!(codec.dstream.obuffer)
     return
 end
 
 function TranscodingStreams.startproc(codec::ZstdDecompressor, mode::Symbol, error::Error)
+    if codec.dstream.ptr == C_NULL
+        codec.dstream.ptr = LibZstd.ZSTD_createDStream()
+        if codec.dstream.ptr == C_NULL
+            throw(OutOfMemoryError())
+        end
+        i_code = initialize!(codec.dstream)
+        if iserror(i_code)
+            error[] = ErrorException("zstd initialization error")
+            return :error
+        end
+    end
     code = reset!(codec.dstream)
     if iserror(code)
         error[] = ErrorException("zstd error")
@@ -66,6 +65,9 @@ function TranscodingStreams.startproc(codec::ZstdDecompressor, mode::Symbol, err
 end
 
 function TranscodingStreams.process(codec::ZstdDecompressor, input::Memory, output::Memory, error::Error)
+    if codec.dstream.ptr == C_NULL
+        error("startproc must be called before process")
+    end
     dstream = codec.dstream
     dstream.ibuffer.src = input.ptr
     dstream.ibuffer.size = input.size

src/libzstd.jl

@@ -44,11 +44,7 @@ mutable struct CStream
     obuffer::OutBuffer
 
     function CStream()
-        ptr = LibZstd.ZSTD_createCStream()
-        if ptr == C_NULL
-            throw(OutOfMemoryError())
-        end
-        return new(ptr, InBuffer(), OutBuffer())
+        return new(C_NULL, InBuffer(), OutBuffer())
     end
 end
 
@@ -115,11 +111,7 @@ mutable struct DStream
     obuffer::OutBuffer
 
     function DStream()
-        ptr = LibZstd.ZSTD_createDStream()
-        if ptr == C_NULL
-            throw(OutOfMemoryError())
-        end
-        return new(ptr, InBuffer(), OutBuffer())
+        return new(C_NULL, InBuffer(), OutBuffer())
     end
 end
 Base.unsafe_convert(::Type{Ptr{LibZstd.ZSTD_DStream}}, dstream::DStream) = dstream.ptr
@@ -133,6 +125,8 @@ end
 function reset!(dstream::DStream)
     # LibZstd.ZSTD_resetDStream is deprecated
     # https://github.com/facebook/zstd/blob/9d2a45a705e22ad4817b41442949cd0f78597154/lib/zstd.h#L2332-L2339
+    reset!(dstream.ibuffer)
+    reset!(dstream.obuffer)
     return LibZstd.ZSTD_DCtx_reset(dstream, LibZstd.ZSTD_reset_session_only)
 end
 

test/compress_endOp.jl

@@ -3,27 +3,29 @@ using Test
 
 @testset "compress! endOp = :continue" begin
     data = rand(1:100, 1024*1024)
-    cstream = CodecZstd.CStream()
-    cstream.ibuffer.src = pointer(data)
-    cstream.ibuffer.size = sizeof(data)
-    cstream.ibuffer.pos = 0
-    cstream.obuffer.dst = Base.Libc.malloc(sizeof(data)*2)
-    cstream.obuffer.size = sizeof(data)*2
-    cstream.obuffer.pos = 0
-    try
-        GC.@preserve data begin
+    GC.@preserve data begin
+        cstream = CodecZstd.CStream()
+        cstream.ptr = CodecZstd.LibZstd.ZSTD_createCStream()
+        cstream.ibuffer.src = pointer(data)
+        cstream.ibuffer.size = sizeof(data)
+        cstream.ibuffer.pos = 0
+        cstream.obuffer.dst = Base.Libc.malloc(sizeof(data)*2)
+        cstream.obuffer.size = sizeof(data)*2
+        cstream.obuffer.pos = 0
+        try
             # default endOp
             @test CodecZstd.compress!(cstream; endOp=:continue) == 0
             @test CodecZstd.find_decompressed_size(cstream.obuffer.dst, cstream.obuffer.pos) == CodecZstd.ZSTD_CONTENTSIZE_UNKNOWN
+        finally
+            Base.Libc.free(cstream.obuffer.dst)
         end
-    finally
-        Base.Libc.free(cstream.obuffer.dst)
     end
 end
 
 @testset "compress! endOp = :flush" begin
     data = rand(1:100, 1024*1024)
     cstream = CodecZstd.CStream()
+    cstream.ptr = CodecZstd.LibZstd.ZSTD_createCStream()
     cstream.ibuffer.src = pointer(data)
     cstream.ibuffer.size = sizeof(data)
     cstream.ibuffer.pos = 0
@@ -43,6 +45,7 @@ end
 @testset "compress! endOp = :end" begin
     data = rand(1:100, 1024*1024)
     cstream = CodecZstd.CStream()
+    cstream.ptr = CodecZstd.LibZstd.ZSTD_createCStream()
     cstream.ibuffer.src = pointer(data)
     cstream.ibuffer.size = sizeof(data)
     cstream.ibuffer.pos = 0