Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Decompressing with Zstd set bufferSizePrecheck flag to false #258

Merged
merged 4 commits into from
Mar 28, 2024
Merged

Decompressing with Zstd set bufferSizePrecheck flag to false #258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
22d3980
ZstdNet unwrap false for preCheck size #257
Nikolajls Mar 28, 2024
d0f2b27
Add test case that validates that Zstd with false flag for preSize ch…
Nikolajls Mar 28, 2024
21d229a
Update tests/UnitTests/Internal/CompressionCodecTests.fs
Nikolajls Mar 28, 2024
19d51f6
Update wording in new added test
Nikolajls Mar 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/Pulsar.Client/Internal/Compression.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
namespace Pulsar.Client.Internal
namespace Pulsar.Client.Internal

open System
open System.Buffers
Expand Down Expand Up @@ -127,7 +127,7 @@ module internal CompressionCodec =
let zstdDecompressor = new Decompressor()
try
let sourceSpan = payload.ToArray().AsSpan()
zstdDecompressor.Unwrap(sourceSpan, target) |> ignore
zstdDecompressor.Unwrap(sourceSpan, target, false) |> ignore
let ms = MemoryStreamManager.GetStream(null, uncompressedSize)
ms.Write(target, 0, uncompressedSize)
ms
Expand Down
18 changes: 18 additions & 0 deletions tests/UnitTests/Internal/CompressionCodecTests.fs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ let tests =
let helloLZ4 = System.Convert.FromBase64String "UEhlbGxv"
let helloSnappy = System.Convert.FromBase64String "BRBIZWxsbw=="
let helloZStd = System.Convert.FromBase64String "KLUv/SAFKQAASGVsbG8="
let helloWorldZstdWithoutDecompressedContentSizeInPayload = System.Convert.FromBase64String "KLUv/QBQ/AAAuGhlbGxvIHdvcmxkIGxvcmVtIGlwc3VtAQDN/zWfTUwAAAhpAQD8/zkQAkwAAAhvAQD8/zkQAkwAAAhyAQD8/zkQAkwAAAhsAQD8/zkQAkwAAAh1AQD8/zkQAkwAAAhtAQD8/zkQAkwAAAggAQD8/zkQAkwAAAh3AQD8/zkQAkwAAAhlAQD8/zkQAkwAAAhwAQD8/zkQAkwAAAhyAQD8/zkQAkwAAAhsAQD8/zkQAkwAAAhvAQD8/zkQAkwAAAhtAQD8/zkQAkwAAAggAQD8/zkQAkUAAAhsAQDkKyAE"

let testEncode compressionType expectedBytes =
let codec = compressionType |> createCodec
Expand Down Expand Up @@ -79,4 +80,21 @@ let tests =
test "Codec should make ZStd decoding" {
helloZStd |> testDecode CompressionType.ZStd
}

test "Zstd should decode content with non specififed size in the compressed payload" {
// The compressed string above has been compressed with zstd using this library: https://www.npmjs.com/package/[email protected]
// It seems this library does not specify the decompressed size in the payload.
//
// The raw string has been built in JS with the snippet below and then compressed afterwards.
// const bytes = 1024 * 1024 * 2 + 1000
// let str = "";
// while (str.length < bytes)
// str += "hello world lorem ipsum"

let uncompressedSize = (1024 * 1024 * 2 + 1000)
let codec = CompressionType.ZStd |> createCodec
let ms = new MemoryStream(helloWorldZstdWithoutDecompressedContentSizeInPayload, 0, helloWorldZstdWithoutDecompressedContentSizeInPayload.Length, true, true)
let decoded = codec.Decode(uncompressedSize, ms) |> (fun ms -> ms.ToArray())|> getString
decoded |> Expect.stringStarts "" "hello world lorem ipsum"
}
]
Loading