Decompressing with Zstd set bufferSizePrecheck flag to false (#258)

* ZstdNet unwrap false for preCheck size #257 As part of the Unwrap method in ZstdNet it checks if the compressed payload contains the size of the decompressed payload. However it is not a guarrantee that it exists in the compressed payload, causing an exception * Add test case that validates that Zstd with false flag for preSize check works * Update tests/UnitTests/Internal/CompressionCodecTests.fs Co-authored-by: Edgaras Petovradzius <[email protected]> * Update wording in new added test --------- Co-authored-by: Edgaras Petovradzius <[email protected]>
fsprojects · Mar 28, 2024 · 4ec1ab6 · 4ec1ab6
1 parent 8df5b0e
commit 4ec1ab6
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 2 deletions.
diff --git a/src/Pulsar.Client/Internal/Compression.fs b/src/Pulsar.Client/Internal/Compression.fs
@@ -1,4 +1,4 @@
-namespace Pulsar.Client.Internal
+namespace Pulsar.Client.Internal
 
 open System
 open System.Buffers
@@ -127,7 +127,7 @@ module internal CompressionCodec =
                 let zstdDecompressor = new Decompressor()
                 try
                     let sourceSpan = payload.ToArray().AsSpan()
-                    zstdDecompressor.Unwrap(sourceSpan, target) |> ignore
+                    zstdDecompressor.Unwrap(sourceSpan, target, false) |> ignore
                     let ms = MemoryStreamManager.GetStream(null, uncompressedSize)
                     ms.Write(target, 0, uncompressedSize)
                     ms

diff --git a/tests/UnitTests/Internal/CompressionCodecTests.fs b/tests/UnitTests/Internal/CompressionCodecTests.fs
@@ -24,6 +24,7 @@ let tests =
     let helloLZ4 = System.Convert.FromBase64String "UEhlbGxv"
     let helloSnappy = System.Convert.FromBase64String "BRBIZWxsbw=="
     let helloZStd = System.Convert.FromBase64String "KLUv/SAFKQAASGVsbG8="
+    let helloWorldZstdWithoutDecompressedContentSizeInPayload = System.Convert.FromBase64String "KLUv/QBQ/AAAuGhlbGxvIHdvcmxkIGxvcmVtIGlwc3VtAQDN/zWfTUwAAAhpAQD8/zkQAkwAAAhvAQD8/zkQAkwAAAhyAQD8/zkQAkwAAAhsAQD8/zkQAkwAAAh1AQD8/zkQAkwAAAhtAQD8/zkQAkwAAAggAQD8/zkQAkwAAAh3AQD8/zkQAkwAAAhlAQD8/zkQAkwAAAhwAQD8/zkQAkwAAAhyAQD8/zkQAkwAAAhsAQD8/zkQAkwAAAhvAQD8/zkQAkwAAAhtAQD8/zkQAkwAAAggAQD8/zkQAkUAAAhsAQDkKyAE"
 
     let testEncode compressionType expectedBytes =
         let codec = compressionType |> createCodec
@@ -79,4 +80,21 @@ let tests =
         test "Codec should make ZStd decoding" {
             helloZStd |> testDecode CompressionType.ZStd
         }
+
+        test "Zstd should decode content with non specififed size in the compressed payload" {
+          // The compressed string above has been compressed with zstd using this library: https://www.npmjs.com/package/[email protected]
+          // It seems this library does not specify the decompressed size in the payload.
+          //
+          // The raw string has been built in JS with the snippet below and then compressed afterwards.
+          // const bytes = 1024 * 1024 * 2 + 1000
+          // let str = "";
+          // while (str.length < bytes)
+          //   str += "hello world lorem ipsum"
+
+          let uncompressedSize = (1024 * 1024 * 2 + 1000)
+          let codec = CompressionType.ZStd |> createCodec
+          let ms = new MemoryStream(helloWorldZstdWithoutDecompressedContentSizeInPayload, 0, helloWorldZstdWithoutDecompressedContentSizeInPayload.Length, true, true)
+          let decoded = codec.Decode(uncompressedSize, ms) |> (fun ms -> ms.ToArray())|>  getString
+          decoded |> Expect.stringStarts "" "hello world lorem ipsum"
+        }
     ]