Skip to content

Commit

Permalink
slicer: Support payload buffer in write flow and optimize performance
Browse files Browse the repository at this point in the history
WIP

 * `slicer.Put` and `slicer.Slicer.Put` now uses buffer passed into
   `SetPayloadBuffer`
 * if the buffer has enough size, all slicing ops do not allocate
   additional payload buffer

Benchmark results:
```
goos: linux
goarch: amd64
pkg: github.com/nspcc-dev/neofs-sdk-go/object/slicer
cpu: Intel(R) Core(TM) i5-10210U CPU @ 1.60GHz
                                                                     │    old.txt     │               new.txt                │
                                                                     │     sec/op     │    sec/op     vs base                │
SliceDataIntoObjects/slice_1-128/reader-8                                42.13µ ±  5%   46.21µ ± 13%   +9.68% (p=0.009 n=10)
SliceDataIntoObjects/slice_1-128/writer-8                                46.29µ ±  1%   47.64µ ±  5%   +2.92% (p=0.000 n=10)
SliceDataIntoObjects/slice_4-128/reader-8                                41.37µ ±  0%   41.66µ ±  2%        ~ (p=0.280 n=10)
SliceDataIntoObjects/slice_4-128/writer-8                                43.44µ ±  1%   45.94µ ±  9%   +5.76% (p=0.003 n=10)
SliceDataIntoObjects/slice_16-128/reader-8                               42.59µ ±  1%   42.62µ ±  8%        ~ (p=0.971 n=10)
SliceDataIntoObjects/slice_16-128/writer-8                               44.73µ ±  1%   43.75µ ±  1%   -2.21% (p=0.000 n=10)
SliceDataIntoObjects/slice_64-128/reader-8                               45.02µ ±  0%   44.45µ ± 26%        ~ (p=0.063 n=10)
SliceDataIntoObjects/slice_64-128/writer-8                               48.03µ ±  1%   49.77µ ± 30%        ~ (p=0.105 n=10)
SliceDataIntoObjects/slice_256-128/reader-8                              43.03µ ±  1%   43.73µ ±  5%        ~ (p=0.063 n=10)
SliceDataIntoObjects/slice_256-128/writer-8                              153.6µ ±  0%   155.4µ ±  2%   +1.17% (p=0.035 n=10)
SliceDataIntoObjects/slice_1024-128/reader-8                             42.61µ ±  5%   41.32µ ±  2%   -3.03% (p=0.023 n=10)
SliceDataIntoObjects/slice_1024-128/writer-8                             335.4µ ±  1%   340.5µ ±  9%   +1.50% (p=0.043 n=10)
SliceDataIntoObjects/slice_4096-128/reader-8                             44.25µ ±  1%   44.06µ ±  2%        ~ (p=0.436 n=10)
SliceDataIntoObjects/slice_4096-128/writer-8                             1.192m ±  1%   1.211m ±  1%   +1.58% (p=0.001 n=10)
SliceDataIntoObjects/slice_16384-128/reader-8                            43.07µ ±  1%   43.05µ ±  2%        ~ (p=0.971 n=10)
SliceDataIntoObjects/slice_16384-128/writer-8                            4.020m ±  0%   3.995m ±  0%   -0.61% (p=0.000 n=10)
SliceDataIntoObjects/slice_65536-128/reader-8                            42.71µ ±  1%   42.29µ ±  2%        ~ (p=0.063 n=10)
SliceDataIntoObjects/slice_65536-128/writer-8                            18.35m ±  4%   18.02m ±  3%        ~ (p=0.315 n=10)
SliceDataIntoObjects/slice_262144-128/reader-8                           44.91µ ± 22%   44.75µ ±  1%        ~ (p=0.853 n=10)
SliceDataIntoObjects/slice_262144-128/writer-8                           70.78m ±  1%   72.79m ±  3%   +2.83% (p=0.000 n=10)
SliceDataIntoObjects/slice_1048576-128/reader-8                          62.26µ ± 15%   60.66µ ±  2%        ~ (p=0.393 n=10)
SliceDataIntoObjects/slice_1048576-128/writer-8                          288.0m ±  1%   297.4m ± 81%   +3.25% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=1/with_payload_buffer-8               39.49µ ±  1%   39.50µ ±  1%        ~ (p=0.739 n=10)
WritePayloadBuffer/limit=1024,size=1/without_payload_buffer-8            39.60µ ±  1%   40.02µ ±  3%   +1.07% (p=0.019 n=10)
WritePayloadBuffer/limit=1024,size=1024/with_payload_buffer-8            57.08µ ± 10%   52.36µ ±  0%   -8.26% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=1024/without_payload_buffer-8         55.64µ ±  3%   52.36µ ±  1%   -5.90% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=10240/with_payload_buffer-8           703.9µ ±  5%   677.3µ ±  0%   -3.78% (p=0.004 n=10)
WritePayloadBuffer/limit=1024,size=10240/without_payload_buffer-8        708.5µ ± 11%   677.9µ ±  0%   -4.32% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=204800/with_payload_buffer-8          12.16m ±  3%   12.20m ±  0%        ~ (p=0.105 n=10)
WritePayloadBuffer/limit=1024,size=204800/without_payload_buffer-8       12.17m ±  1%   12.79m ± 10%   +5.09% (p=0.000 n=10)
WritePayloadBuffer/limit=67108864,size=1024/with_payload_buffer-8      3983.39µ ±  2%   51.34µ ±  4%  -98.71% (p=0.000 n=10)
WritePayloadBuffer/limit=67108864,size=1024/without_payload_buffer-8     4.100m ±  2%   5.679m ±  6%  +38.52% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=1/with_payload_buffer-8                37.28µ ±  1%   36.75µ ±  1%   -1.42% (p=0.001 n=10)
ReadPayloadBuffer/limit=1024,size=1/without_payload_buffer-8             37.58µ ±  0%   37.52µ ±  2%        ~ (p=0.796 n=10)
ReadPayloadBuffer/limit=1024,size=1024/with_payload_buffer-8             52.06µ ±  0%   57.71µ ±  8%  +10.85% (p=0.001 n=10)
ReadPayloadBuffer/limit=1024,size=1024/without_payload_buffer-8          52.42µ ±  1%   52.38µ ±  7%        ~ (p=0.592 n=10)
ReadPayloadBuffer/limit=1024,size=10240/with_payload_buffer-8            679.2µ ±  0%   684.6µ ±  1%        ~ (p=0.218 n=10)
ReadPayloadBuffer/limit=1024,size=10240/without_payload_buffer-8         679.6µ ±  0%   690.0µ ±  1%   +1.54% (p=0.003 n=10)
ReadPayloadBuffer/limit=1024,size=204800/with_payload_buffer-8           7.025m ±  1%   7.142m ±  2%   +1.66% (p=0.005 n=10)
ReadPayloadBuffer/limit=1024,size=204800/without_payload_buffer-8        7.026m ±  1%   7.241m ±  5%   +3.07% (p=0.000 n=10)
ReadPayloadBuffer/limit=67108864,size=1024/with_payload_buffer-8       5169.13µ ± 29%   37.92µ ±  2%  -99.27% (p=0.000 n=10)
ReadPayloadBuffer/limit=67108864,size=1024/without_payload_buffer-8      8.630m ±  2%   5.765m ± 78%  -33.20% (p=0.023 n=10)
geomean                                                                  336.1µ         270.3µ        -19.57%

                                                                     │     old.txt      │               new.txt                │
                                                                     │       B/op       │     B/op      vs base                │
SliceDataIntoObjects/slice_1-128/reader-8                                  8.817Ki ± 0%   8.653Ki ± 0%   -1.86% (p=0.000 n=10)
SliceDataIntoObjects/slice_1-128/writer-8                                  8.692Ki ± 0%   8.653Ki ± 0%   -0.45% (p=0.000 n=10)
SliceDataIntoObjects/slice_4-128/reader-8                                  7.066Ki ± 0%   6.902Ki ± 0%   -2.32% (p=0.000 n=10)
SliceDataIntoObjects/slice_4-128/writer-8                                  6.941Ki ± 0%   6.902Ki ± 0%   -0.56% (p=0.000 n=10)
SliceDataIntoObjects/slice_16-128/reader-8                                 7.535Ki ± 0%   7.371Ki ± 0%   -2.18% (p=0.000 n=10)
SliceDataIntoObjects/slice_16-128/writer-8                                 7.410Ki ± 0%   7.371Ki ± 0%   -0.53% (p=0.000 n=10)
SliceDataIntoObjects/slice_64-128/reader-8                                 9.161Ki ± 0%   8.997Ki ± 0%   -1.79% (p=0.000 n=10)
SliceDataIntoObjects/slice_64-128/writer-8                                 9.036Ki ± 0%   8.997Ki ± 0%   -0.43% (p=0.000 n=10)
SliceDataIntoObjects/slice_256-128/reader-8                                7.942Ki ± 0%   7.778Ki ± 0%   -2.07% (p=0.000 n=10)
SliceDataIntoObjects/slice_256-128/writer-8                                23.71Ki ± 0%   23.65Ki ± 0%   -0.23% (p=0.000 n=10)
SliceDataIntoObjects/slice_1024-128/reader-8                               7.067Ki ± 0%   6.903Ki ± 0%   -2.32% (p=0.000 n=10)
SliceDataIntoObjects/slice_1024-128/writer-8                               45.36Ki ± 0%   45.31Ki ± 0%   -0.12% (p=0.000 n=10)
SliceDataIntoObjects/slice_4096-128/reader-8                               8.823Ki ± 0%   8.659Ki ± 0%   -1.86% (p=0.000 n=10)
SliceDataIntoObjects/slice_4096-128/writer-8                               156.8Ki ± 0%   156.8Ki ± 0%   -0.03% (p=0.000 n=10)
SliceDataIntoObjects/slice_16384-128/reader-8                              7.961Ki ± 0%   7.797Ki ± 0%   -2.06% (p=0.000 n=10)
SliceDataIntoObjects/slice_16384-128/writer-8                              543.5Ki ± 0%   543.5Ki ± 0%   -0.01% (p=0.000 n=10)
SliceDataIntoObjects/slice_65536-128/reader-8                              7.497Ki ± 0%   7.333Ki ± 0%   -2.19% (p=0.000 n=10)
SliceDataIntoObjects/slice_65536-128/writer-8                              2.185Mi ± 0%   2.185Mi ± 0%   -0.00% (p=0.004 n=10)
SliceDataIntoObjects/slice_262144-128/reader-8                             7.765Ki ± 1%   7.589Ki ± 0%   -2.26% (p=0.000 n=10)
SliceDataIntoObjects/slice_262144-128/writer-8                             8.810Mi ± 0%   8.810Mi ± 0%        ~ (p=0.280 n=10)
SliceDataIntoObjects/slice_1048576-128/reader-8                            11.40Ki ± 3%   10.91Ki ± 2%   -4.35% (p=0.001 n=10)
SliceDataIntoObjects/slice_1048576-128/writer-8                            35.20Mi ± 0%   35.20Mi ± 0%        ~ (p=0.853 n=10)
WritePayloadBuffer/limit=1024,size=1/with_payload_buffer-8                 8.907Ki ± 0%   7.852Ki ± 0%  -11.85% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=1/without_payload_buffer-8              8.907Ki ± 0%   8.978Ki ± 0%   +0.79% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=1024/with_payload_buffer-8              7.422Ki ± 0%   6.366Ki ± 0%  -14.22% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=1024/without_payload_buffer-8           7.422Ki ± 0%   7.492Ki ± 0%   +0.95% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=10240/with_payload_buffer-8             60.79Ki ± 0%   59.72Ki ± 0%   -1.76% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=10240/without_payload_buffer-8          60.79Ki ± 0%   60.84Ki ± 0%   +0.09% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=204800/with_payload_buffer-8            889.5Ki ± 0%   888.4Ki ± 0%   -0.12% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=204800/without_payload_buffer-8         889.5Ki ± 0%   889.5Ki ± 0%   +0.00% (p=0.023 n=10)
WritePayloadBuffer/limit=67108864,size=1024/with_payload_buffer-8      65543.057Ki ± 0%   6.363Ki ± 0%  -99.99% (p=0.000 n=10)
WritePayloadBuffer/limit=67108864,size=1024/without_payload_buffer-8       64.01Mi ± 0%   64.02Mi ± 0%   +0.01% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=1/with_payload_buffer-8                  7.469Ki ± 0%   6.413Ki ± 0%  -14.13% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=1/without_payload_buffer-8               8.470Ki ± 0%   7.539Ki ± 0%  -10.99% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=1024/with_payload_buffer-8               7.469Ki ± 0%   6.413Ki ± 0%  -14.13% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=1024/without_payload_buffer-8            8.469Ki ± 0%   7.539Ki ± 0%  -10.98% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=10240/with_payload_buffer-8              60.84Ki ± 0%   59.76Ki ± 0%   -1.76% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=10240/without_payload_buffer-8           61.83Ki ± 0%   60.89Ki ± 0%   -1.53% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=204800/with_payload_buffer-8             845.4Ki ± 0%   844.3Ki ± 0%   -0.13% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=204800/without_payload_buffer-8          846.4Ki ± 0%   845.4Ki ± 0%   -0.11% (p=0.000 n=10)
ReadPayloadBuffer/limit=67108864,size=1024/with_payload_buffer-8       65542.827Ki ± 0%   6.129Ki ± 0%  -99.99% (p=0.000 n=10)
ReadPayloadBuffer/limit=67108864,size=1024/without_payload_buffer-8       128.01Mi ± 0%   64.01Mi ± 0%  -49.99% (p=0.000 n=10)
geomean                                                                    76.81Ki        47.32Ki       -38.39%

                                                                     │   old.txt   │              new.txt               │
                                                                     │  allocs/op  │  allocs/op   vs base               │
SliceDataIntoObjects/slice_1-128/reader-8                               134.0 ± 0%    131.0 ± 0%  -2.24% (p=0.000 n=10)
SliceDataIntoObjects/slice_1-128/writer-8                               133.0 ± 0%    131.0 ± 0%  -1.50% (p=0.000 n=10)
SliceDataIntoObjects/slice_4-128/reader-8                               110.0 ± 0%    107.0 ± 0%  -2.73% (p=0.000 n=10)
SliceDataIntoObjects/slice_4-128/writer-8                               109.0 ± 0%    107.0 ± 0%  -1.83% (p=0.000 n=10)
SliceDataIntoObjects/slice_16-128/reader-8                              117.0 ± 0%    114.0 ± 0%  -2.56% (p=0.000 n=10)
SliceDataIntoObjects/slice_16-128/writer-8                              116.0 ± 0%    114.0 ± 0%  -1.72% (p=0.000 n=10)
SliceDataIntoObjects/slice_64-128/reader-8                              140.0 ± 0%    137.0 ± 0%  -2.14% (p=0.000 n=10)
SliceDataIntoObjects/slice_64-128/writer-8                              139.0 ± 0%    137.0 ± 0%  -1.44% (p=0.000 n=10)
SliceDataIntoObjects/slice_256-128/reader-8                             123.0 ± 0%    120.0 ± 0%  -2.44% (p=0.000 n=10)
SliceDataIntoObjects/slice_256-128/writer-8                             348.0 ± 0%    346.0 ± 0%  -0.57% (p=0.000 n=10)
SliceDataIntoObjects/slice_1024-128/reader-8                            110.0 ± 0%    107.0 ± 0%  -2.73% (p=0.000 n=10)
SliceDataIntoObjects/slice_1024-128/writer-8                            642.0 ± 0%    640.0 ± 0%  -0.31% (p=0.000 n=10)
SliceDataIntoObjects/slice_4096-128/reader-8                            134.0 ± 0%    131.0 ± 0%  -2.24% (p=0.000 n=10)
SliceDataIntoObjects/slice_4096-128/writer-8                           2.207k ± 0%   2.205k ± 0%  -0.09% (p=0.000 n=10)
SliceDataIntoObjects/slice_16384-128/reader-8                           123.0 ± 0%    120.0 ± 0%  -2.44% (p=0.000 n=10)
SliceDataIntoObjects/slice_16384-128/writer-8                          7.601k ± 0%   7.599k ± 0%  -0.03% (p=0.000 n=10)
SliceDataIntoObjects/slice_65536-128/reader-8                           116.0 ± 0%    113.0 ± 0%  -2.59% (p=0.000 n=10)
SliceDataIntoObjects/slice_65536-128/writer-8                          31.18k ± 0%   31.18k ± 0%  -0.01% (p=0.007 n=10)
SliceDataIntoObjects/slice_262144-128/reader-8                          120.0 ± 1%    117.0 ± 0%  -2.50% (p=0.000 n=10)
SliceDataIntoObjects/slice_262144-128/writer-8                         124.1k ± 0%   124.1k ± 0%       ~ (p=0.195 n=10)
SliceDataIntoObjects/slice_1048576-128/reader-8                         171.0 ± 3%    163.0 ± 2%  -4.68% (p=0.000 n=10)
SliceDataIntoObjects/slice_1048576-128/writer-8                        496.1k ± 0%   496.1k ± 0%       ~ (p=0.956 n=10)
WritePayloadBuffer/limit=1024,size=1/with_payload_buffer-8              117.0 ± 0%    114.0 ± 0%  -2.56% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=1/without_payload_buffer-8           117.0 ± 0%    115.0 ± 0%  -1.71% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=1024/with_payload_buffer-8           85.00 ± 0%    82.00 ± 0%  -3.53% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=1024/without_payload_buffer-8        85.00 ± 0%    83.00 ± 0%  -2.35% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=10240/with_payload_buffer-8          822.0 ± 0%    819.0 ± 0%  -0.36% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=10240/without_payload_buffer-8       822.0 ± 0%    820.0 ± 0%  -0.24% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=204800/with_payload_buffer-8        12.32k ± 0%   12.32k ± 0%  -0.02% (p=0.000 n=10)
WritePayloadBuffer/limit=1024,size=204800/without_payload_buffer-8     12.32k ± 0%   12.32k ± 0%  -0.02% (p=0.000 n=10)
WritePayloadBuffer/limit=67108864,size=1024/with_payload_buffer-8       88.00 ± 0%    82.00 ± 0%  -6.82% (p=0.000 n=10)
WritePayloadBuffer/limit=67108864,size=1024/without_payload_buffer-8    90.50 ± 1%    89.00 ± 1%  -1.66% (p=0.001 n=10)
ReadPayloadBuffer/limit=1024,size=1/with_payload_buffer-8               86.00 ± 0%    83.00 ± 0%  -3.49% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=1/without_payload_buffer-8            87.00 ± 0%    84.00 ± 0%  -3.45% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=1024/with_payload_buffer-8            86.00 ± 0%    83.00 ± 0%  -3.49% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=1024/without_payload_buffer-8         87.00 ± 0%    84.00 ± 0%  -3.45% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=10240/with_payload_buffer-8           823.0 ± 0%    820.0 ± 0%  -0.36% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=10240/without_payload_buffer-8        824.0 ± 0%    821.0 ± 0%  -0.36% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=204800/with_payload_buffer-8         11.72k ± 0%   11.71k ± 0%  -0.03% (p=0.000 n=10)
ReadPayloadBuffer/limit=1024,size=204800/without_payload_buffer-8      11.72k ± 0%   11.72k ± 0%  -0.03% (p=0.000 n=10)
ReadPayloadBuffer/limit=67108864,size=1024/with_payload_buffer-8        84.00 ± 0%    78.00 ± 0%  -7.14% (p=0.000 n=10)
ReadPayloadBuffer/limit=67108864,size=1024/without_payload_buffer-8     87.50 ± 1%    85.00 ± 1%  -2.86% (p=0.000 n=10)
```

Signed-off-by: Leonard Lyubich <[email protected]>
  • Loading branch information
cthulhu-rider committed Dec 19, 2023
1 parent b987c17 commit 122f2bd
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 22 deletions.
3 changes: 2 additions & 1 deletion object/slicer/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@ func (x *Options) SetCopiesNumber(copiesNumber uint32) {
}

// SetPayloadBuffer sets pre-allocated payloadBuffer to be used to object uploading.
// For better performance payloadBuffer length should be MaxObjectSize from NeoFS.
// The payloadBuffer should have length at least MaxObjectSize+1 from NeoFS,
// otherwise, it does not affect anything.
func (x *Options) SetPayloadBuffer(payloadBuffer []byte) {
x.payloadBuffer = payloadBuffer
}
Expand Down
45 changes: 24 additions & 21 deletions object/slicer/slicer.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package slicer

import (
"bytes"
"context"
"crypto/sha256"
"errors"
Expand Down Expand Up @@ -160,24 +159,17 @@ func childPayloadSizeLimit(opts Options) uint64 {

func slice(ctx context.Context, ow ObjectWriter, header object.Object, data io.Reader, signer user.Signer, opts Options) (oid.ID, error) {
var rootID oid.ID

objectPayloadLimit := childPayloadSizeLimit(opts)

var n int
bChunk := opts.payloadBuffer
if bChunk == nil {
bChunk = make([]byte, objectPayloadLimit)
}

writer, err := initPayloadStream(ctx, ow, header, signer, opts)
if err != nil {
return rootID, fmt.Errorf("init writter: %w", err)
}

for {
n, err = data.Read(bChunk)
n, err = data.Read(writer.buf[writer.currentWriter.written:])
if n > 0 {
if _, err = writer.Write(bChunk[:n]); err != nil {
if _, err = writer.Write(writer.buf[writer.currentWriter.written:][:n]); err != nil {
return oid.ID{}, err
}
}
Expand Down Expand Up @@ -269,9 +261,16 @@ func initPayloadStream(ctx context.Context, ow ObjectWriter, header object.Objec

maxObjSize := childPayloadSizeLimit(opts)

res.buf.Grow(int(maxObjSize))
if uint64(len(opts.payloadBuffer)) > maxObjSize {
res.buf = opts.payloadBuffer[:maxObjSize+1]
} else {
// TODO: support custom allocations
res.buf = make([]byte, maxObjSize+1)
}

res.currentWriter = newLimitedWriter(&res.rootMeta, maxObjSize, res.buf)

res.rootMeta.reset()
res.currentWriter = newLimitedWriter(io.MultiWriter(&res.buf, &res.rootMeta), maxObjSize)

return res, nil
}
Expand All @@ -290,7 +289,7 @@ type PayloadWriter struct {
currentEpoch uint64
sessionToken *session.Object

buf bytes.Buffer
buf []byte

rootMeta dynamicObjectMetadata
childMeta dynamicObjectMetadata
Expand Down Expand Up @@ -330,7 +329,7 @@ func (x *PayloadWriter) Write(chunk []byte) (int, error) {
return n, fmt.Errorf("write 1st child: %w", err)
}

x.currentWriter.reset(io.MultiWriter(&x.buf, &x.rootMeta, &x.childMeta))
x.currentWriter.reset(io.MultiWriter(&x.rootMeta, &x.childMeta))
} else {
err = x.writeIntermediateChild(x.ctx, x.childMeta)
if err != nil {
Expand All @@ -340,7 +339,6 @@ func (x *PayloadWriter) Write(chunk []byte) (int, error) {
x.currentWriter.resetProgress()
}

x.buf.Reset()
x.childMeta.reset()

n2, err := x.Write(chunk[n:]) // here n > 0 so infinite recursion shouldn't occur
Expand Down Expand Up @@ -418,7 +416,7 @@ func (x *PayloadWriter) _writeChild(ctx context.Context, meta dynamicObjectMetad
var id oid.ID
var err error

id, err = writeInMemObject(ctx, x.signer, x.stream, obj, x.buf.Bytes(), meta, x.prmObjectPutInit)
id, err = writeInMemObject(ctx, x.signer, x.stream, obj, x.buf[:x.currentWriter.written], meta, x.prmObjectPutInit)

if err != nil {
return fmt.Errorf("write formed object: %w", err)
Expand Down Expand Up @@ -573,14 +571,17 @@ type limitedWriter struct {
base io.Writer

limit, written uint64

buf []byte
}

// newLimitedWriter initializes limiterWriter which writes data to the base
// writer before the specified limit.
func newLimitedWriter(base io.Writer, limit uint64) limitedWriter {
// writer and provided buffer before the specified limit.
func newLimitedWriter(base io.Writer, limit uint64, buf []byte) limitedWriter {
return limitedWriter{
base: base,
limit: limit,
buf: buf,
}
}

Expand All @@ -602,11 +603,13 @@ func (x *limitedWriter) Write(p []byte) (n int, err error) {
overflow := uint64(len(p)) > x.limit-x.written

if overflow {
n, err = x.base.Write(p[:x.limit-x.written])
} else {
n, err = x.base.Write(p)
p = p[:x.limit-x.written]
}

n, err = x.base.Write(p)

copy(x.buf[x.written:], p[:n])

x.written += uint64(n)

if overflow && err == nil {
Expand Down

0 comments on commit 122f2bd

Please sign in to comment.