Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

upd/HRW library #2629

Merged
merged 2 commits into from
Nov 15, 2023
Merged

upd/HRW library #2629

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b67a268
go.mod: Update HRW library
carpawell Oct 25, 2023
a1cbaac
engine: Do not use zero weights for sorting shards
carpawell Oct 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ Changelog for NeoFS Node
- Update `tzhash` to `v1.7.1`
- `github.com/panjf2000/ants/v2` to `v2.8.2`
- `github.com/nspcc-dev/neofs-contract` to `v0.18.0` (#2580)
- `hrw` library to its generic version (#2629)

### Updating from v0.38.1
Blobovniczas are gone from the node with this release, see 0.38.0 upgrade
Expand Down
6 changes: 3 additions & 3 deletions cmd/blobovnicza-to-peapod/blobovniczatree/blobovnicza.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"sync"

"github.com/hashicorp/golang-lru/v2/simplelru"
"github.com/nspcc-dev/hrw"
"github.com/nspcc-dev/hrw/v2"
"github.com/nspcc-dev/neofs-node/cmd/blobovnicza-to-peapod/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression"
Expand Down Expand Up @@ -208,14 +208,14 @@ func (b *Blobovniczas) updateAndGet(lvlPath string, old *uint64) (blobovniczaWit
}

// returns hash of the object address.
func addressHash(addr *oid.Address, path string) uint64 {
func addressHash(addr *oid.Address, path string) hrw.Hashable {
var a string

if addr != nil {
a = addr.EncodeToString()
}

return hrw.Hash([]byte(a + path))
return hrw.WrapBytes([]byte(a + path))
}

// converts uint64 to hex string.
Expand Down
36 changes: 27 additions & 9 deletions cmd/blobovnicza-to-peapod/blobovniczatree/iterate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package blobovniczatree

import (
"encoding/binary"
"fmt"
"path/filepath"

"github.com/nspcc-dev/hrw"
"github.com/nspcc-dev/hrw/v2"
"github.com/nspcc-dev/neofs-node/cmd/blobovnicza-to-peapod/blobovnicza"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
Expand Down Expand Up @@ -94,17 +95,17 @@ func (b *Blobovniczas) iterateDeepest(addr oid.Address, f func(string) (bool, er

// iterator over particular level of directories.
func (b *Blobovniczas) iterateSorted(addr *oid.Address, curPath []string, execDepth uint64, f func([]string) (bool, error)) (bool, error) {
indices := indexSlice(b.blzShallowWidth)
indices := indexHashableSlice(b.blzShallowWidth)

hrw.SortSliceByValue(indices, addressHash(addr, filepath.Join(curPath...)))
hrw.Sort(indices, addressHash(addr, filepath.Join(curPath...)))

exec := uint64(len(curPath)) == execDepth

for i := range indices {
if i == 0 {
curPath = append(curPath, u64ToHexString(indices[i]))
curPath = append(curPath, u64ToHexString(indices[i].index()))
} else {
curPath[len(curPath)-1] = u64ToHexString(indices[i])
curPath[len(curPath)-1] = u64ToHexString(indices[i].index())
}

if exec {
Expand All @@ -128,12 +129,29 @@ func (b *Blobovniczas) iterateLeaves(f func(string) (bool, error)) error {
return b.iterateSortedLeaves(nil, f)
}

// makes slice of uint64 values from 0 to number-1.
func indexSlice(number uint64) []uint64 {
s := make([]uint64, number)
type hashableIndex struct {
ind uint64
hash uint64
}

func (hi hashableIndex) Hash() uint64 {
return hi.hash
}

func (hi hashableIndex) index() uint64 {
return hi.ind
}

// makes slice of index hashes from 0 to number-1.
func indexHashableSlice(number uint64) []hashableIndex {
s := make([]hashableIndex, number)

for i := range s {
s[i] = uint64(i)
indexU := uint64(i)
bytes := make([]byte, 8)
binary.BigEndian.PutUint64(bytes, indexU)

s[i] = hashableIndex{ind: indexU, hash: hrw.WrapBytes(bytes).Hash()}
}

return s
Expand Down
8 changes: 4 additions & 4 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,12 @@ require (
github.com/mr-tron/base58 v1.2.0
github.com/multiformats/go-multiaddr v0.12.0
github.com/nats-io/nats.go v1.31.0
github.com/nspcc-dev/hrw v1.0.9
github.com/nspcc-dev/hrw/v2 v2.0.0-20231115095647-bf62f4ad0a43
github.com/nspcc-dev/locode-db v0.4.1-0.20231110164540-a21ea2774f2e
github.com/nspcc-dev/neo-go v0.103.1
github.com/nspcc-dev/neofs-api-go/v2 v2.14.0
github.com/nspcc-dev/neofs-contract v0.18.0
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20231011074223-dd4e3e09a100
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20231113180740-3b24af0410c0
github.com/nspcc-dev/tzhash v1.7.1
github.com/olekukonko/tablewriter v0.0.5
github.com/panjf2000/ants/v2 v2.8.2
Expand Down Expand Up @@ -92,11 +92,11 @@ require (
github.com/spf13/afero v1.10.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 // indirect
github.com/twmb/murmur3 v1.1.5 // indirect
github.com/twmb/murmur3 v1.1.8 // indirect
github.com/urfave/cli v1.22.5 // indirect
go.uber.org/multierr v1.10.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/text v0.13.0 // indirect
Expand Down
23 changes: 10 additions & 13 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,8 +185,6 @@ github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc=
github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
Expand Down Expand Up @@ -245,8 +243,8 @@ github.com/nspcc-dev/dbft v0.0.0-20230515113611-25db6ba61d5c h1:uyK5aLbAhrnZtnvo
github.com/nspcc-dev/dbft v0.0.0-20230515113611-25db6ba61d5c/go.mod h1:kjBC9F8L25GR+kIHy/1KgG/KfcoGnVwIiyovgq1uszk=
github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22 h1:n4ZaFCKt1pQJd7PXoMJabZWK9ejjbLOVrkl/lOUmshg=
github.com/nspcc-dev/go-ordered-json v0.0.0-20220111165707-25110be27d22/go.mod h1:79bEUDEviBHJMFV6Iq6in57FEOCMcRhfQnfaf0ETA5U=
github.com/nspcc-dev/hrw v1.0.9 h1:17VcAuTtrstmFppBjfRiia4K2wA/ukXZhLFS8Y8rz5Y=
github.com/nspcc-dev/hrw v1.0.9/go.mod h1:l/W2vx83vMQo6aStyx2AuZrJ+07lGv2JQGlVkPG06MU=
github.com/nspcc-dev/hrw/v2 v2.0.0-20231115095647-bf62f4ad0a43 h1:zXkCRGTHqhkBRJo+EOSnwfDyv7excpz2qN/RQUknm5A=
github.com/nspcc-dev/hrw/v2 v2.0.0-20231115095647-bf62f4ad0a43/go.mod h1:BGU4YsuoFXjQddsCfUXpq5uNr2A8W4PrWbiljdD/TpU=
github.com/nspcc-dev/locode-db v0.4.1-0.20231110164540-a21ea2774f2e h1:mcL78g9av0/CYVrgMgGqRJLwZAfvLdIaoaPnEDLC6n8=
github.com/nspcc-dev/locode-db v0.4.1-0.20231110164540-a21ea2774f2e/go.mod h1:M+v8C+t9vwYbIVEdTPX/0w2LFRY+FJnYbVUdrzq7iWQ=
github.com/nspcc-dev/neo-go v0.103.1 h1:BfRBceHUu8jSc1KQy7CzmQ/pa+xzAmgcyteGf0/IGgM=
Expand All @@ -259,8 +257,8 @@ github.com/nspcc-dev/neofs-contract v0.18.0 h1:9g50b16s0mQFFskG93yRSWh4KL7yYOW+x
github.com/nspcc-dev/neofs-contract v0.18.0/go.mod h1:UQr1rUjg0eibLwJd6vfsJJEUBnmRysCg8XQd1HYiS2w=
github.com/nspcc-dev/neofs-crypto v0.4.0 h1:5LlrUAM5O0k1+sH/sktBtrgfWtq1pgpDs09fZo+KYi4=
github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20231011074223-dd4e3e09a100 h1:1m/7YyGJk32IEAl4/NFKY6Li/iqZlQyhpLR5MhUtQts=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20231011074223-dd4e3e09a100/go.mod h1:4LToeC6jfed7PWoJXDTaj9sS4W+C1xm2SbCe709VR8U=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20231113180740-3b24af0410c0 h1:y24pnVwDLaBAyh8fFcptAvtVfVDay/5jONgtO+cWZfI=
github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20231113180740-3b24af0410c0/go.mod h1:6ok9MU40TP0xAVfzisIg4KP+qhP670hra0mKpIGlWT8=
github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE=
github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso=
github.com/nspcc-dev/tzhash v1.7.1 h1:6zmexLqdTF/ssbUAh7XJS7RxgKWaw28kdNpE/4UFdEU=
Expand Down Expand Up @@ -324,7 +322,6 @@ github.com/spf13/viper v1.17.0/go.mod h1:BmMMMLQXSbcHK6KAOiFLz0l5JHrU89OdIRHvsk0
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
Expand All @@ -337,8 +334,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs=
github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM=
github.com/twmb/murmur3 v1.1.5 h1:i9OLS9fkuLzBXjt6dptlAEyk58fJsSTXbRg3SgVyqgk=
github.com/twmb/murmur3 v1.1.5/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/twmb/murmur3 v1.1.8 h1:8Yt9taO/WN3l08xErzjeschgZU2QSrwm1kclYq+0aRg=
github.com/twmb/murmur3 v1.1.8/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 h1:JwtAtbp7r/7QSyGz8mKUbYJBg2+6Cd7OjM8o/GNOcVo=
Expand Down Expand Up @@ -380,8 +377,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
Expand All @@ -405,7 +402,7 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand Down Expand Up @@ -576,7 +573,7 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
10 changes: 3 additions & 7 deletions pkg/local_object_storage/engine/evacuate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import (
"errors"
"fmt"

"github.com/nspcc-dev/hrw"
"github.com/nspcc-dev/hrw/v2"
meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
"github.com/nspcc-dev/neofs-node/pkg/util"
Expand Down Expand Up @@ -96,11 +96,6 @@ func (e *StorageEngine) Evacuate(prm EvacuateShardPrm) (EvacuateShardRes, error)
}
e.mtx.RUnlock()

weights := make([]float64, 0, len(shards))
for i := range shards {
weights = append(weights, e.shardWeight(shards[i].Shard))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if we have different capacity for every shard?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we can add weight to sort it more "honestly" and will not know what weight to use when a Get request is here

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We better keep the functionality as is then and create an issue to solve it in the future. We will need this, shard can be put into/retrieved from the metabase.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shard can be put into/retrieved from the metabase.

if meta is not per shard (not the current case)

Copy link
Member

@roman-khimov roman-khimov Nov 2, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, sure. But the weighting itself should be kept.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

But the weighting itself should be kept.

you mean this PR? this code is currently broken for sure: it always uses 0 and confuses. i suggest returning weighting (reworking in fact, the current version can't be used at all) when we need it and when we are ready for it

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why do you think it needs a complete rework?

Copy link
Member Author

@carpawell carpawell Nov 3, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. it is not used and not usable. i mean totally. it is commented as Amount of free disk space. Measured in kilobytes. and this field is never changed (always zero). and never was changed from the beginnings
  2. shard currently does not know how much it stores. just no code uses it, nobody ever asked about it. some estimation metrics per container, some calculations inside bbolt's transactions, some internal changes by background workers, and so on. all of it does not use this field (see 1.)
  3. if we want it to be used as a weight, it will be the first storage use case, so meta per shard should be reworked

i do not mean weighting is hard to implement, but some things should be done first, and currently it just does wrong things

}

shardMap := make(map[string]*shard.Shard)
for i := range sidList {
for j := range shards {
Expand Down Expand Up @@ -139,6 +134,7 @@ mainLoop:
loop:
for i := range lst {
addr := lst[i].Address
addrHash := hrw.WrapBytes([]byte(addr.EncodeToString()))

var getPrm shard.GetPrm
getPrm.SetAddress(addr)
Expand All @@ -151,7 +147,7 @@ mainLoop:
return res, err
}

hrw.SortSliceByWeightValue(shards, weights, hrw.Hash([]byte(addr.EncodeToString())))
hrw.Sort(shards, addrHash)
for j := range shards {
if _, ok := shardMap[shards[j].ID().String()]; ok {
continue
Expand Down
13 changes: 2 additions & 11 deletions pkg/local_object_storage/engine/shards.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import (
"sync/atomic"

"github.com/google/uuid"
"github.com/nspcc-dev/hrw"
"github.com/nspcc-dev/hrw/v2"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/util/logicerr"
Expand Down Expand Up @@ -192,25 +192,16 @@ func generateShardID() (*shard.ID, error) {
return shard.NewIDFromBytes(bin), nil
}

func (e *StorageEngine) shardWeight(sh *shard.Shard) float64 {
weightValues := sh.WeightValues()

return float64(weightValues.FreeSpace)
}

func (e *StorageEngine) sortShardsByWeight(objAddr interface{ EncodeToString() string }) []hashedShard {
e.mtx.RLock()
defer e.mtx.RUnlock()

shards := make([]hashedShard, 0, len(e.shards))
weights := make([]float64, 0, len(e.shards))

for _, sh := range e.shards {
shards = append(shards, hashedShard(sh))
weights = append(weights, e.shardWeight(sh.Shard))
}

hrw.SortSliceByWeightValue(shards, weights, hrw.Hash([]byte(objAddr.EncodeToString())))
hrw.Sort(shards, hrw.WrapBytes([]byte(objAddr.EncodeToString())))

return shards
}
Expand Down
3 changes: 0 additions & 3 deletions pkg/local_object_storage/shard/info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,6 @@ type Info struct {
// Information about the Write Cache.
WriteCacheInfo writecache.Info

// Weight parameters of the shard.
WeightValues WeightValues

// ErrorCount contains amount of errors occurred in shard operations.
ErrorCount uint32

Expand Down
12 changes: 0 additions & 12 deletions pkg/local_object_storage/shard/weight.go
View file
Open in desktop

This file was deleted.

10 changes: 8 additions & 2 deletions pkg/services/reputation/common/managers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ package common
import (
"fmt"

"github.com/nspcc-dev/hrw"
"github.com/nspcc-dev/hrw/v2"
netmapcore "github.com/nspcc-dev/neofs-node/pkg/core/netmap"
apiNetmap "github.com/nspcc-dev/neofs-sdk-go/netmap"
apireputation "github.com/nspcc-dev/neofs-sdk-go/reputation"
Expand Down Expand Up @@ -72,6 +72,12 @@ func (x nodeServer) ExternalAddresses() []string {
return (apiNetmap.NodeInfo)(x).ExternalAddresses()
}

type hashableUint uint64

func (h hashableUint) Hash() uint64 {
return uint64(h)
}

// BuildManagers sorts nodes in NetMap with HRW algorithms and
// takes the next node after the current one as the only manager.
func (mb *managerBuilder) BuildManagers(epoch uint64, p apireputation.PeerID) ([]ServerInfo, error) {
Expand All @@ -92,7 +98,7 @@ func (mb *managerBuilder) BuildManagers(epoch uint64, p apireputation.PeerID) ([

copy(nodes, nmNodes)

hrw.SortSliceByValue(nodes, epoch)
hrw.Sort(nodes, hashableUint(epoch))

for i := range nodes {
if apireputation.ComparePeerKey(p, nodes[i].PublicKey()) {
Expand Down
Loading