Skip to content

Commit

Permalink
node/object/put: push first and previous parts to chain
Browse files Browse the repository at this point in the history 
A first object has its own OID to simplify detection of such objects by indexed
meta-information. Closes #2935.

Signed-off-by: Pavel Karpy <[email protected]>
  • Loading branch information
@carpawell
carpawell committed Dec 28, 2024
1 parent a87f16d commit 0fa7ad9
Show file tree
Hide file tree
Showing 5 changed files with 49 additions and 13 deletions.
16 changes: 13 additions & 3 deletions pkg/core/object/replicate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@ const (
networkMagicKey = "network"

// optional fields.
deletedKey = "deleted"
lockedKey = "locked"
firstPartKey = "firstPart"
previousPartKey = "previousPart"
deletedKey = "deleted"
lockedKey = "locked"
)

// EncodeReplicationMetaInfo uses NEO's map (strict order) serialized format as a raw
Expand All @@ -31,9 +33,11 @@ const (
// "size": payload size
// "validUntil": last valid block number for meta information
// "network": network magic
// "firstPart": [OPTIONAL] _raw_ object ID (32 bytes)
// "previousPart": [OPTIONAL] _raw_ object ID (32 bytes)
// "deleted": [OPTIONAL] array of _raw_ object IDs
// "locked": [OPTIONAL] array of _raw_ object IDs
func EncodeReplicationMetaInfo(cID cid.ID, oID oid.ID, pSize uint64,
func EncodeReplicationMetaInfo(cID cid.ID, oID, firstPart, previousPart oid.ID, pSize uint64,
deleted, locked []oid.ID, vub uint64, magicNumber uint32) []byte {
kvs := []stackitem.MapElement{
kv(cidKey, cID[:]),
Expand All @@ -43,6 +47,12 @@ func EncodeReplicationMetaInfo(cID cid.ID, oID oid.ID, pSize uint64,
kv(networkMagicKey, magicNumber),
}

if !firstPart.IsZero() {
kvs = append(kvs, kv(firstPartKey, firstPart[:]))
}
if !previousPart.IsZero() {
kvs = append(kvs, kv(previousPartKey, previousPart[:]))
}
if len(deleted) > 0 {
kvs = append(kvs, oidsKV(deletedKey, deleted))
}
Expand Down
22 changes: 17 additions & 5 deletions pkg/core/object/replicate_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ type m struct {
vub uint64
magic uint32

first oid.ID
prev oid.ID
deleted []oid.ID
locked []oid.ID
}
Expand All @@ -31,6 +33,8 @@ func TestMetaInfo(t *testing.T) {
size: rand.Uint64(),
vub: rand.Uint64(),
magic: rand.Uint32(),
first: oidtest.ID(),
prev: oidtest.ID(),
deleted: oidtest.IDs(10),
locked: oidtest.IDs(10),
}
Expand All @@ -40,6 +44,8 @@ func TestMetaInfo(t *testing.T) {
})

t.Run("no optional", func(t *testing.T) {
meta.first = oid.ID{}
meta.prev = oid.ID{}
meta.deleted = nil
meta.deleted = nil
meta.locked = nil
Expand All @@ -49,7 +55,7 @@ func TestMetaInfo(t *testing.T) {
}

func testMeta(t *testing.T, m m, full bool) {
raw := EncodeReplicationMetaInfo(m.cID, m.oID, m.size, m.deleted, m.locked, m.vub, m.magic)
raw := EncodeReplicationMetaInfo(m.cID, m.oID, m.first, m.prev, m.size, m.deleted, m.locked, m.vub, m.magic)
item, err := stackitem.Deserialize(raw)
require.NoError(t, err)

Expand Down Expand Up @@ -77,11 +83,17 @@ func testMeta(t *testing.T, m m, full bool) {
return
}

require.Equal(t, deletedKey, string(mm[5].Key.Value().([]byte)))
require.Equal(t, m.deleted, stackItemToOIDs(t, mm[5].Value))
require.Equal(t, firstPartKey, string(mm[5].Key.Value().([]byte)))
require.Equal(t, m.first[:], mm[5].Value.Value().([]byte))

require.Equal(t, lockedKey, string(mm[6].Key.Value().([]byte)))
require.Equal(t, m.locked, stackItemToOIDs(t, mm[6].Value))
require.Equal(t, previousPartKey, string(mm[6].Key.Value().([]byte)))
require.Equal(t, m.prev[:], mm[6].Value.Value().([]byte))

require.Equal(t, deletedKey, string(mm[7].Key.Value().([]byte)))
require.Equal(t, m.deleted, stackItemToOIDs(t, mm[7].Value))

require.Equal(t, lockedKey, string(mm[8].Key.Value().([]byte)))
require.Equal(t, m.locked, stackItemToOIDs(t, mm[8].Value))
}

func stackItemToOIDs(t *testing.T, value stackitem.Item) []oid.ID {
Expand Down
13 changes: 10 additions & 3 deletions pkg/network/transport/object/grpc/replication.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,13 @@ func objectFromMessage(gMsg *objectGRPC.Object) (*object.Object, error) {
}

func (s *Server) metaInfoSignature(o object.Object) ([]byte, error) {
firstObj := o.GetFirstID()
if o.HasParent() && firstObj.IsZero() {
// object itself is the first one
firstObj = o.GetID()
}
prevObj := o.GetPreviousID()

var deleted []oid.ID
var locked []oid.ID
switch o.Type() {
Expand Down Expand Up @@ -235,9 +242,9 @@ func (s *Server) metaInfoSignature(o object.Object) ([]byte, error) {
secondBlock := firstBlock + currentEpochDuration
thirdBlock := secondBlock + currentEpochDuration

firstMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), o.PayloadSize(), deleted, locked, firstBlock, s.mNumber)
secondMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), o.PayloadSize(), deleted, locked, secondBlock, s.mNumber)
thirdMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), o.PayloadSize(), deleted, locked, thirdBlock, s.mNumber)
firstMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), deleted, locked, firstBlock, s.mNumber)
secondMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), deleted, locked, secondBlock, s.mNumber)
thirdMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), deleted, locked, thirdBlock, s.mNumber)

var firstSig neofscrypto.Signature
var secondSig neofscrypto.Signature
Expand Down
2 changes: 1 addition & 1 deletion pkg/network/transport/object/grpc/replication_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,7 @@ func TestServer_Replicate(t *testing.T) {

require.Equal(t, signer.PublicKeyBytes, sig.PublicKeyBytes())
require.True(t, sig.Verify(objectcore.EncodeReplicationMetaInfo(
o.GetContainerID(), o.GetID(), o.PayloadSize(), nil, nil,
o.GetContainerID(), o.GetID(), o.GetFirstID(), o.GetPreviousID(), o.PayloadSize(), nil, nil,
uint64((123+1+i)*240), mNumber)))

sigsRaw = sigsRaw[:4+l]
Expand Down
9 changes: 8 additions & 1 deletion pkg/services/object/put/distributed.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,13 @@ func (t *distributedTarget) Close() (oid.ID, error) {
t.encodedObject.b = nil
}()

firstObj := t.obj.GetFirstID()
if t.obj.HasParent() && firstObj.IsZero() && t.obj.SplitID() == nil {
// object itself is the first one
firstObj = t.obj.GetID()
}
prevObj := t.obj.GetPreviousID()

t.obj.SetPayload(t.encodedObject.b[t.encodedObject.pldOff:])

tombOrLink := t.obj.Type() == objectSDK.TypeLink || t.obj.Type() == objectSDK.TypeTombstone
Expand Down Expand Up @@ -152,7 +159,7 @@ func (t *distributedTarget) Close() (oid.ID, error) {
}

expectedVUB := (uint64(t.currentBlock)/t.currentEpochDuration + 2) * t.currentEpochDuration
t.objSharedMeta = object.EncodeReplicationMetaInfo(t.obj.GetContainerID(), t.obj.GetID(), t.obj.PayloadSize(), deletedObjs,
t.objSharedMeta = object.EncodeReplicationMetaInfo(t.obj.GetContainerID(), t.obj.GetID(), firstObj, prevObj, t.obj.PayloadSize(), deletedObjs,
lockedObjs, expectedVUB, t.networkMagicNumber)
id := t.obj.GetID()
err := t.placementIterator.iterateNodesForObject(id, t.sendObject)
Expand Down

0 comments on commit 0fa7ad9

Please sign in to comment.