Revive dead object via control service (#2968)

Closes #1450.

CHANGELOG.md

@@ -14,6 +14,7 @@ attribute, which is used for container domain name in NNS contracts (#2954)
 - Save last epoch when metabase was resynchronized (#2966)
 - `neofs-lens meta last-resync-epoch` command (#2966)
 - `neofs-lens fstree cleanup-tmp` command (#2967)
+- `neofs-cli control object revive` command (#2968)
 
 ### Fixed
 - Do not search for tombstones when handling their expiration, use local indexes instead (#2929)

cmd/neofs-cli/modules/control/object.go

@@ -4,6 +4,8 @@ import (
 	"github.com/spf13/cobra"
 )
 
+const objectFlag = "object"
+
 var objectCmd = &cobra.Command{
 	Use:   "object",
 	Short: "Direct object operations with storage engine",
@@ -12,7 +14,9 @@ var objectCmd = &cobra.Command{
 func initControlObjectsCmd() {
 	objectCmd.AddCommand(listObjectsCmd)
 	objectCmd.AddCommand(objectStatusCmd)
+	objectCmd.AddCommand(reviveObjectCmd)
 
+	initControlObjectReviveCmd()
 	initControlObjectsListCmd()
 	initObjectStatusFlags()
 }

cmd/neofs-cli/modules/control/object_revive.go

@@ -0,0 +1,82 @@
+package control
+
+import (
+	"fmt"
+
+	rawclient "github.com/nspcc-dev/neofs-api-go/v2/rpc/client"
+	"github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/commonflags"
+	"github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/key"
+	"github.com/nspcc-dev/neofs-node/pkg/services/control"
+	"github.com/spf13/cobra"
+)
+
+var reviveObjectCmd = &cobra.Command{
+	Use:   "revive",
+	Short: "Forcefully revive object",
+	Long:  "Purge removal marks from metabases",
+	Args:  cobra.NoArgs,
+	RunE:  reviveObject,
+}
+
+func initControlObjectReviveCmd() {
+	initControlFlags(reviveObjectCmd)
+
+	flags := reviveObjectCmd.Flags()
+	flags.String(objectFlag, "", "Object address")
+}
+
+func reviveObject(cmd *cobra.Command, _ []string) error {
+	ctx, cancel := commonflags.GetCommandContext(cmd)
+	defer cancel()
+
+	pk, err := key.Get(cmd)
+	if err != nil {
+		return err
+	}
+	addressRaw, err := cmd.Flags().GetString(objectFlag)
+	if err != nil {
+		return fmt.Errorf("reading %s flag: %w", objectFlag, err)
+	}
+
+	var resp *control.ReviveObjectResponse
+	req := &control.ReviveObjectRequest{
+		Body: &control.ReviveObjectRequest_Body{
+			ObjectAddress: []byte(addressRaw),
+		},
+	}
+	err = signRequest(pk, req)
+	if err != nil {
+		return err
+	}
+
+	cli, err := getClient(ctx)
+	if err != nil {
+		return err
+	}
+
+	err = cli.ExecRaw(func(client *rawclient.Client) error {
+		resp, err = control.ReviveObject(client, req)
+		return err
+	})
+	if err != nil {
+		return fmt.Errorf("rpc error: %w", err)
+	}
+
+	err = verifyResponse(resp.GetSignature(), resp.GetBody())
+	if err != nil {
+		return err
+	}
+
+	shards := resp.GetBody().GetShards()
+	if len(shards) == 0 {
+		cmd.Println("<empty response>")
+		return nil
+	}
+
+	for _, shard := range shards {
+		cmd.Printf("Shard ID: %s\n", shard.ShardId)
+		cmd.Printf("Revival status: %s\n", shard.Status)
+	}
+
+	return nil
+}

cmd/neofs-cli/modules/control/object_status.go

@@ -11,8 +11,6 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const objectFlag = "object"
-
 var objectStatusCmd = &cobra.Command{
 	Use:          "status",
 	Short:        "Check current object status",

pkg/local_object_storage/engine/revive.go

@@ -0,0 +1,41 @@
+package engine
+
+import (
+	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
+	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
+	"go.uber.org/zap"
+)
+
+// ReviveShardStatus contains the Status of the object's revival in the Shard and Shard ID.
+type ReviveShardStatus struct {
+	ID     string
+	Status meta.ReviveStatus
+}
+
+// ReviveStatus represents the status of the object's revival in the StorageEngine.
+type ReviveStatus struct {
+	Shards []ReviveShardStatus
+}
+
+// ReviveObject forcefully revives object by oid.Address in the StorageEngine.
+// Iterate over all shards despite errors and purge all removal marks from all metabases.
+func (e *StorageEngine) ReviveObject(address oid.Address) (res ReviveStatus, err error) {
+	e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) {
+		reviveStatus, err := sh.ReviveObject(address)
+		id := *sh.ID()
+		res.Shards = append(res.Shards, ReviveShardStatus{
+			ID:     id.String(),
+			Status: reviveStatus,
+		})
+		if err != nil {
+			e.log.Warn("failed to revive object in shard",
+				zap.String("shard", id.String()),
+				zap.String("address", address.EncodeToString()),
+				zap.Error(err),
+			)
+		}
+
+		return false
+	})
+	return
+}

pkg/local_object_storage/metabase/revive.go

@@ -0,0 +1,147 @@
+package meta
+
+import (
+	"fmt"
+
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/util/logicerr"
+	"github.com/nspcc-dev/neofs-sdk-go/object"
+	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
+	"go.etcd.io/bbolt"
+)
+
+// ErrObjectWasNotRemoved is returned when object neither in the graveyard nor was marked with GC mark.
+var ErrObjectWasNotRemoved = logicerr.New("object neither in the graveyard nor was marked with GC mark")
+
+// ErrReviveFromContainerGarbage is returned when the object is in the container that marked with GC mark.
+var ErrReviveFromContainerGarbage = logicerr.New("revive from container marked with GC mark")
+
+type reviveStatusType int
+
+const (
+	// ReviveStatusGraveyard is the type of revival status of an object from a graveyard.
+	ReviveStatusGraveyard reviveStatusType = iota
+	// ReviveStatusGarbage is the type of revival status of an object from the garbage bucket.
+	ReviveStatusGarbage
+	// ReviveStatusError is the type of status when an error occurs during revive.
+	ReviveStatusError
+)
+
+// ReviveStatus groups the resulting values of ReviveObject operation.
+// Contains the type of revival status and message for details.
+type ReviveStatus struct {
+	statusType reviveStatusType
+	message    string
+}
+
+// Message returns message of status.
+func (s *ReviveStatus) Message() string {
+	return s.message
+}
+
+// StatusType returns the type of revival status.
+func (s *ReviveStatus) StatusType() reviveStatusType {
+	return s.statusType
+}
+
+func (s *ReviveStatus) setStatusGraveyard(tomb string) {
+	s.statusType = ReviveStatusGraveyard
+	s.message = fmt.Sprintf("successful revival from graveyard, tomb: %s", tomb)
+}
+
+func (s *ReviveStatus) setStatusGarbage() {
+	s.statusType = ReviveStatusGarbage
+	s.message = "successful revival from garbage bucket"
+}
+
+func (s *ReviveStatus) setStatusError(err error) {
+	s.statusType = ReviveStatusError
+	s.message = fmt.Sprintf("didn't revive, err: %v", err)
+}
+
+// ReviveObject revives object by oid.Address. Removes GCMark/Tombstone records in the corresponding buckets
+// and restore metrics.
+func (db *DB) ReviveObject(addr oid.Address) (res ReviveStatus, err error) {
+	db.modeMtx.RLock()
+	defer db.modeMtx.RUnlock()
+
+	if db.mode.ReadOnly() {
+		res.setStatusError(ErrReadOnlyMode)
+		return res, ErrReadOnlyMode
+	} else if db.mode.NoMetabase() {
+		res.setStatusError(ErrDegradedMode)
+		return res, ErrDegradedMode
+	}
+
+	currEpoch := db.epochState.CurrentEpoch()
+
+	err = db.boltDB.Update(func(tx *bbolt.Tx) error {
+		garbageObjectsBKT := tx.Bucket(garbageObjectsBucketName)
+		garbageContainersBKT := tx.Bucket(garbageContainersBucketName)
+		graveyardBKT := tx.Bucket(graveyardBucketName)
+
+		buf := make([]byte, addressKeySize)
+
+		targetKey := addressKey(addr, buf)
+
+		if graveyardBKT == nil || garbageObjectsBKT == nil {
+			// incorrect metabase state, does not make
+			// sense to check garbage bucket
+			return ErrObjectWasNotRemoved
+		}
+
+		val := graveyardBKT.Get(targetKey)
+		if val != nil {
+			// object in the graveyard
+			if err := graveyardBKT.Delete(targetKey); err != nil {
+				return err
+			}
+
+			var tombAddress oid.Address
+			if err := decodeAddressFromKey(&tombAddress, val[:addressKeySize]); err != nil {
+				return err
+			}
+			res.setStatusGraveyard(tombAddress.EncodeToString())
+		} else {
+			val = garbageContainersBKT.Get(targetKey[:cidSize])
+			if val != nil {
+				return ErrReviveFromContainerGarbage
+			}
+
+			val = garbageObjectsBKT.Get(targetKey)
+			if val != nil {
+				// object marked with GC mark
+				res.setStatusGarbage()
+			} else {
+				// neither in the graveyard
+				// nor was marked with GC mark
+				return ErrObjectWasNotRemoved
+			}
+		}
+
+		if err := garbageObjectsBKT.Delete(targetKey); err != nil {
+			return err
+		}
+
+		if obj, err := db.get(tx, addr, buf, false, true, currEpoch); err == nil {
+			// if object is stored, and it is regular object then update bucket
+			// with container size estimations
+			if obj.Type() == object.TypeRegular {
+				if err := changeContainerSize(tx, addr.Container(), obj.PayloadSize(), true); err != nil {
+					return err
+				}
+			}
+
+			// also need to restore logical counter
+			if err := db.updateCounter(tx, logical, 1, true); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+	if err != nil {
+		res.setStatusError(err)
+	}
+
+	return
+}