Merge pull request #387 from onflow/fxamacker/add-fix-broken-referenc…

…e-function

Add feature to enable migrations to fix references to non-existent registers

.github/workflows/ci.yml

@@ -31,7 +31,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-latest, ubuntu-latest]
-        go-version: [1.17, 1.18, 1.19]
+        go-version: ['1.20']
 
     steps:
     - name: Install Go

.github/workflows/coverage.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.19'
+          go-version: '1.20'
           check-latest: true
 
       - name: Get dependencies

.github/workflows/safer-golangci-lint.yml

@@ -1,4 +1,4 @@
-# Copyright © 2021 Montgomery Edwards⁴⁴⁸ (github.com/x448).
+# Copyright © 2021-2023 Montgomery Edwards⁴⁴⁸ (github.com/x448).
 # This file is licensed under MIT License.
 #
 # Safer GitHub Actions Workflow for golangci-lint.
@@ -27,12 +27,13 @@
 #   1. GOLINTERS_VERSION
 #   2. GOLINTERS_TGZ_DGST
 #
-# Release v1.51.1 (February 5, 2023)
-#   - Bump golangci-lint to 1.51.1
-#   - Shuffle some comments
-#   - Hash of golangci-lint-1.50.1-linux-amd64.tar.gz
-#     - SHA-256: 17aeb26c76820c22efa0e1838b0ab93e90cfedef43fbfc9a2f33f27eb9e5e070
-#                This SHA-256 digest matches golangci-lint-1.51.1-checksums.txt at
+# Release v1.52.2 (May 14, 2023)
+#   - Bump Go to 1.20
+#   - Bump actions/setup-go to v4
+#   - Bump golangci-lint to 1.52.2
+#   - Hash of golangci-lint-1.52.2-linux-amd64.tar.gz
+#     - SHA-256: c9cf72d12058a131746edd409ed94ccd578fbd178899d1ed41ceae3ce5f54501
+#                This SHA-256 digest matches golangci-lint-1.52.2-checksums.txt at
 #                https://github.com/golangci/golangci-lint/releases
 #
 name: linters
@@ -43,15 +44,14 @@ permissions: {}
 on:
   workflow_dispatch:
   pull_request:
-    types: [opened, synchronize, closed]
   push:
     branches: [main, master]
 
 env:
-  GO_VERSION: 1.19
-  GOLINTERS_VERSION: 1.51.1
+  GO_VERSION: '1.20'
+  GOLINTERS_VERSION: 1.52.2
   GOLINTERS_ARCH: linux-amd64
-  GOLINTERS_TGZ_DGST: 17aeb26c76820c22efa0e1838b0ab93e90cfedef43fbfc9a2f33f27eb9e5e070
+  GOLINTERS_TGZ_DGST: c9cf72d12058a131746edd409ed94ccd578fbd178899d1ed41ceae3ce5f54501
   GOLINTERS_TIMEOUT: 15m
   OPENSSL_DGST_CMD: openssl dgst -sha256 -r
   CURL_CMD: curl --proto =https --tlsv1.2 --location --silent --show-error --fail

storage.go

@@ -21,6 +21,7 @@ package atree
 import (
 	"bytes"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"sort"
 	"strings"
@@ -1027,3 +1028,271 @@ func storeSlab(storage SlabStorage, slab Slab) error {
 	}
 	return nil
 }
+
+// FixLoadedBrokenReferences traverses loaded slabs and fixes broken references in maps.
+// A broken reference is a StorageID referencing a non-existent slab.
+// To fix a map containing broken references, this function replaces broken map with
+// empty map having the same StorageID and also removes all slabs in the old map.
+// Limitations:
+// - only fix broken references in map
+// - only traverse loaded slabs in deltas and cache
+// NOTE: The intended use case is to enable migration programs in onflow/flow-go to
+// fix broken references.  As of April 2024, only 10 registers in testnet (not mainnet)
+// were found to have broken references and they seem to have resulted from a bug
+// that was fixed 2 years ago by https://github.com/onflow/cadence/pull/1565.
+func (s *PersistentSlabStorage) FixLoadedBrokenReferences(needToFix func(old Value) bool) (
+	fixedStorageIDs map[StorageID][]StorageID, // key: root slab ID, value: slab IDs containing broken refs
+	skippedStorageIDs map[StorageID][]StorageID, // key: root slab ID, value: slab IDs containing broken refs
+	err error,
+) {
+
+	// parentOf is used to find root slab from non-root slab.
+	// Broken reference can be in non-root slab, and we need StorageID of root slab
+	// to replace broken map by creating an empty new map with same StorageID.
+	parentOf := make(map[StorageID]StorageID)
+
+	getRootSlabID := func(id StorageID) StorageID {
+		for {
+			parentID, ok := parentOf[id]
+			if ok {
+				id = parentID
+			} else {
+				return id
+			}
+		}
+	}
+
+	hasBrokenReferenceInSlab := func(id StorageID, slab Slab) bool {
+		if slab == nil {
+			return false
+		}
+
+		var isMetaDataSlab bool
+
+		switch slab.(type) {
+		case *ArrayMetaDataSlab, *MapMetaDataSlab:
+			isMetaDataSlab = true
+		}
+
+		var foundBrokenRef bool
+		for _, childStorable := range slab.ChildStorables() {
+
+			storageIDStorable, ok := childStorable.(StorageIDStorable)
+			if !ok {
+				continue
+			}
+
+			childID := StorageID(storageIDStorable)
+
+			// Track parent-child relationship of root slabs and non-root slabs.
+			if isMetaDataSlab {
+				parentOf[childID] = id
+			}
+
+			if s.existIfLoaded(childID) {
+				continue
+			}
+
+			foundBrokenRef = true
+
+			if !isMetaDataSlab {
+				return true
+			}
+		}
+
+		return foundBrokenRef
+	}
+
+	var brokenStorageIDs []StorageID
+
+	// Iterate delta slabs.
+	for id, slab := range s.deltas {
+		if hasBrokenReferenceInSlab(id, slab) {
+			brokenStorageIDs = append(brokenStorageIDs, id)
+		}
+	}
+
+	// Iterate cache slabs.
+	for id, slab := range s.cache {
+		if _, ok := s.deltas[id]; ok {
+			continue
+		}
+		if hasBrokenReferenceInSlab(id, slab) {
+			brokenStorageIDs = append(brokenStorageIDs, id)
+		}
+	}
+
+	if len(brokenStorageIDs) == 0 {
+		return nil, nil, nil
+	}
+
+	rootSlabStorageIDsWithBrokenData := make(map[StorageID][]StorageID)
+	var errs []error
+
+	// Find StorageIDs of root slab for slabs containing broken references.
+	for _, id := range brokenStorageIDs {
+		rootID := getRootSlabID(id)
+		if rootID == StorageIDUndefined {
+			errs = append(errs, fmt.Errorf("failed to get root slab id for slab %s", id))
+			continue
+		}
+		rootSlabStorageIDsWithBrokenData[rootID] = append(rootSlabStorageIDsWithBrokenData[rootID], id)
+	}
+
+	for rootStorageID, brokenStorageIDs := range rootSlabStorageIDsWithBrokenData {
+		rootSlab := s.RetrieveIfLoaded(rootStorageID)
+		if rootSlab == nil {
+			errs = append(errs, fmt.Errorf("failed to retrieve loaded root slab %s", rootStorageID))
+			continue
+		}
+
+		switch rootSlab := rootSlab.(type) {
+		case MapSlab:
+			value, err := rootSlab.StoredValue(s)
+			if err != nil {
+				errs = append(errs, fmt.Errorf("failed to convert slab %s into value", rootSlab.ID()))
+				continue
+			}
+
+			if needToFix(value) {
+				err := s.fixBrokenReferencesInMap(rootSlab)
+				if err != nil {
+					errs = append(errs, err)
+					continue
+				}
+			} else {
+				if skippedStorageIDs == nil {
+					skippedStorageIDs = make(map[StorageID][]StorageID)
+				}
+				skippedStorageIDs[rootStorageID] = brokenStorageIDs
+			}
+
+		default:
+			// IMPORTANT: Only handle map slabs for now.  DO NOT silently fix currently unknown problems.
+			errs = append(errs, fmt.Errorf("failed to fix broken references in non-map slab %s (%T)", rootSlab.ID(), rootSlab))
+		}
+	}
+
+	for id := range skippedStorageIDs {
+		delete(rootSlabStorageIDsWithBrokenData, id)
+	}
+
+	return rootSlabStorageIDsWithBrokenData, skippedStorageIDs, errors.Join(errs...)
+}
+
+// fixBrokenReferencesInMap replaces replaces broken map with empty map
+// having the same StorageID and also removes all slabs in the old map.
+func (s *PersistentSlabStorage) fixBrokenReferencesInMap(old MapSlab) error {
+	id := old.ID()
+
+	oldExtraData := old.ExtraData()
+
+	// Create an empty map with the same StorgeID, type, and seed as the old map.
+	new := &MapDataSlab{
+		header: MapSlabHeader{
+			id:   id,
+			size: mapRootDataSlabPrefixSize + hkeyElementsPrefixSize,
+		},
+		extraData: &MapExtraData{
+			TypeInfo: oldExtraData.TypeInfo,
+			Seed:     oldExtraData.Seed,
+		},
+		elements: newHkeyElements(0),
+	}
+
+	// Store new empty map with the same StorageID.
+	err := s.Store(id, new)
+	if err != nil {
+		return err
+	}
+
+	// Remove all slabs and references in old map.
+	references, _, err := s.getAllChildReferences(old)
+	if err != nil {
+		return err
+	}
+
+	for _, childID := range references {
+		err = s.Remove(childID)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *PersistentSlabStorage) existIfLoaded(id StorageID) bool {
+	// Check deltas.
+	if slab, ok := s.deltas[id]; ok {
+		return slab != nil
+	}
+
+	// Check read cache.
+	if slab, ok := s.cache[id]; ok {
+		return slab != nil
+	}
+
+	return false
+}
+
+// GetAllChildReferences returns child references of given slab (all levels),
+// including nested container and theirs child references.
+func (s *PersistentSlabStorage) GetAllChildReferences(id StorageID) (
+	references []StorageID,
+	brokenReferences []StorageID,
+	err error,
+) {
+	slab, found, err := s.Retrieve(id)
+	if err != nil {
+		return nil, nil, err
+	}
+	if !found {
+		return nil, nil, NewSlabNotFoundErrorf(id, fmt.Sprintf("failed to get root slab by id %s", id))
+	}
+	return s.getAllChildReferences(slab)
+}
+
+// getAllChildReferences returns child references of given slab (all levels).
+func (s *PersistentSlabStorage) getAllChildReferences(slab Slab) (
+	references []StorageID,
+	brokenReferences []StorageID,
+	err error,
+) {
+	childStorables := slab.ChildStorables()
+
+	for len(childStorables) > 0 {
+
+		var nextChildStorables []Storable
+
+		for _, childStorable := range childStorables {
+
+			storageIDStorable, ok := childStorable.(StorageIDStorable)
+			if !ok {
+				continue
+			}
+
+			childID := StorageID(storageIDStorable)
+
+			childSlab, ok, err := s.Retrieve(childID)
+			if err != nil {
+				return nil, nil, err
+			}
+			if !ok {
+				brokenReferences = append(brokenReferences, childID)
+				continue
+			}
+
+			references = append(references, childID)
+
+			nextChildStorables = append(
+				nextChildStorables,
+				childSlab.ChildStorables()...,
+			)
+		}
+
+		childStorables = nextChildStorables
+	}
+
+	return references, brokenReferences, nil
+}