diff --git a/persistence/blockstore/block_store.go b/persistence/blockstore/block_store.go index 647d3e634..360804a33 100644 --- a/persistence/blockstore/block_store.go +++ b/persistence/blockstore/block_store.go @@ -93,6 +93,10 @@ func (bs *blockStore) Stop() error { return bs.kv.Stop() } +func (bs *blockStore) Backup(path string) error { + return bs.kv.Backup(path) +} + func (bs *blockStore) Delete(key []byte) error { return bs.kv.Delete(key) } func (bs *blockStore) Exists(key []byte) (bool, error) { return bs.kv.Exists(key) } func (bs *blockStore) GetAll(prefixKey []byte, descending bool) (keys, values [][]byte, err error) { diff --git a/persistence/kvstore/kvstore.go b/persistence/kvstore/kvstore.go index a352e4257..f4ea132d2 100644 --- a/persistence/kvstore/kvstore.go +++ b/persistence/kvstore/kvstore.go @@ -4,7 +4,10 @@ package kvstore import ( "errors" + "fmt" "log" + "os" + "path/filepath" badger "github.com/dgraph-io/badger/v3" "github.com/pokt-network/smt" @@ -22,6 +25,8 @@ type KVStore interface { GetAll(prefixKey []byte, descending bool) (keys, values [][]byte, err error) Exists(key []byte) (bool, error) ClearAll() error + + Backup(filepath string) error } const ( @@ -141,6 +146,30 @@ func (store *badgerKVStore) Stop() error { return store.db.Close() } +// Backup creates a backup for the badgerDB at the provided path. +// It creates a file +func (store *badgerKVStore) Backup(backupPath string) error { + // create backup directory if it doesn't exist + if err := os.MkdirAll(filepath.Dir(backupPath), os.ModePerm); err != nil { + return fmt.Errorf("failed to create backup directory: %v", err) + } + + // create the backup file itself + backupFile, err := os.Create(backupPath) + if err != nil { + return fmt.Errorf("failed to create backup file: %v", err) + } + defer backupFile.Close() + + // dump the database to the backup file + _, err = store.db.Backup(backupFile, 0) + if err != nil { + return err + } + + return nil +} + // PrefixEndBytes returns the end byteslice for a noninclusive range // that would include all byte slices for which the input is the prefix func prefixEndBytes(prefix []byte) []byte { diff --git a/persistence/kvstore/kvstore_test.go b/persistence/kvstore/kvstore_test.go index 402ce4a98..4f272621d 100644 --- a/persistence/kvstore/kvstore_test.go +++ b/persistence/kvstore/kvstore_test.go @@ -2,6 +2,9 @@ package kvstore import ( "encoding/hex" + "io" + "os" + "path/filepath" "strings" "testing" @@ -330,6 +333,55 @@ func TestKVStore_ClearAll(t *testing.T) { require.NoError(t, err) } +func TestKVStore_Backup(t *testing.T) { + t.Run("should backup an in-memory database", func(t *testing.T) { + store := NewMemKVStore() + require.NotNil(t, store) + + tmpdir := t.TempDir() + path := filepath.Join(tmpdir, "TestKVStore_Backup_InMemory.bak") + require.NoError(t, store.Backup(path)) + + empty, err := isEmpty(t, tmpdir) + require.NoError(t, err) + require.False(t, empty) + + // open the directory and assert on individual files + dir, err := os.Open(tmpdir) + require.NoError(t, err) + defer dir.Close() + + files, err := dir.Readdir(0) // 0 means read all directory entries + require.NoError(t, err) + require.Equal(t, len(files), 1) + }) + t.Run("should backup an on-disk store database", func(t *testing.T) { + tmpdir := t.TempDir() + kvpath := filepath.Join(tmpdir, "TestKVStore_Backup_OnDisk_Source.bak") + store, err := NewKVStore(kvpath) + require.NoError(t, err) + require.NotNil(t, store) + + backupDir := t.TempDir() + path := filepath.Join(backupDir, "TestKVStore_Backup_OnDisk_Destination.bak") + require.NoError(t, store.Backup(path)) + + empty, err := isEmpty(t, backupDir) + require.NoError(t, err) + require.False(t, empty) + + // open the directory and assert on individual files + dir, err := os.Open(backupDir) + require.NoError(t, err) + defer dir.Close() + + files, err := dir.Readdir(0) // 0 means read all directory entries + require.NoError(t, err) + require.NoError(t, err) + require.Equal(t, len(files), 1) + }) +} + func setupStore(t *testing.T, store KVStore) { t.Helper() err := store.Set([]byte("foo"), []byte("bar")) @@ -337,3 +389,18 @@ func setupStore(t *testing.T, store KVStore) { err = store.Set([]byte("baz"), []byte("bin")) require.NoError(t, err) } + +func isEmpty(t *testing.T, dir string) (bool, error) { + t.Helper() + f, err := os.Open(dir) + if err != nil { + return false, err + } + defer f.Close() + + _, err = f.Readdirnames(1) + if err == io.EOF { + return true, nil + } + return false, err +} diff --git a/persistence/module.go b/persistence/module.go index 0d6acd17f..d16c27a6f 100644 --- a/persistence/module.go +++ b/persistence/module.go @@ -104,7 +104,7 @@ func (*persistenceModule) Create(bus modules.Bus, options ...modules.ModuleOptio trees.WithTreeStoreDirectory(persistenceCfg.TreesStoreDir), trees.WithLogger(m.logger)) if err != nil { - return nil, fmt.Errorf("failed to create TreeStoreModule: %w", err) + return nil, fmt.Errorf("failed to create %s: %w", modules.TreeStoreSubmoduleName, err) } m.config = persistenceCfg diff --git a/persistence/trees/atomic_test.go b/persistence/trees/atomic_test.go index 371c0d0c4..454e0822a 100644 --- a/persistence/trees/atomic_test.go +++ b/persistence/trees/atomic_test.go @@ -3,15 +3,16 @@ package trees import ( "encoding/hex" "fmt" + "io" + "os" "testing" "github.com/pokt-network/pocket/logger" mock_types "github.com/pokt-network/pocket/persistence/types/mocks" "github.com/pokt-network/pocket/shared/modules" - mockModules "github.com/pokt-network/pocket/shared/modules/mocks" + mock_modules "github.com/pokt-network/pocket/shared/modules/mocks" "github.com/golang/mock/gomock" - "github.com/rs/zerolog" "github.com/stretchr/testify/require" ) @@ -26,8 +27,8 @@ func TestTreeStore_AtomicUpdatesWithSuccessfulRollback(t *testing.T) { ctrl := gomock.NewController(t) mockTxIndexer := mock_types.NewMockTxIndexer(ctrl) - mockBus := mockModules.NewMockBus(ctrl) - mockPersistenceMod := mockModules.NewMockPersistenceModule(ctrl) + mockBus := mock_modules.NewMockBus(ctrl) + mockPersistenceMod := mock_modules.NewMockPersistenceModule(ctrl) mockBus.EXPECT().GetPersistenceModule().AnyTimes().Return(mockPersistenceMod) mockPersistenceMod.EXPECT().GetTxIndexer().AnyTimes().Return(mockTxIndexer) @@ -95,7 +96,7 @@ func TestTreeStore_AtomicUpdatesWithSuccessfulRollback(t *testing.T) { hash3 := ts.getStateHash() require.Equal(t, hash3, hash2) require.Equal(t, hash3, h1) - ts.Rollback() + require.NoError(t, ts.Rollback()) // confirm it's not in the tree v, err := ts.merkleTrees[TransactionsTreeName].tree.Get([]byte("fiz")) @@ -104,18 +105,94 @@ func TestTreeStore_AtomicUpdatesWithSuccessfulRollback(t *testing.T) { } func TestTreeStore_SaveAndLoad(t *testing.T) { + t.Parallel() + t.Run("should save a backup in a directory", func(t *testing.T) { + ts := newTestTreeStore(t) + tmpdir := t.TempDir() + // assert that the directory is empty before backup + ok, err := isEmpty(tmpdir) + require.NoError(t, err) + require.True(t, ok) + + // Trigger a backup + require.NoError(t, ts.Backup(tmpdir)) + + // assert that the directory is not empty after Backup has returned + ok, err = isEmpty(tmpdir) + require.NoError(t, err) + require.False(t, ok) + }) + t.Run("should load a backup and maintain TreeStore hash integrity", func(t *testing.T) { + ctrl := gomock.NewController(t) + tmpDir := t.TempDir() + + mockTxIndexer := mock_types.NewMockTxIndexer(ctrl) + mockBus := mock_modules.NewMockBus(ctrl) + mockPersistenceMod := mock_modules.NewMockPersistenceModule(ctrl) + + mockBus.EXPECT().GetPersistenceModule().AnyTimes().Return(mockPersistenceMod) + mockPersistenceMod.EXPECT().GetTxIndexer().AnyTimes().Return(mockTxIndexer) + + ts := &treeStore{ + logger: logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName), + treeStoreDir: tmpDir, + } + require.NoError(t, ts.Start()) + require.NotNil(t, ts.rootTree.tree) + + for _, treeName := range stateTreeNames { + err := ts.merkleTrees[treeName].tree.Update([]byte("foo"), []byte("bar")) + require.NoError(t, err) + } + + err := ts.Commit() + require.NoError(t, err) + + hash1 := ts.getStateHash() + require.NotEmpty(t, hash1) + + w, err := ts.save() + require.NoError(t, err) + require.NotNil(t, w) + require.NotNil(t, w.rootHash) + require.NotNil(t, w.merkleRoots) + + // Stop the first tree store so that it's databases are no longer used + require.NoError(t, ts.Stop()) + + // declare a second TreeStore with no trees then load the first worldstate into it + ts2 := &treeStore{ + logger: logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName), + treeStoreDir: tmpDir, + } + + // Load sets a tree store to the provided worldstate + err = ts2.Load(w) + require.NoError(t, err) + + hash2 := ts2.getStateHash() + + // Assert that hash is unchanged from save and load + require.Equal(t, hash1, hash2) + }) +} + +// creates a new tree store with a tmp directory for nodestore persistence +// and then starts the tree store and returns its pointer. +func newTestTreeStore(t *testing.T) *treeStore { + t.Helper() ctrl := gomock.NewController(t) tmpDir := t.TempDir() mockTxIndexer := mock_types.NewMockTxIndexer(ctrl) - mockBus := mockModules.NewMockBus(ctrl) - mockPersistenceMod := mockModules.NewMockPersistenceModule(ctrl) + mockBus := mock_modules.NewMockBus(ctrl) + mockPersistenceMod := mock_modules.NewMockPersistenceModule(ctrl) mockBus.EXPECT().GetPersistenceModule().AnyTimes().Return(mockPersistenceMod) mockPersistenceMod.EXPECT().GetTxIndexer().AnyTimes().Return(mockTxIndexer) ts := &treeStore{ - logger: &zerolog.Logger{}, + logger: logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName), treeStoreDir: tmpDir, } require.NoError(t, ts.Start()) @@ -132,29 +209,19 @@ func TestTreeStore_SaveAndLoad(t *testing.T) { hash1 := ts.getStateHash() require.NotEmpty(t, hash1) - w, err := ts.save() - require.NoError(t, err) - require.NotNil(t, w) - require.NotNil(t, w.rootHash) - require.NotNil(t, w.merkleRoots) - - // Stop the first tree store so that it's databases are no longer used - require.NoError(t, ts.Stop()) + return ts +} - // declare a second TreeStore with no trees then load the first worldstate into it - ts2 := &treeStore{ - logger: logger.Global.CreateLoggerForModule(modules.TreeStoreSubmoduleName), - treeStoreDir: tmpDir, +func isEmpty(dir string) (bool, error) { + f, err := os.Open(dir) + if err != nil { + return false, err } - // TODO IN THIS COMMIT do we need to start this treestore? - // require.NoError(t, ts2.Start()) - - // Load sets a tree store to the provided worldstate - err = ts2.Load(w) - require.NoError(t, err) - - hash2 := ts2.getStateHash() + defer f.Close() - // Assert that hash is unchanged from save and load - require.Equal(t, hash1, hash2) + _, err = f.Readdirnames(1) // Or f.Readdir(1) + if err == io.EOF { + return true, nil + } + return false, err // Either not empty or error, suits both cases } diff --git a/persistence/trees/module.go b/persistence/trees/module.go index abb3ee306..2cce38ac4 100644 --- a/persistence/trees/module.go +++ b/persistence/trees/module.go @@ -2,16 +2,18 @@ package trees import ( "encoding/hex" + "errors" "fmt" "github.com/pokt-network/pocket/persistence/kvstore" "github.com/pokt-network/pocket/shared/modules" "github.com/pokt-network/smt" - "go.uber.org/multierr" ) var _ modules.TreeStoreModule = &treeStore{} +// Create returns a TreeStoreSubmodule that has been setup with the provided TreeStoreOptions, started, +// and then registered to the bus. func (*treeStore) Create(bus modules.Bus, options ...modules.TreeStoreOption) (modules.TreeStoreModule, error) { m := &treeStore{} @@ -19,6 +21,10 @@ func (*treeStore) Create(bus modules.Bus, options ...modules.TreeStoreOption) (m option(m) } + if err := m.Start(); err != nil { + return nil, fmt.Errorf("failed to start %s: %w", modules.TreeStoreSubmoduleName, err) + } + bus.RegisterModule(m) return m, nil @@ -66,11 +72,12 @@ func (t *treeStore) Stop() error { errs = append(errs, err) } } - return multierr.Combine(errs...) + return errors.Join(errs...) } func (t *treeStore) GetModuleName() string { return modules.TreeStoreSubmoduleName } +// setupTrees is called by Start and it loads the treestore at the given directory func (t *treeStore) setupTrees() error { if t.treeStoreDir == ":memory:" { return t.setupInMemory() diff --git a/persistence/trees/module_test.go b/persistence/trees/module_test.go index 6cfce7bac..ef66770fa 100644 --- a/persistence/trees/module_test.go +++ b/persistence/trees/module_test.go @@ -1,7 +1,6 @@ package trees_test import ( - "fmt" "testing" "github.com/golang/mock/gomock" @@ -9,22 +8,13 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/pokt-network/pocket/internal/testutil" "github.com/pokt-network/pocket/p2p/providers/peerstore_provider" "github.com/pokt-network/pocket/persistence/trees" "github.com/pokt-network/pocket/runtime" - "github.com/pokt-network/pocket/runtime/genesis" - "github.com/pokt-network/pocket/runtime/test_artifacts" - coreTypes "github.com/pokt-network/pocket/shared/core/types" - cryptoPocket "github.com/pokt-network/pocket/shared/crypto" "github.com/pokt-network/pocket/shared/modules" mockModules "github.com/pokt-network/pocket/shared/modules/mocks" ) -const ( - serviceURLFormat = "node%d.consensus:42069" -) - // DISCUSS: This is duplicated from inside trees package. Is it worth exporting or is it better as duplicate code? var stateTreeNames = []string{ "root", @@ -48,7 +38,12 @@ func TestTreeStore_Create(t *testing.T) { treemod, err := trees.Create(mockBus, trees.WithTreeStoreDirectory(":memory:")) assert.NoError(t, err) - require.NoError(t, treemod.Start()) + // Create should setup a value for each tree + for _, v := range stateTreeNames { + root, ns := treemod.GetTree(v) + require.NotEmpty(t, root) + require.NotEmpty(t, ns) + } got := treemod.GetBus() assert.Equal(t, got, mockBus) @@ -68,21 +63,13 @@ func TestTreeStore_StartAndStop(t *testing.T) { mockRuntimeMgr := mockModules.NewMockRuntimeMgr(ctrl) mockBus := createMockBus(t, mockRuntimeMgr) + // Create returns a started TreeStoreSubmodule treemod, err := trees.Create( mockBus, trees.WithTreeStoreDirectory(":memory:"), trees.WithLogger(&zerolog.Logger{})) assert.NoError(t, err) - // GetTree should return nil for each tree if Start has not been called - for _, v := range stateTreeNames { - root, ns := treemod.GetTree(v) - require.Empty(t, root) - require.Empty(t, ns) - } - // Should start without error - require.NoError(t, treemod.Start()) - // Should stop without error require.NoError(t, treemod.Stop()) @@ -99,78 +86,6 @@ func TestTreeStore_DebugClearAll(t *testing.T) { t.Skip("TODO: Write test case for DebugClearAll method") } -// createMockGenesisState configures and returns a mocked GenesisState -func createMockGenesisState(valKeys []cryptoPocket.PrivateKey) *genesis.GenesisState { - genesisState := new(genesis.GenesisState) - validators := make([]*coreTypes.Actor, len(valKeys)) - for i, valKey := range valKeys { - addr := valKey.Address().String() - mockActor := &coreTypes.Actor{ - ActorType: coreTypes.ActorType_ACTOR_TYPE_VAL, - Address: addr, - PublicKey: valKey.PublicKey().String(), - ServiceUrl: validatorId(i + 1), - StakedAmount: test_artifacts.DefaultStakeAmountString, - PausedHeight: int64(0), - UnstakingHeight: int64(0), - Output: addr, - } - validators[i] = mockActor - } - genesisState.Validators = validators - - return genesisState -} - -// Persistence mock - only needed for validatorMap access -func preparePersistenceMock(t *testing.T, busMock *mockModules.MockBus, genesisState *genesis.GenesisState) *mockModules.MockPersistenceModule { - ctrl := gomock.NewController(t) - - persistenceModuleMock := mockModules.NewMockPersistenceModule(ctrl) - readCtxMock := mockModules.NewMockPersistenceReadContext(ctrl) - - readCtxMock.EXPECT(). - GetAllValidators(gomock.Any()). - Return(genesisState.GetValidators(), nil).AnyTimes() - readCtxMock.EXPECT(). - GetAllStakedActors(gomock.Any()). - DoAndReturn(func(height int64) ([]*coreTypes.Actor, error) { - return testutil.Concatenate[*coreTypes.Actor]( - genesisState.GetValidators(), - genesisState.GetServicers(), - genesisState.GetFishermen(), - genesisState.GetApplications(), - ), nil - }). - AnyTimes() - persistenceModuleMock.EXPECT(). - NewReadContext(gomock.Any()). - Return(readCtxMock, nil). - AnyTimes() - readCtxMock.EXPECT(). - Release(). - AnyTimes() - persistenceModuleMock.EXPECT(). - GetBus(). - Return(busMock). - AnyTimes() - persistenceModuleMock.EXPECT(). - SetBus(busMock). - AnyTimes() - persistenceModuleMock.EXPECT(). - GetModuleName(). - Return(modules.PersistenceModuleName). - AnyTimes() - busMock. - RegisterModule(persistenceModuleMock) - - return persistenceModuleMock -} - -func validatorId(i int) string { - return fmt.Sprintf(serviceURLFormat, i) -} - // createMockBus returns a mock bus with stubbed out functions for bus registration func createMockBus(t *testing.T, runtimeMgr modules.RuntimeMgr) *mockModules.MockBus { t.Helper() diff --git a/persistence/trees/trees.go b/persistence/trees/trees.go index 6fdae98b8..87dcce0a2 100644 --- a/persistence/trees/trees.go +++ b/persistence/trees/trees.go @@ -15,9 +15,11 @@ package trees import ( "crypto/sha256" "encoding/hex" + "errors" "fmt" "hash" "log" + "path/filepath" "github.com/jackc/pgx/v5" "github.com/pokt-network/pocket/persistence/indexer" @@ -379,7 +381,10 @@ func (t *treeStore) save() (*worldState, error) { w := &worldState{ treeStoreDir: t.treeStoreDir, - merkleTrees: map[string]*stateTree{}, + merkleRoots: make(map[string][]byte), + merkleTrees: make(map[string]*stateTree), + rootHash: t.rootTree.tree.Root(), + rootTree: t.rootTree, } for treeName := range t.merkleTrees { @@ -403,6 +408,20 @@ func (t *treeStore) save() (*worldState, error) { return w, nil } +// Backup creates a new backup of each tree in the tree store to the provided directory. +// Each tree is backed up in an eponymous file in the provided backupDir. +func (t *treeStore) Backup(backupDir string) error { + errs := []error{} + for _, st := range t.merkleTrees { + treePath := filepath.Join(backupDir, st.name) + if err := st.nodeStore.Backup(treePath); err != nil { + t.logger.Err(err).Msgf("failed to backup %s tree: %+v", st.name, err) + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + //////////////////////// // Actor Tree Helpers // //////////////////////// diff --git a/persistence/trees/trees_test.go b/persistence/trees/trees_test.go index aa8c41ab4..cac314198 100644 --- a/persistence/trees/trees_test.go +++ b/persistence/trees/trees_test.go @@ -50,8 +50,6 @@ func TestTreeStore_Update(t *testing.T) { pmod := newTestPersistenceModule(t, dbUrl) context := newTestPostgresContext(t, 0, pmod) - require.NoError(t, context.SetSavePoint()) - hash1, err := context.ComputeStateHash() require.NoError(t, err) require.NotEmpty(t, hash1) @@ -60,8 +58,6 @@ func TestTreeStore_Update(t *testing.T) { _, err = createAndInsertDefaultTestApp(t, context) require.NoError(t, err) - require.NoError(t, context.SetSavePoint()) - hash2, err := context.ComputeStateHash() require.NoError(t, err) require.NotEmpty(t, hash2)