Cleanup

cmd/postcli/main.go

@@ -292,8 +292,8 @@ func cmdVerifyPos(opts config.InitOpts, fraction float64, logger *zap.Logger) {
 	}
 	pub := ed25519.NewKeyFromSeed(dst[:ed25519.SeedSize]).Public().(ed25519.PublicKey)
 
-	metafile := filepath.Join(opts.DataDir, shared.MetadataFileName)
-	meta, err := shared.LoadMetadata(opts.DataDir)
+	metafile := filepath.Join(opts.DataDir, initialization.MetadataFileName)
+	meta, err := initialization.LoadMetadata(opts.DataDir)
 	if err != nil {
 		log.Fatalf("failed to load metadata from %s: %s\n", opts.DataDir, err)
 	}

go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/davecgh/go-spew v1.1.1
 	github.com/golang/mock v1.6.0
+	github.com/natefinch/atomic v1.0.1
 	github.com/stretchr/testify v1.8.4
 	github.com/zeebo/blake3 v0.2.3
 	go.uber.org/zap v1.24.0

go.sum

@@ -16,6 +16,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A=
+github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

initialization/initialization.go

@@ -407,7 +407,7 @@ func removeRedundantFiles(cfg config.Config, opts config.InitOpts, logger *zap.L
 	for _, file := range files {
 		name := file.Name()
 		fileIndex, err := shared.ParseFileIndex(name)
-		if err != nil && name != shared.MetadataFileName {
+		if err != nil && name != MetadataFileName {
 			logger.Warn("found unrecognized file", zap.String("fileName", name))
 			continue
 		}
@@ -457,7 +457,7 @@ func (init *Initializer) Reset() error {
 			continue
 		}
 		name := file.Name()
-		if shared.IsInitFile(info) || name == shared.MetadataFileName {
+		if shared.IsInitFile(info) || name == MetadataFileName {
 			path := filepath.Join(init.opts.DataDir, name)
 			if err := os.Remove(path); err != nil {
 				return fmt.Errorf("failed to delete file (%v): %w", path, err)
@@ -689,10 +689,10 @@ func (init *Initializer) saveMetadata() error {
 	if init.nonceValue.Load() != nil {
 		v.NonceValue = *init.nonceValue.Load()
 	}
-	return shared.SaveMetadata(init.opts.DataDir, &v)
+	return SaveMetadata(init.opts.DataDir, &v)
 }
 
 func (init *Initializer) loadMetadata() (*shared.PostMetadata, error) {
 	// TODO(mafa): migrate metadata if needed before loading it
-	return shared.LoadMetadata(init.opts.DataDir)
+	return LoadMetadata(init.opts.DataDir)
 }

initialization/initialization_test.go

@@ -95,15 +95,15 @@ func TestInitialize_BeforeNonceValue(t *testing.T) {
 	cancel()
 	require.Equal(t, uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten())
 
-	meta, err := shared.LoadMetadata(opts.DataDir)
+	meta, err := LoadMetadata(opts.DataDir)
 	require.NoError(t, err)
 	require.NotNil(t, meta.Nonce)
 	require.NotNil(t, meta.NonceValue)
 	nonceValue := meta.NonceValue
 
 	// delete nonce value
 	meta.NonceValue = nil
-	require.NoError(t, shared.SaveMetadata(opts.DataDir, meta))
+	require.NoError(t, SaveMetadata(opts.DataDir, meta))
 
 	// just creating a new initializer should update the metadata
 	init, err = NewInitializer(
@@ -116,7 +116,7 @@ func TestInitialize_BeforeNonceValue(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, init)
 
-	meta, err = shared.LoadMetadata(opts.DataDir)
+	meta, err = LoadMetadata(opts.DataDir)
 	require.NoError(t, err)
 	require.NotNil(t, meta.Nonce)
 	require.NotNil(t, meta.NonceValue)
@@ -218,7 +218,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) {
 	r.NoError(init.Initialize(context.Background()))
 	r.Equal(uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten())
 
-	m, err := shared.LoadMetadata(opts.DataDir)
+	m, err := LoadMetadata(opts.DataDir)
 	r.NoError(err)
 	r.Equal(origNonce, *m.Nonce)
 	r.EqualValues(origNonceValue, m.NonceValue)
@@ -227,7 +227,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) {
 	// lastPos lower than numLabels is ignored
 	m.LastPosition = new(uint64)
 	*m.LastPosition = uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit - 10
-	r.NoError(shared.SaveMetadata(opts.DataDir, m))
+	r.NoError(SaveMetadata(opts.DataDir, m))
 
 	init, err = NewInitializer(
 		WithNodeId(nodeId),
@@ -241,7 +241,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) {
 	r.NoError(init.Initialize(context.Background()))
 	r.Equal(uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten())
 
-	m, err = shared.LoadMetadata(opts.DataDir)
+	m, err = LoadMetadata(opts.DataDir)
 	r.NoError(err)
 	r.Equal(origNonce, *m.Nonce)
 
@@ -251,7 +251,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) {
 	// the range of the PoST
 	m.Nonce = nil
 	m.LastPosition = nil
-	r.NoError(shared.SaveMetadata(opts.DataDir, m))
+	r.NoError(SaveMetadata(opts.DataDir, m))
 
 	init, err = NewInitializer(
 		WithNodeId(nodeId),
@@ -265,7 +265,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) {
 	r.NoError(init.Initialize(context.Background()))
 	r.Equal(uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten())
 
-	m, err = shared.LoadMetadata(opts.DataDir)
+	m, err = LoadMetadata(opts.DataDir)
 	r.NoError(err)
 	r.NotNil(m.Nonce)
 	r.NotNil(m.NonceValue)
@@ -285,7 +285,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) {
 	lastPos := *m.Nonce + 10
 	*m.LastPosition = lastPos
 	m.Nonce = nil
-	r.NoError(shared.SaveMetadata(opts.DataDir, m))
+	r.NoError(SaveMetadata(opts.DataDir, m))
 
 	init, err = NewInitializer(
 		WithNodeId(nodeId),
@@ -299,7 +299,7 @@ func TestInitialize_ContinueWithLastPos(t *testing.T) {
 	r.NoError(init.Initialize(context.Background()))
 	r.Equal(uint64(cfg.MinNumUnits)*cfg.LabelsPerUnit, init.NumLabelsWritten())
 
-	m, err = shared.LoadMetadata(opts.DataDir)
+	m, err = LoadMetadata(opts.DataDir)
 	r.NoError(err)
 	r.NotNil(m.Nonce)
 	r.NotNil(m.LastPosition)
@@ -1073,7 +1073,7 @@ func TestInitializeSubset_NoNonce(t *testing.T) {
 	require.Nil(t, init.Nonce())
 	require.Nil(t, init.NonceValue())
 
-	meta, err := shared.LoadMetadata(opts.DataDir)
+	meta, err := LoadMetadata(opts.DataDir)
 	require.NoError(t, err)
 	require.Nil(t, meta.Nonce)
 

initialization/metadata.go

@@ -0,0 +1,49 @@
+package initialization
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/spacemeshos/post/shared"
+)
+
+const MetadataFileName = "postdata_metadata.json"
+
+func SaveMetadata(dir string, v *shared.PostMetadata) error {
+	err := os.MkdirAll(dir, shared.OwnerReadWriteExec)
+	if err != nil && !os.IsExist(err) {
+		return fmt.Errorf("dir creation failure: %w", err)
+	}
+
+	data, err := json.Marshal(v)
+	if err != nil {
+		return fmt.Errorf("serialization failure: %w", err)
+	}
+
+	err = os.WriteFile(filepath.Join(dir, MetadataFileName), data, shared.OwnerReadWrite)
+	if err != nil {
+		return fmt.Errorf("write to disk failure: %w", err)
+	}
+
+	return nil
+}
+
+func LoadMetadata(dir string) (*shared.PostMetadata, error) {
+	filename := filepath.Join(dir, MetadataFileName)
+	data, err := os.ReadFile(filename)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, shared.ErrStateMetadataFileMissing
+		}
+		return nil, fmt.Errorf("read file failure: %w", err)
+	}
+
+	metadata := shared.PostMetadata{}
+	if err := json.Unmarshal(data, &metadata); err != nil {
+		return nil, err
+	}
+
+	return &metadata, nil
+}

initialization/migrate_metadata.go

@@ -29,7 +29,7 @@ type MetadataVersion struct {
 func MigratePoST(dir string, logger *zap.Logger) (err error) {
 	logger.Info("checking PoST for migrations")
 
-	filename := filepath.Join(dir, shared.MetadataFileName)
+	filename := filepath.Join(dir, MetadataFileName)
 	file, err := os.Open(filename)
 	switch {
 	case os.IsNotExist(err):
@@ -85,7 +85,7 @@ type postMetadataV0 struct {
 // - add NonceValue field to postdata_metadata.json if missing (was introduced before migrations, not every PoST version 0 metadata file has it)
 // - re-encode NodeId and CommitmentAtxId as hex strings.
 func migrateV0(dir string, logger *zap.Logger) (err error) {
-	filename := filepath.Join(dir, shared.MetadataFileName)
+	filename := filepath.Join(dir, MetadataFileName)
 	file, err := os.Open(filename)
 	switch {
 	case os.IsNotExist(err):

initialization/vrf_search.go

@@ -54,7 +54,7 @@ func SearchForNonce(ctx context.Context, cfg Config, initOpts InitOpts, opts ...
 	}
 	logger := options.logger
 
-	metadata, err := shared.LoadMetadata(initOpts.DataDir)
+	metadata, err := LoadMetadata(initOpts.DataDir)
 	if err != nil {
 		return 0, nil, fmt.Errorf("failed to load metadata: %w", err)
 	}
@@ -145,7 +145,7 @@ func persistNonce(nonce uint64, label []byte, metadata *shared.PostMetadata, dat
 	logger.Info("found nonce: updating postdata_metadata.json", zap.Uint64("nonce", nonce), zap.String("NonceValue", hex.EncodeToString(label)))
 	metadata.Nonce = &nonce
 	metadata.NonceValue = shared.NonceValue(label)
-	if err := shared.SaveMetadata(datadir, metadata); err != nil {
+	if err := SaveMetadata(datadir, metadata); err != nil {
 		return fmt.Errorf("failed to save metadata: %w", err)
 	}
 	return nil

initialization/vrf_search_test.go

@@ -93,15 +93,15 @@ func TestSearchForNonce(t *testing.T) {
 	err = init.Initialize(context.Background())
 	require.NoError(t, err)
 
-	metadata, err := shared.LoadMetadata(opts.DataDir)
+	metadata, err := LoadMetadata(opts.DataDir)
 	require.NoError(t, err)
 
 	expectedNonce := metadata.Nonce
 	expectedNonceValue := metadata.NonceValue
 	// remove Nonce and NonceValue from metadata
 	metadata.Nonce = nil
 	metadata.NonceValue = nil
-	err = shared.SaveMetadata(opts.DataDir, metadata)
+	err = SaveMetadata(opts.DataDir, metadata)
 	require.NoError(t, err)
 
 	nonce, value, err := SearchForNonce(
@@ -115,7 +115,7 @@ func TestSearchForNonce(t *testing.T) {
 	require.EqualValues(t, expectedNonceValue, value)
 
 	// Verify that nonce was written to the metatada file
-	metadata, err = shared.LoadMetadata(opts.DataDir)
+	metadata, err = LoadMetadata(opts.DataDir)
 	require.NoError(t, err)
 	require.Equal(t, expectedNonce, metadata.Nonce)
 	require.EqualValues(t, expectedNonceValue, metadata.NonceValue)

proving/proving_options.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 
 	"github.com/spacemeshos/post/config"
+	"github.com/spacemeshos/post/initialization"
 	"github.com/spacemeshos/post/shared"
 )
 
@@ -38,7 +39,7 @@ type OptionFunc func(*option) error
 // WithDataSource sets the data source to use for the proof.
 func WithDataSource(cfg config.Config, nodeId, commitmentAtxId []byte, datadir string) OptionFunc {
 	return func(o *option) error {
-		m, err := shared.LoadMetadata(datadir)
+		m, err := initialization.LoadMetadata(datadir)
 		if err != nil {
 			return err
 		}

shared/post_metadata.go

@@ -4,9 +4,6 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"errors"
-	"fmt"
-	"os"
-	"path/filepath"
 )
 
 // ErrStateMetadataFileMissing is returned when the metadata file is missing.
@@ -90,42 +87,3 @@ func (a *ATXID) UnmarshalJSON(data []byte) (err error) {
 	*a, err = hex.DecodeString(hexString)
 	return
 }
-
-const MetadataFileName = "postdata_metadata.json"
-
-func SaveMetadata(dir string, v *PostMetadata) error {
-	err := os.MkdirAll(dir, OwnerReadWriteExec)
-	if err != nil && !os.IsExist(err) {
-		return fmt.Errorf("dir creation failure: %w", err)
-	}
-
-	data, err := json.Marshal(v)
-	if err != nil {
-		return fmt.Errorf("serialization failure: %w", err)
-	}
-
-	err = os.WriteFile(filepath.Join(dir, MetadataFileName), data, OwnerReadWrite)
-	if err != nil {
-		return fmt.Errorf("write to disk failure: %w", err)
-	}
-
-	return nil
-}
-
-func LoadMetadata(dir string) (*PostMetadata, error) {
-	filename := filepath.Join(dir, MetadataFileName)
-	data, err := os.ReadFile(filename)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, ErrStateMetadataFileMissing
-		}
-		return nil, fmt.Errorf("read file failure: %w", err)
-	}
-
-	metadata := PostMetadata{}
-	if err := json.Unmarshal(data, &metadata); err != nil {
-		return nil, err
-	}
-
-	return &metadata, nil
-}