Fix block zero; load genesis file and use it for getGenesisHash and block 0 timestamp.

cmd-rpc.go

@@ -153,6 +153,12 @@ func newCmd_rpc() *cli.Command {
 				EpochSearchConcurrency: epochSearchConcurrency,
 			})
 
+			defer func() {
+				if err := multi.Close(); err != nil {
+					klog.Errorf("error closing multi-epoch: %s", err.Error())
+				}
+			}()
+
 			for _, epoch := range epochs {
 				if err := multi.AddEpoch(epoch.Epoch(), epoch); err != nil {
 					return cli.Exit(fmt.Sprintf("failed to add epoch %d: %s", epoch.Epoch(), err.Error()), 1)

cmd-x-index-all.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"math/rand"
@@ -211,7 +212,7 @@ func createAllIndexes(
 	for {
 		_cid, sectionLength, block, err := rd.NextNode()
 		if err != nil {
-			if err == io.EOF {
+			if errors.Is(err, io.EOF) {
 				break
 			}
 			return nil, err
@@ -611,7 +612,7 @@ func verifyAllIndexes(
 	for {
 		_cid, sectionLength, block, err := rd.NextNode()
 		if err != nil {
-			if err == io.EOF {
+			if errors.Is(err, io.EOF) {
 				break
 			}
 			return err

config.go

@@ -125,6 +125,9 @@ type Config struct {
 			URI URI `json:"uri" yaml:"uri"`
 		} `json:"sig_exists" yaml:"sig_exists"`
 	} `json:"indexes" yaml:"indexes"`
+	Genesis struct {
+		URI URI `json:"uri" yaml:"uri"`
+	} `json:"genesis" yaml:"genesis"`
 }
 
 func (c *Config) ConfigFilepath() string {
@@ -296,5 +299,20 @@ func (c *Config) Validate() error {
 			}
 		}
 	}
+	{
+		// if epoch is 0, then the genesis URI must be set:
+		if *c.Epoch == 0 {
+			if c.Genesis.URI.IsZero() {
+				return fmt.Errorf("epoch is 0, but genesis.uri is not set")
+			}
+			if !c.Genesis.URI.IsValid() {
+				return fmt.Errorf("genesis.uri is invalid")
+			}
+			// only support local genesis files for now:
+			if !c.Genesis.URI.IsLocal() {
+				return fmt.Errorf("genesis.uri must be a local file")
+			}
+		}
+	}
 	return nil
 }

epoch.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"bytes"
 	"context"
 	"crypto/rand"
 	"encoding/binary"
@@ -12,6 +13,7 @@ import (
 
 	"github.com/gagliardetto/solana-go"
 	"github.com/ipfs/go-cid"
+	carv1 "github.com/ipld/go-car"
 	"github.com/ipld/go-car/util"
 	carv2 "github.com/ipld/go-car/v2"
 	"github.com/libp2p/go-libp2p/core/peer"
@@ -22,6 +24,7 @@ import (
 	"github.com/rpcpool/yellowstone-faithful/gsfa"
 	"github.com/rpcpool/yellowstone-faithful/ipld/ipldbindcode"
 	"github.com/rpcpool/yellowstone-faithful/iplddecoders"
+	"github.com/rpcpool/yellowstone-faithful/radiance/genesis"
 	"github.com/urfave/cli/v2"
 	"k8s.io/klog/v2"
 )
@@ -30,20 +33,22 @@ type Epoch struct {
 	epoch          uint64
 	isFilecoinMode bool // true if the epoch is in Filecoin mode (i.e. Lassie mode)
 	config         *Config
+	// genesis:
+	genesis *GenesisContainer
 	// contains indexes and block data for the epoch
-	lassieFetcher       *lassieWrapper
-	localCarReader      *carv2.Reader
-	remoteCarReader     ReaderAtCloser
-	remoteCarHeaderSize uint64
-	cidToOffsetIndex    *compactindex.DB
-	slotToCidIndex      *compactindex36.DB
-	sigToCidIndex       *compactindex36.DB
-	sigExists           *bucketteer.Reader
-	gsfaReader          *gsfa.GsfaReader
-	cidToNodeCache      *cache.Cache // TODO: prevent OOM
-	onClose             []func() error
-	slotToCidCache      *cache.Cache
-	cidToOffsetCache    *cache.Cache
+	lassieFetcher    *lassieWrapper
+	localCarReader   *carv2.Reader
+	remoteCarReader  ReaderAtCloser
+	carHeaderSize    uint64
+	cidToOffsetIndex *compactindex.DB
+	slotToCidIndex   *compactindex36.DB
+	sigToCidIndex    *compactindex36.DB
+	sigExists        *bucketteer.Reader
+	gsfaReader       *gsfa.GsfaReader
+	cidToNodeCache   *cache.Cache // TODO: prevent OOM
+	onClose          []func() error
+	slotToCidCache   *cache.Cache
+	cidToOffsetCache *cache.Cache
 }
 
 func (r *Epoch) getSlotToCidFromCache(slot uint64) (cid.Cid, error, bool) {
@@ -92,6 +97,10 @@ func (e *Epoch) Close() error {
 	return errors.Join(multiErr...)
 }
 
+func (e *Epoch) GetGenesis() *GenesisContainer {
+	return e.genesis
+}
+
 func NewEpochFromConfig(config *Config, c *cli.Context) (*Epoch, error) {
 	if config == nil {
 		return nil, fmt.Errorf("config must not be nil")
@@ -105,6 +114,19 @@ func NewEpochFromConfig(config *Config, c *cli.Context) (*Epoch, error) {
 		config:         config,
 		onClose:        make([]func() error, 0),
 	}
+	{
+		// if epoch is 0, then try loading the genesis from the config:
+		if *config.Epoch == 0 {
+			genesisConfig, ha, err := genesis.ReadGenesisFromFile(string(config.Genesis.URI))
+			if err != nil {
+				return nil, fmt.Errorf("failed to read genesis: %w", err)
+			}
+			ep.genesis = &GenesisContainer{
+				Hash:   solana.HashFromBytes(ha[:]),
+				Config: genesisConfig,
+			}
+		}
+	}
 
 	if isCarMode {
 		// The CAR-mode requires a cid-to-offset index.
@@ -207,7 +229,7 @@ func NewEpochFromConfig(config *Config, c *cli.Context) (*Epoch, error) {
 		ep.localCarReader = localCarReader
 		ep.remoteCarReader = remoteCarReader
 		if remoteCarReader != nil {
-			// read 10 bytes from the CAR file to get the header size
+			// determine the header size so that we know where the data starts:
 			headerSizeBuf, err := readSectionFromReaderAt(remoteCarReader, 0, 10)
 			if err != nil {
 				return nil, fmt.Errorf("failed to read CAR header: %w", err)
@@ -217,7 +239,24 @@ func NewEpochFromConfig(config *Config, c *cli.Context) (*Epoch, error) {
 			if n <= 0 {
 				return nil, fmt.Errorf("failed to decode CAR header size")
 			}
-			ep.remoteCarHeaderSize = uint64(n) + headerSize
+			ep.carHeaderSize = uint64(n) + headerSize
+		}
+		if localCarReader != nil {
+			// determine the header size so that we know where the data starts:
+			dr, err := localCarReader.DataReader()
+			if err != nil {
+				return nil, fmt.Errorf("failed to get local CAR data reader: %w", err)
+			}
+			header, err := readHeader(dr)
+			if err != nil {
+				return nil, fmt.Errorf("failed to read local CAR header: %w", err)
+			}
+			var buf bytes.Buffer
+			if err = carv1.WriteHeader(header, &buf); err != nil {
+				return nil, fmt.Errorf("failed to encode local CAR header: %w", err)
+			}
+			headerSize := uint64(buf.Len())
+			ep.carHeaderSize = headerSize
 		}
 	}
 	{

genesis.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"github.com/gagliardetto/solana-go"
+	"github.com/rpcpool/yellowstone-faithful/radiance/genesis"
+)
+
+type GenesisContainer struct {
+	Hash solana.Hash
+	// The genesis config.
+	Config *genesis.Genesis
+}

gsfa/manifest/manifest.go

@@ -2,6 +2,7 @@ package manifest
 
 import (
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -213,7 +214,7 @@ func (m *Manifest) readAllContent() (Values, error) {
 	values := make([][2]uint64, 0, currentContentSize/16)
 	for {
 		_, err := io.ReadFull(sectionReader, buf)
-		if err == io.EOF {
+		if errors.Is(err, io.EOF) {
 			break
 		}
 		if err != nil {

index-cid-to-offset.go

@@ -89,7 +89,7 @@ func CreateIndex_cid2offset(ctx context.Context, tmpDir string, carPath string,
 	for {
 		c, sectionLength, err := rd.NextInfo()
 		if err != nil {
-			if err == io.EOF {
+			if errors.Is(err, io.EOF) {
 				break
 			}
 			return "", err