Merge pull request #35 from streamingfast/feature/sol-poller

Solana block. polling integration

.github/workflows/docker.yml

@@ -22,7 +22,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [ 1.20.x ]
+        go-version: [ 1.22.x ]
 
     outputs:
       tags: ${{ steps.meta.outputs.tags }}

CHANGELOG.md

@@ -8,11 +8,25 @@ Operators, you should copy/paste content of this content straight to your projec
 
 If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you should copy the content between those 2 version to your own repository, replacing placeholder value `fire{chain}` with your chain's own binary.
 
-## Unreleased
+## UNRELEASED
 
-* add 'x-trace-id' header to auth requests when using --common-auth-plugin=grpc
-* fix Substreams scheduler sometimes taking a long time to spawn more than a single worker.
-* add ACCEPT_SOLANA_LEGACY_BLOCK_FORMAT env var to allow special tweak operations
+* Poller is now fetching blocks in an optimized way, it will fetch several blocks at once and then process them.
+
+* Poller is now handling skipped blocks, it will fetch the next blocks until it find a none skipped block.
+
+* Poller now has default retry value of infinite.
+
+* Compare tool is now using dynamic protobuf unmarshaler, it will be able to compare any block type.
+
+* Print tool is now using dynamic protobuf unmarshaler, it will be able to print any block type.
+
+* Print tool is encoding bytes in base64 by default, it can be changed to hex or base58 by using parameter `bytes-encoding`.
+
+* Added 'x-trace-id' header to auth requests when using --common-auth-plugin=grpc
+
+* Fixed Substreams scheduler sometimes taking a long time to spawn more than a single worker.
+
+* Added ACCEPT_SOLANA_LEGACY_BLOCK_FORMAT env var to allow special tweak operations
 
 ## v1.1.3
 
@@ -34,6 +48,10 @@ If you were at `firehose-core` version `1.0.0` and are bumping to `1.1.0`, you s
 
 * The `tools print one-block` now works correctly on blocks generated by omni-chain `firecore` binary.
 
+* Tools printing Firehose `Block` model to JSON now have `--proto-paths` take higher precedence over well-known types and even the chain itself, the order is `--proto-paths` > `chain` > `well-known` (so `well-known` is lookup last).
+
+* The `tools print one-block` now works correctly on blocks generated by omni-chain `firecore` binary.
+
 * The various health endpoint now sets `Content-Type: application/json` header prior sending back their response to the client.
 
 * The `firehose`, `substreams-tier1` and `substream-tier2` health endpoint now respects the `common-system-shutdown-signal-delay` configuration value meaning that the health endpoint will return `false` now if `SIGINT` has been received but we are still in the shutdown unready period defined by the config value. If you use some sort of load balancer, you should make sure they are configured to use the health endpoint and you should `common-system-shutdown-signal-delay` to something like `15s`.

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21-alpine as build
+FROM golang:1.22-alpine as build
 WORKDIR /app
 
 COPY go.mod go.sum ./

blockpoller/fetcher.go

@@ -7,5 +7,6 @@ import (
 )
 
 type BlockFetcher interface {
-	Fetch(ctx context.Context, blkNum uint64) (*pbbstream.Block, error)
+	IsBlockAvailable(requestedSlot uint64) bool
+	Fetch(ctx context.Context, blkNum uint64) (b *pbbstream.Block, skipped bool, err error)
 }

blockpoller/init_test.go

@@ -47,13 +47,17 @@ func (b *TestBlockFetcher) PollingInterval() time.Duration {
 	return 0
 }
 
-func (b *TestBlockFetcher) Fetch(_ context.Context, blkNum uint64) (*pbbstream.Block, error) {
+func (b *TestBlockFetcher) IsBlockAvailable(requestedSlot uint64) bool {
+	return true
+}
+
+func (b *TestBlockFetcher) Fetch(_ context.Context, blkNum uint64) (*pbbstream.Block, bool, error) {
 	if len(b.blocks) == 0 {
 		assert.Fail(b.t, fmt.Sprintf("should not have fetched block %d", blkNum))
 	}
 
 	if b.idx >= uint64(len(b.blocks)) {
-		return nil, derr.NewFatalError(TestErrCompleteDone)
+		return nil, false, derr.NewFatalError(TestErrCompleteDone)
 	}
 
 	if blkNum != b.blocks[b.idx].expect.Number {
@@ -62,7 +66,7 @@ func (b *TestBlockFetcher) Fetch(_ context.Context, blkNum uint64) (*pbbstream.B
 
 	blkToSend := b.blocks[b.idx].send
 	b.idx++
-	return blkToSend, nil
+	return blkToSend, false, nil
 }
 
 func (b *TestBlockFetcher) check(t *testing.T) {