From 1eadea5cb2ae45361e0f20c78a17bb169780356a Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Tue, 31 Jan 2023 15:57:07 -0500 Subject: [PATCH] Update substreams - add default host (optional) - set default startblock - improve stopBlock handling - update example using ETH --- README.md | 25 +++++++++++-------------- example.js | 27 ++++++++++++--------------- examples/subtivity.ts | 35 +++++++++++++++++++++++++++++++++++ package.json | 2 +- src/index.ts | 31 +++++++++++-------------------- src/utils.spec.ts | 7 ++++++- src/utils.ts | 7 +++++++ 7 files changed, 83 insertions(+), 51 deletions(-) create mode 100644 examples/subtivity.ts diff --git a/README.md b/README.md index 82ae833..ae41f37 100644 --- a/README.md +++ b/README.md @@ -35,32 +35,29 @@ yarn add substreams const { Substreams, download } = require("substreams"); // User input -const host = ""; -const substream = "https://ipfs.pinax.network/ipfs/QmfE7kdRAPihhvij4ej3rUM2Sp3PcXQ9rTFCQPhPGB5dr5"; -const outputModules = ["map_action_traces"]; -const startBlockNum = "283000000"; -const stopBlockNum = "283001000"; +const spkg = "https://github.com/pinax-network/subtivity-substreams/releases/download/v0.1.0/subtivity-ethereum-v0.1.0.spkg"; +const outputModule = "map_block_stats"; +const startBlockNum = "300000"; +const stopBlockNum = "+10"; // Initialize Substreams -const substreams = new Substreams(host, { +const substreams = new Substreams(outputModule, { startBlockNum, stopBlockNum, - outputModules, + authorization: process.env.SUBSTREAMS_API_TOKEN }); (async () => { // download Substream from IPFS - const {modules, registry} = await download(substream); + const {modules, registry} = await download(spkg); // Find Protobuf message types from registry - const ActionTraces = registry.findMessage("sf.antelope.type.v1.ActionTraces"); - if ( !ActionTraces) throw new Error("Could not find ActionTraces message type"); + const BlockStats = registry.findMessage("subtivity.v1.BlockStats"); + if ( !BlockStats) throw new Error("Could not find BlockStats message type"); substreams.on("mapOutput", output => { - const { actionTraces } = ActionTraces.fromBinary(output.data.mapOutput.value); - for ( const actionTrace of actionTraces ) { - console.log(actionTrace); - } + const decoded = BlockStats.fromBinary(output.data.mapOutput.value); + console.log(decoded); }); // start streaming Substream diff --git a/example.js b/example.js index e6db972..e0c52f0 100644 --- a/example.js +++ b/example.js @@ -1,32 +1,29 @@ -const { Substreams, download } = require("substreams"); +const { Substreams, download } = require("./"); // User input -const host = ""; -const substream = "https://ipfs.pinax.network/ipfs/QmfE7kdRAPihhvij4ej3rUM2Sp3PcXQ9rTFCQPhPGB5dr5"; -const outputModules = ["map_action_traces"]; -const startBlockNum = "283000000"; -const stopBlockNum = "283001000"; +const spkg = "https://github.com/pinax-network/subtivity-substreams/releases/download/v0.1.0/subtivity-ethereum-v0.1.0.spkg"; +const outputModule = "map_block_stats"; +const startBlockNum = "300000"; +const stopBlockNum = "+10"; // Initialize Substreams -const substreams = new Substreams(host, { +const substreams = new Substreams(outputModule, { startBlockNum, stopBlockNum, - outputModules, + authorization: process.env.SUBSTREAMS_API_TOKEN }); (async () => { // download Substream from IPFS - const {modules, registry} = await download(substream); + const {modules, registry} = await download(spkg); // Find Protobuf message types from registry - const ActionTraces = registry.findMessage("sf.antelope.type.v1.ActionTraces"); - if ( !ActionTraces) throw new Error("Could not find ActionTraces message type"); + const BlockStats = registry.findMessage("subtivity.v1.BlockStats"); + if ( !BlockStats) throw new Error("Could not find BlockStats message type"); substreams.on("mapOutput", output => { - const { actionTraces } = ActionTraces.fromBinary(output.data.mapOutput.value); - for ( const actionTrace of actionTraces ) { - console.log(actionTrace); - } + const decoded = BlockStats.fromBinary(output.data.mapOutput.value); + console.log(decoded); }); // start streaming Substream diff --git a/examples/subtivity.ts b/examples/subtivity.ts new file mode 100644 index 0000000..5dc0417 --- /dev/null +++ b/examples/subtivity.ts @@ -0,0 +1,35 @@ +import { Substreams, download } from "../src"; + +// User input +const spkg = "https://github.com/pinax-network/subtivity-substreams/releases/download/v0.1.0/subtivity-ethereum-v0.1.0.spkg"; +const outputModule = "map_block_stats"; +const startBlockNum = "300000"; +const stopBlockNum = "+10"; + +// Initialize Substreams +const substreams = new Substreams(outputModule, { + startBlockNum, + stopBlockNum, + authorization: process.env.SUBSTREAMS_API_TOKEN +}); + +(async () => { + // download Substream from IPFS + const {modules, registry} = await download(spkg); + + // Find Protobuf message types from registry + const BlockStats = registry.findMessage("subtivity.v1.BlockStats"); + if ( !BlockStats) throw new Error("Could not find BlockStats message type"); + + substreams.on("mapOutput", output => { + const decoded = BlockStats.fromBinary(output.data.mapOutput.value); + console.log(decoded); + }); + + // start streaming Substream + await substreams.start(modules); + + // end of Substream + console.log("done"); + process.exit(); +})(); \ No newline at end of file diff --git a/package.json b/package.json index 0aefe71..44f0b90 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "substreams", "description": "Substreams Javascript consumer", - "version": "0.2.1", + "version": "0.3.0", "homepage": "https://github.com/pinax-network/substreams-js", "main": "dist/index.js", "types": "dist/index.d.js", diff --git a/src/index.ts b/src/index.ts index 5d5b5d6..9870fd2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,7 +20,7 @@ export * from "./generated/sf/substreams/v1/substreams" export * from "./utils"; // Utils -import { parseBlockData } from './utils'; +import { parseBlockData, parseStopBlock } from './utils'; interface MapOutput extends ModuleOutput { data: { @@ -48,21 +48,22 @@ export class Substreams extends (EventEmitter as new () => TypedEmitter TypedEmitter TypedEmitter TypedEmitter TypedEmitter { it("formatDate", () => { @@ -13,6 +13,11 @@ describe('substreams', () => { assert.equal(isIpfs("QmUatvHNjq696qkB8SBz5VBytcEeTrM1VwFyy4Rt4Z43mX"), true); assert.equal(isIpfs("not IPFS"), false); }); + + it("parseStopBlock", async () => { + assert.equal(parseStopBlock("0", "+100"), "100"); + assert.equal(parseStopBlock("400", "+100"), "500"); + }); }); /** diff --git a/src/utils.ts b/src/utils.ts index 1072bc7..393537b 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -31,6 +31,13 @@ export function getSeconds( clock?: Clock ) { export const isIpfs = ( str: string ) => /^Qm[1-9A-Za-z]{44}$/.test(str); +export function parseStopBlock( startBlock: string, stopBlock?: string ) { + if (!stopBlock) return; + if ( stopBlock.includes("+")) return String(Number(startBlock) + Number(stopBlock)); + if ( stopBlock.includes("-")) throw new Error(`stopBlock cannot be negative: ${stopBlock}`); + return stopBlock; +} + export async function download( url: string ) { if ( isIpfs(url) ) url = `https://ipfs.pinax.network/ipfs/${url}`; const binary = await downloadBuffer(url);