From f03e960626806f05f19abea980f8d5c7f447d696 Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Thu, 7 Mar 2024 10:46:15 -0500 Subject: [PATCH] update docs and examples --- .env.example | 22 ++++++++++++++++++---- README.md | 42 +++++++++++++++++++++++++++--------------- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/.env.example b/.env.example index 9d5523a..dd6c889 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,20 @@ +# Substreams Credentials +# https://app.pinax.network +# https://app.streamingfast.io SUBSTREAMS_API_KEY= -MANIFEST=https://github.com/streamingfast/substreams-eth-block-meta/releases/download/v0.5.1/substreams-eth-block-meta-v0.5.1.spkg -MODULE_NAME=graph_out SUBSTREAMS_ENDPOINT=eth.substreams.pinax.network:443 -SCHEMA=schema.example.sql -DELIMITER="," \ No newline at end of file + +# Substreams Package +MANIFEST=https://spkg.io/streamingfast/substreams-eth-block-meta-v0.4.3.spkg +MODULE_NAME=graph_out + +# Substreams Package (Optional) +START_BLOCK=2 +FINAL_BLOCKS_ONLY=true + +# CSV Input +SCHEMA=schema.example.block_meta.sql + +# CSV Output (Optional) +DELIMITER="," +FILENAME="data.csv" diff --git a/README.md b/README.md index 60673ac..f0fc0a3 100644 --- a/README.md +++ b/README.md @@ -44,14 +44,26 @@ CREATE TABLE block_meta **.env** ```env +# Substreams Credentials +# https://app.pinax.network +# https://app.streamingfast.io SUBSTREAMS_API_KEY= -MANIFEST=https://github.com/streamingfast/substreams-eth-block-meta/releases/download/v0.5.1/substreams-eth-block-meta-v0.5.1.spkg -MODULE_NAME=graph_out SUBSTREAMS_ENDPOINT=eth.substreams.pinax.network:443 -SCHEMA=schema.example.sql + +# Substreams Package +MANIFEST=https://spkg.io/streamingfast/substreams-eth-block-meta-v0.4.3.spkg +MODULE_NAME=graph_out + +# Substreams Package (Optional) +START_BLOCK=-7200 FINAL_BLOCKS_ONLY=true -START_BLOCK=2 + +# CSV Input +SCHEMA=schema.example.block_meta.sql + +# CSV Output (Optional) DELIMITER="," +FILENAME="data.csv" ``` **CLI** with `.env` file ```bash @@ -59,7 +71,12 @@ $ substreams-sink-csv ``` **CLI** with `params` ```bash -$ substreams-sink-csv --schema schema.example.sql -e eth.substreams.pinax.network:443 --module-name graph_out --manifest https://github.com/streamingfast/substreams-eth-block-meta/releases/download/v0.5.1/substreams-eth-block-meta-v0.5.1.spkg --substreams-api-key +$ substreams-sink-csv \ + -e eth.substreams.pinax.network:443 \ + --schema schema.example.block_meta.sql \ + --module-name graph_out \ + --manifest https://spkg.io/streamingfast/substreams-eth-block-meta-v0.4.3.spkg \ + --substreams-api-key ``` ### Substreams Support @@ -71,13 +88,13 @@ $ substreams-sink-csv --schema schema.example.sql -e eth.substreams.pinax.networ ### CSV filename schema -The CSV filename is generated using the following pattern: +If `FILENAME` is not provided, the CSV output filename is generated using the following pattern: ```yml ---.csv ``` -Additionally, `*.clock` & `*.cursor` files are generated to keep track of the last block processed. +Additionally, `*.clock`, `*.session` & `*.cursor` files are generated to keep track of the last block processed. **Example:** @@ -133,12 +150,7 @@ module.exports = { script: "./node_modules/substreams-sink-csv/dist/bin/cli.mjs", env: { SUBSTREAMS_API_KEY: '', - MANIFEST: 'https://github.com/streamingfast/substreams-eth-block-meta/releases/download/v0.5.1/substreams-eth-block-meta-v0.5.1.spkg', - MODULE_NAME: 'graph_out', - SUBSTREAMS_ENDPOINT: 'eth.substreams.pinax.network:443', - FINAL_BLOCKS_ONLY: 'true', - START_BLOCK: '2', - SCHEMA: 'schema.example.sql', + ... } }] } @@ -187,14 +199,14 @@ ORDER BY block_num; > To import data from the CSV file to the `block_meta` table, we can pipe our file directly to the clickhouse-client: ```bash -$ clickhouse-client --query="INSERT INTO block_meta FORMAT CSV" < eth.substreams.pinax.network-3b180e1d2390afef1f22651581304e04245ba001-graph_out-block_meta.csv +$ clickhouse-client --query="INSERT INTO block_meta FORMAT CSV" < data-block_meta.csv ``` > Note that we use `FORMAT CSV` to let ClickHouse know we’re ingesting CSV formatted data. Alternatively, we can load data from a local file using the `FROM INFILE` clause: ```sql INSERT INTO block_meta -FROM INFILE 'eth.substreams.pinax.network-3b180e1d2390afef1f22651581304e04245ba001-graph_out-block_meta.csv' +FROM INFILE 'data-block_meta.csv' FORMAT CSV ```