From ba4b653a77e6eccae7d0dd4c78795f7191333064 Mon Sep 17 00:00:00 2001 From: YaroShkvorets Date: Thu, 25 Jul 2024 19:00:36 -0400 Subject: [PATCH 1/4] add extra index --- antelope-common/Cargo.lock | 2 +- antelope-common/Cargo.toml | 2 +- antelope-common/src/actions.rs | 17 +++++++++- antelope-common/src/index.rs | 15 ++++++++- antelope-common/src/maps.rs | 46 ++++++++++++++++++++++++++- antelope-common/substreams.yaml | 55 ++++++++++++++++++++++++++++++--- 6 files changed, 128 insertions(+), 9 deletions(-) diff --git a/antelope-common/Cargo.lock b/antelope-common/Cargo.lock index 315dd21..b6802ce 100644 --- a/antelope-common/Cargo.lock +++ b/antelope-common/Cargo.lock @@ -13,7 +13,7 @@ dependencies = [ [[package]] name = "antelope-common" -version = "0.3.0" +version = "0.4.0" dependencies = [ "serde_json", "substreams", diff --git a/antelope-common/Cargo.toml b/antelope-common/Cargo.toml index 6ab8949..facfa2c 100644 --- a/antelope-common/Cargo.toml +++ b/antelope-common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "antelope-common" -version = "0.3.0" +version = "0.4.0" description = "Foundational Substreams Modules for Antelope Chains" edition = "2021" repository = "https://github.com/streamingfast/substreams-foundational-modules/antelope-common" diff --git a/antelope-common/src/actions.rs b/antelope-common/src/actions.rs index e3c60b0..d2309bc 100644 --- a/antelope-common/src/actions.rs +++ b/antelope-common/src/actions.rs @@ -2,8 +2,11 @@ use serde_json::Value; use substreams_antelope::pb::{ActionTrace, PermissionLevel}; // i.e. https://docs.dfuse.eosnation.io/eosio/public-apis/reference/search/terms/ -pub fn action_keys(trace: &ActionTrace) -> Vec { +pub fn action_keys_extra(trace: &ActionTrace) -> Vec { let action = trace.action.as_ref().unwrap(); + if action.account == "eosio" && action.name == "onblock" { + return vec![]; + } let mut keys = Vec::with_capacity(action.authorization.len() * 2 + 5 + 3); let json_data: Value = match serde_json::from_str(&action.json_data) { Ok(data) => data, @@ -47,3 +50,15 @@ pub fn action_keys(trace: &ActionTrace) -> Vec { keys } + +pub fn action_keys(trace: &ActionTrace) -> Vec { + let action = trace.action.as_ref().unwrap(); + if action.account == "eosio" && action.name == "onblock" { + return vec![]; + } + vec![ + format!("code:{}", action.account), + format!("receiver:{}", trace.receiver), + format!("action:{}", action.name), + ] +} diff --git a/antelope-common/src/index.rs b/antelope-common/src/index.rs index 6294782..6aaee92 100644 --- a/antelope-common/src/index.rs +++ b/antelope-common/src/index.rs @@ -3,7 +3,7 @@ use std::collections::HashSet; use substreams::pb::sf::substreams::index::v1::Keys; use substreams_antelope::pb::ActionTraces; -use crate::actions::action_keys; +use crate::actions::{action_keys, action_keys_extra}; #[substreams::handlers::map] fn index_actions(actions: ActionTraces) -> Result { @@ -17,3 +17,16 @@ fn index_actions(actions: ActionTraces) -> Result Result { + let keys = actions + .action_traces + .into_iter() + .flat_map(|action| action_keys_extra(&action)) + .collect::>() + .into_iter() + .collect(); + + Ok(Keys { keys }) +} diff --git a/antelope-common/src/maps.rs b/antelope-common/src/maps.rs index 1365358..e549c04 100644 --- a/antelope-common/src/maps.rs +++ b/antelope-common/src/maps.rs @@ -1,6 +1,6 @@ use std::collections::HashSet; -use crate::actions::action_keys; +use crate::actions::{action_keys, action_keys_extra}; use substreams::matches_keys_in_parsed_expr; use substreams_antelope::{ pb::{ActionTraces, TransactionTraces}, @@ -68,3 +68,47 @@ fn filtered_transactions( Ok(TransactionTraces { transaction_traces }) } + +#[substreams::handlers::map] +fn filtered_actions_extra( + query: String, + actions: ActionTraces, +) -> Result { + let action_traces = actions + .action_traces + .into_iter() + .filter(|action| { + let keys = action_keys_extra(action); + + // will panic if the query is invalid + matches_keys_in_parsed_expr(&keys, &query).unwrap() + }) + .collect(); + + Ok(ActionTraces { action_traces }) +} + +#[substreams::handlers::map] +fn filtered_transactions_extra( + query: String, + transactions: TransactionTraces, +) -> Result { + let transaction_traces = transactions + .transaction_traces + .into_iter() + .filter(|trx| { + let keys = trx + .action_traces + .iter() + .flat_map(action_keys_extra) + .collect::>() + .into_iter() + .collect::>(); + + // will panic if the query is invalid + matches_keys_in_parsed_expr(&keys, &query).unwrap() + }) + .collect(); + + Ok(TransactionTraces { transaction_traces }) +} diff --git a/antelope-common/substreams.yaml b/antelope-common/substreams.yaml index 5b5e907..e4725e5 100644 --- a/antelope-common/substreams.yaml +++ b/antelope-common/substreams.yaml @@ -2,7 +2,7 @@ specVersion: v0.1.0 package: name: antelope_common image: ./antelope.png - version: v0.3.0 + version: v0.4.0 url: https://github.com/streamingfast/substreams-foundational-modules/antelope-common doc: | common Antelope substreams modules to extract transactions with indexing @@ -10,6 +10,8 @@ package: Use one of those optimized modules with a query string as a parameter: * filtered_transactions * filtered_actions + * filtered_transactions_extra + * filtered_actions_extra The query string will be used for the blockfilter as well as the actual filtering of actions/transactions @@ -35,13 +37,24 @@ modules: type: proto:sf.substreams.index.v1.Keys doc: | `index_actions` sets the following keys on the block: - * Authorized by auth:{account}[@{permission}] + * Contract account code:{account} + * Action action:{action} * Receiver receiver:{account} + + - name: index_actions_extra + kind: blockIndex + inputs: + - map: all_actions + output: + type: proto:sf.substreams.index.v1.Keys + doc: | + `index_actions_extra` sets the following keys on the block: * Contract account code:{account} * Action action:{action} + * Receiver receiver:{account} + * Authorized by auth:{account}[@{permission}] * Action params data.{param}:{value} - - name: all_transactions kind: map inputs: @@ -74,7 +87,7 @@ modules: doc: | `filtered_transactions` reads from all_transactions and applies a filter on the transactions Supported operators are: logical or `||`, logical and `&&` and parenthesis: `()` - Example: `code:eosio.token && action:transfer && (data.to:myaccount || data.from:myaccount)` + Example: `code:eosio.token && action:transfer` - name: filtered_actions kind: map @@ -90,8 +103,42 @@ modules: doc: | `filtered_actions` reads from all_actions and applies a filter on the actions Supported operators are: logical or `||`, logical and `&&` and parenthesis: `()` + Example: `code:eosio.token && action:transfer` + + - name: filtered_transactions_extra + kind: map + blockFilter: + module: index_actions_extra + query: + params: true + inputs: + - params: string + - map: all_transactions + output: + type: proto:sf.antelope.type.v1.TransactionTraces + doc: | + `filtered_transactions_extra` reads from all_transactions and applies a filter on the transactions + Supported operators are: logical or `||`, logical and `&&` and parenthesis: `()` + Example: `code:eosio.token && action:transfer && (data.to:myaccount || data.from:myaccount)` + + - name: filtered_actions_extra + kind: map + blockFilter: + module: index_actions_extra + query: + params: true + inputs: + - params: string + - map: all_actions + output: + type: proto:sf.antelope.type.v1.ActionTraces + doc: | + `filtered_actions_extra` reads from all_actions and applies a filter on the actions + Supported operators are: logical or `||`, logical and `&&` and parenthesis: `()` Example: `code:eosio.token && action:transfer && (data.to:myaccount || data.from:myaccount)` params: filtered_transactions: "code:eosio.token && action:transfer" filtered_actions: "code:eosio.token && action:transfer" + filtered_transactions_extra: "code:eosio.token && action:transfer" + filtered_actions_extra: "code:eosio.token && action:transfer" From 30c4a329fca454fcc7984e9ea3f58f767ebd5ff6 Mon Sep 17 00:00:00 2001 From: YaroShkvorets Date: Thu, 25 Jul 2024 20:18:39 -0400 Subject: [PATCH 2/4] refactor DRY --- antelope-common/src/actions.rs | 64 ------------------------ antelope-common/src/index.rs | 91 ++++++++++++++++++++++++++++------ antelope-common/src/lib.rs | 1 - antelope-common/src/maps.rs | 81 +++++++++++++----------------- 4 files changed, 112 insertions(+), 125 deletions(-) delete mode 100644 antelope-common/src/actions.rs diff --git a/antelope-common/src/actions.rs b/antelope-common/src/actions.rs deleted file mode 100644 index d2309bc..0000000 --- a/antelope-common/src/actions.rs +++ /dev/null @@ -1,64 +0,0 @@ -use serde_json::Value; -use substreams_antelope::pb::{ActionTrace, PermissionLevel}; - -// i.e. https://docs.dfuse.eosnation.io/eosio/public-apis/reference/search/terms/ -pub fn action_keys_extra(trace: &ActionTrace) -> Vec { - let action = trace.action.as_ref().unwrap(); - if action.account == "eosio" && action.name == "onblock" { - return vec![]; - } - let mut keys = Vec::with_capacity(action.authorization.len() * 2 + 5 + 3); - let json_data: Value = match serde_json::from_str(&action.json_data) { - Ok(data) => data, - Err(_) => Value::Object(Default::default()), - }; - - keys.extend(vec![ - format!("code:{}", action.account), - format!("receiver:{}", trace.receiver), - format!("action:{}", action.name), - ]); - - keys.extend( - json_data - .as_object() - .unwrap() - .iter() - .filter_map(|(key, value)| match value { - Value::String(value) => Some(format!("data.{}:{}", key, value)), - Value::Number(value) => Some(format!("data.{}:{}", key, value)), - Value::Bool(value) => Some(format!("data.{}:{}", key, value)), - _ => None, - }), - ); - - keys.extend( - action - .authorization - .iter() - .flat_map(|PermissionLevel { actor, permission }| { - vec![ - format!("auth:{actor}@{permission}"), - format!("auth:{actor}"), - ] - }), - ); - - if trace.creator_action_ordinal == 0 { - keys.push("input:true".to_string()); - } - - keys -} - -pub fn action_keys(trace: &ActionTrace) -> Vec { - let action = trace.action.as_ref().unwrap(); - if action.account == "eosio" && action.name == "onblock" { - return vec![]; - } - vec![ - format!("code:{}", action.account), - format!("receiver:{}", trace.receiver), - format!("action:{}", action.name), - ] -} diff --git a/antelope-common/src/index.rs b/antelope-common/src/index.rs index 6aaee92..e9fa8a2 100644 --- a/antelope-common/src/index.rs +++ b/antelope-common/src/index.rs @@ -1,32 +1,95 @@ use std::collections::HashSet; +use serde_json::Value; use substreams::pb::sf::substreams::index::v1::Keys; -use substreams_antelope::pb::ActionTraces; - -use crate::actions::{action_keys, action_keys_extra}; +use substreams_antelope::pb::{ActionTrace, ActionTraces, PermissionLevel}; #[substreams::handlers::map] fn index_actions(actions: ActionTraces) -> Result { - let keys = actions - .action_traces - .into_iter() - .flat_map(|action| action_keys(&action)) - .collect::>() - .into_iter() - .collect(); - - Ok(Keys { keys }) + collect_keys(actions, action_keys) } #[substreams::handlers::map] fn index_actions_extra(actions: ActionTraces) -> Result { + collect_keys(actions, action_keys_extra) +} + +fn collect_keys( + actions: ActionTraces, + key_extractor: F, +) -> Result +where + F: Fn(&ActionTrace) -> Vec, +{ let keys = actions .action_traces - .into_iter() - .flat_map(|action| action_keys_extra(&action)) + .iter() + .flat_map(key_extractor) .collect::>() .into_iter() .collect(); Ok(Keys { keys }) } + +// i.e. https://docs.dfuse.eosnation.io/eosio/public-apis/reference/search/terms/ +pub fn action_keys_extra(trace: &ActionTrace) -> Vec { + let action = trace.action.as_ref().unwrap(); + if action.account == "eosio" && action.name == "onblock" { + return vec![]; + } + let mut keys = Vec::with_capacity(action.authorization.len() * 2 + 5 + 3); + let json_data: Value = match serde_json::from_str(&action.json_data) { + Ok(data) => data, + Err(_) => Value::Object(Default::default()), + }; + + keys.extend(vec![ + format!("code:{}", action.account), + format!("receiver:{}", trace.receiver), + format!("action:{}", action.name), + ]); + + keys.extend( + json_data + .as_object() + .unwrap() + .iter() + .filter_map(|(key, value)| match value { + Value::String(value) => Some(format!("data.{}:{}", key, value)), + Value::Number(value) => Some(format!("data.{}:{}", key, value)), + Value::Bool(value) => Some(format!("data.{}:{}", key, value)), + _ => None, + }), + ); + + keys.extend( + action + .authorization + .iter() + .flat_map(|PermissionLevel { actor, permission }| { + vec![ + format!("auth:{actor}@{permission}"), + format!("auth:{actor}"), + ] + }), + ); + + if trace.creator_action_ordinal == 0 { + keys.push("input:true".to_string()); + } + + keys +} + +pub fn action_keys(trace: &ActionTrace) -> Vec { + let action = trace.action.as_ref().unwrap(); + if action.account == "eosio" && action.name == "onblock" { + return vec![]; + } + vec![ + format!("code:{}", action.account), + format!("receiver:{}", trace.receiver), + format!("action:{}", action.name), + ] +} diff --git a/antelope-common/src/lib.rs b/antelope-common/src/lib.rs index 69432f8..c7f3a32 100644 --- a/antelope-common/src/lib.rs +++ b/antelope-common/src/lib.rs @@ -1,3 +1,2 @@ -mod actions; mod index; mod maps; diff --git a/antelope-common/src/maps.rs b/antelope-common/src/maps.rs index e549c04..bced918 100644 --- a/antelope-common/src/maps.rs +++ b/antelope-common/src/maps.rs @@ -1,21 +1,22 @@ use std::collections::HashSet; -use crate::actions::{action_keys, action_keys_extra}; -use substreams::matches_keys_in_parsed_expr; +use substreams::{errors::Error, matches_keys_in_parsed_expr}; use substreams_antelope::{ - pb::{ActionTraces, TransactionTraces}, + pb::{ActionTrace, ActionTraces, TransactionTraces}, Block, }; +use crate::index::{action_keys, action_keys_extra}; + #[substreams::handlers::map] -fn all_transactions(block: Block) -> Result { +fn all_transactions(block: Block) -> Result { Ok(TransactionTraces { transaction_traces: block.into_transaction_traces().collect(), }) } #[substreams::handlers::map] -fn all_actions(transactions: TransactionTraces) -> Result { +fn all_actions(transactions: TransactionTraces) -> Result { let action_traces = transactions .transaction_traces .into_iter() @@ -26,59 +27,44 @@ fn all_actions(transactions: TransactionTraces) -> Result Result { - let action_traces = actions - .action_traces - .into_iter() - .filter(|action| { - let keys = action_keys(action); - - // will panic if the query is invalid - matches_keys_in_parsed_expr(&keys, &query).unwrap() - }) - .collect(); +fn filtered_actions(query: String, actions: ActionTraces) -> Result { + filter_actions(query, actions, action_keys) +} - Ok(ActionTraces { action_traces }) +#[substreams::handlers::map] +fn filtered_actions_extra(query: String, actions: ActionTraces) -> Result { + filter_actions(query, actions, action_keys_extra) } #[substreams::handlers::map] fn filtered_transactions( query: String, transactions: TransactionTraces, -) -> Result { - let transaction_traces = transactions - .transaction_traces - .into_iter() - .filter(|trx| { - let keys = trx - .action_traces - .iter() - .flat_map(action_keys) - .collect::>() - .into_iter() - .collect::>(); - - // will panic if the query is invalid - matches_keys_in_parsed_expr(&keys, &query).unwrap() - }) - .collect(); - - Ok(TransactionTraces { transaction_traces }) +) -> Result { + filter_transactions(query, transactions, action_keys) } #[substreams::handlers::map] -fn filtered_actions_extra( +fn filtered_transactions_extra( + query: String, + transactions: TransactionTraces, +) -> Result { + filter_transactions(query, transactions, action_keys_extra) +} + +fn filter_actions( query: String, actions: ActionTraces, -) -> Result { + key_extractor: F, +) -> Result +where + F: Fn(&ActionTrace) -> Vec, +{ let action_traces = actions .action_traces .into_iter() .filter(|action| { - let keys = action_keys_extra(action); + let keys = key_extractor(action); // will panic if the query is invalid matches_keys_in_parsed_expr(&keys, &query).unwrap() @@ -88,11 +74,14 @@ fn filtered_actions_extra( Ok(ActionTraces { action_traces }) } -#[substreams::handlers::map] -fn filtered_transactions_extra( +fn filter_transactions( query: String, transactions: TransactionTraces, -) -> Result { + key_extractor: F, +) -> Result +where + F: Fn(&ActionTrace) -> Vec, +{ let transaction_traces = transactions .transaction_traces .into_iter() @@ -100,7 +89,7 @@ fn filtered_transactions_extra( let keys = trx .action_traces .iter() - .flat_map(action_keys_extra) + .flat_map(&key_extractor) .collect::>() .into_iter() .collect::>(); From 37b61d122f20fcc5845b1d71a143ac2e3a27bf4f Mon Sep 17 00:00:00 2001 From: YaroShkvorets Date: Thu, 25 Jul 2024 20:58:17 -0400 Subject: [PATCH 3/4] make default params more generic --- antelope-common/substreams.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/antelope-common/substreams.yaml b/antelope-common/substreams.yaml index e4725e5..0436a8f 100644 --- a/antelope-common/substreams.yaml +++ b/antelope-common/substreams.yaml @@ -138,7 +138,7 @@ modules: Example: `code:eosio.token && action:transfer && (data.to:myaccount || data.from:myaccount)` params: - filtered_transactions: "code:eosio.token && action:transfer" - filtered_actions: "code:eosio.token && action:transfer" - filtered_transactions_extra: "code:eosio.token && action:transfer" - filtered_actions_extra: "code:eosio.token && action:transfer" + filtered_transactions: "code:eosio" + filtered_actions: "code:eosio" + filtered_transactions_extra: "code:eosio" + filtered_actions_extra: "code:eosio" From 0ecba222b8b343a2852b2ab7698fe496b56d850c Mon Sep 17 00:00:00 2001 From: YaroShkvorets Date: Thu, 25 Jul 2024 21:03:58 -0400 Subject: [PATCH 4/4] update readme --- antelope-common/README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/antelope-common/README.md b/antelope-common/README.md index 58fba74..5bd88f2 100644 --- a/antelope-common/README.md +++ b/antelope-common/README.md @@ -10,9 +10,12 @@ There are three types of foundational modules: Antelope foundational modules include: - `all_transactions` - all transactions in a block - `all_actions` - all flattened actions in a block -- `index_actions` - index blocks with relevant action-related keys -- `filtered_actions` - flattened actions in a block filtered based on the query -- `filtered_transactions` - transactions in a block filtered based on the query +- `index_actions` - "light" block index including only action contract, receiver and name +- `index_actions_extra` - "heavy" block index that also includes action authorization and parameters +- `filtered_actions` - flattened actions in a block filtered based on the query and `index_actions` index +- `filtered_actions_extra` - flattened actions in a block filtered based on the query and `index_actions_extra` index +- `filtered_transactions` - transactions in a block filtered based on the query and `index_actions` index +- `filtered_transactions_extra` - transactions in a block filtered based on the query and `index_actions_extra` index Filtered modules with queries take advantage of backend indexing, so every subsequent request with the same query will make the backend stream the response much faster skipping all the empty blocks. Common use cases: @@ -25,7 +28,7 @@ Common use cases: Let's say you want to receive all AtomicAssets NFT create collection events starting from block 370,000,000. Send a substreams request with the desired query as parameter. You can use a gRPC client, substreams sink, or substreams CLI: ```bash -> substreams gui -e eos.substreams.pinax.network:443 https://spkg.io/pinax-network/antelope-common-v0.3.0.spkg filtered_actions -s 370000000 -p filtered_actions="code:atomicassets && action:createcol" --production-mode +> substreams gui -e eos.substreams.pinax.network:443 https://spkg.io/pinax-network/antelope-common-v0.4.0.spkg filtered_actions -s 370000000 -p filtered_actions="code:atomicassets && action:createcol" --production-mode ``` If the request with this query hasn't been run before, substreams backend will start the indexing process and you should start seeing new events. If the request has been run before, you should start seeing sale actions right away jumping over any empty chunks of blocks when there were no sales. Note, we used `--production-mode` flag - this ensures backend writes indexes on disk so they can be re-used in the future. @@ -54,12 +57,13 @@ Queries can include `&&` and `||` logical operands, as well as `(` and `)` paren ### Release -v0.3.0: https://substreams.dev/pinax-network/antelope-common/v0.3.0 +v0.4.0: https://substreams.dev/pinax-network/antelope-common/v0.4.0 ### Usage ```bash -substreams gui -e eos.substreams.pinax.network:443 https://spkg.io/pinax-network/antelope-common-v0.3.0.spkg filtered_actions -s -10000 -p filtered_actions="code:tethertether && data.to:swap.defi" --production-mode +substreams gui -e eos.substreams.pinax.network:443 https://spkg.io/pinax-network/antelope-common-v0.4.0.spkg filtered_actions -s -10000 -p filtered_actions="code:tethertether && action:transfer" --production-mode +substreams gui -e eos.substreams.pinax.network:443 https://spkg.io/pinax-network/antelope-common-v0.4.0.spkg filtered_actions_extra -s -10000 -p filtered_actions_extra="code:eosio.token && action:transfer && (data.to:myaccount || data.from::myaccount)"" --production-mode ``` ### Known issues