Merge pull request #3 from pinax-network/yaro/extra_index

Add "light" index

antelope-common/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "antelope-common"
-version = "0.3.0"
+version = "0.4.0"
 description = "Foundational Substreams Modules for Antelope Chains"
 edition = "2021"
 repository = "https://github.com/streamingfast/substreams-foundational-modules/antelope-common"

antelope-common/README.md

@@ -10,9 +10,12 @@ There are three types of foundational modules:
 Antelope foundational modules include:
 - `all_transactions` - all transactions in a block
 - `all_actions` - all flattened actions in a block
-- `index_actions` - index blocks with relevant action-related keys
-- `filtered_actions` - flattened actions in a block filtered based on the query
-- `filtered_transactions` - transactions in a block filtered based on the query
+- `index_actions` - "light" block index including only action contract, receiver and name
+- `index_actions_extra` - "heavy" block index that also includes action authorization and parameters
+- `filtered_actions` - flattened actions in a block filtered based on the query  and `index_actions` index
+- `filtered_actions_extra` - flattened actions in a block filtered based on the query and `index_actions_extra` index
+- `filtered_transactions` - transactions in a block filtered based on the query and `index_actions` index
+- `filtered_transactions_extra` - transactions in a block filtered based on the query and `index_actions_extra` index
 
 Filtered modules with queries take advantage of backend indexing, so every subsequent request with the same query will make the backend stream the response much faster skipping all the empty blocks.
 Common use cases:
@@ -25,7 +28,7 @@ Common use cases:
 Let's say you want to receive all AtomicAssets NFT create collection events starting from block 370,000,000.
 Send a substreams request with the desired query as parameter. You can use a gRPC client, substreams sink, or substreams CLI:
 ```bash
-> substreams gui -e eos.substreams.pinax.network:443 https://spkg.io/pinax-network/antelope-common-v0.3.0.spkg filtered_actions -s 370000000 -p filtered_actions="code:atomicassets && action:createcol" --production-mode
+> substreams gui -e eos.substreams.pinax.network:443 https://spkg.io/pinax-network/antelope-common-v0.4.0.spkg filtered_actions -s 370000000 -p filtered_actions="code:atomicassets && action:createcol" --production-mode
 ```
 If the request with this query hasn't been run before, substreams backend will start the indexing process and you should start seeing new events. If the request has been run before, you should start seeing sale actions right away jumping over any empty chunks of blocks when there were no sales.
 Note, we used `--production-mode` flag - this ensures backend writes indexes on disk so they can be re-used in the future. 
@@ -54,12 +57,13 @@ Queries can include `&&` and `||` logical operands, as well as `(` and `)` paren
 
 
 ### Release
-v0.3.0: https://substreams.dev/pinax-network/antelope-common/v0.3.0
+v0.4.0: https://substreams.dev/pinax-network/antelope-common/v0.4.0
 
 
 ### Usage
 ```bash
-substreams gui -e eos.substreams.pinax.network:443 https://spkg.io/pinax-network/antelope-common-v0.3.0.spkg filtered_actions -s -10000 -p filtered_actions="code:tethertether && data.to:swap.defi" --production-mode
+substreams gui -e eos.substreams.pinax.network:443 https://spkg.io/pinax-network/antelope-common-v0.4.0.spkg filtered_actions -s -10000 -p filtered_actions="code:tethertether && action:transfer" --production-mode
+substreams gui -e eos.substreams.pinax.network:443 https://spkg.io/pinax-network/antelope-common-v0.4.0.spkg filtered_actions_extra -s -10000 -p filtered_actions_extra="code:eosio.token && action:transfer && (data.to:myaccount || data.from::myaccount)"" --production-mode
 ```
 
 ### Known issues

antelope-common/src/index.rs

@@ -1,19 +1,95 @@
 use std::collections::HashSet;
 
+use serde_json::Value;
 use substreams::pb::sf::substreams::index::v1::Keys;
-use substreams_antelope::pb::ActionTraces;
-
-use crate::actions::action_keys;
+use substreams_antelope::pb::{ActionTrace, ActionTraces, PermissionLevel};
 
 #[substreams::handlers::map]
 fn index_actions(actions: ActionTraces) -> Result<Keys, substreams::errors::Error> {
+    collect_keys(actions, action_keys)
+}
+
+#[substreams::handlers::map]
+fn index_actions_extra(actions: ActionTraces) -> Result<Keys, substreams::errors::Error> {
+    collect_keys(actions, action_keys_extra)
+}
+
+fn collect_keys<F>(
+    actions: ActionTraces,
+    key_extractor: F,
+) -> Result<Keys, substreams::errors::Error>
+where
+    F: Fn(&ActionTrace) -> Vec<String>,
+{
     let keys = actions
         .action_traces
-        .into_iter()
-        .flat_map(|action| action_keys(&action))
+        .iter()
+        .flat_map(key_extractor)
         .collect::<HashSet<_>>()
         .into_iter()
         .collect();
 
     Ok(Keys { keys })
 }
+
+// i.e. https://docs.dfuse.eosnation.io/eosio/public-apis/reference/search/terms/
+pub fn action_keys_extra(trace: &ActionTrace) -> Vec<String> {
+    let action = trace.action.as_ref().unwrap();
+    if action.account == "eosio" && action.name == "onblock" {
+        return vec![];
+    }
+    let mut keys = Vec::with_capacity(action.authorization.len() * 2 + 5 + 3);
+    let json_data: Value = match serde_json::from_str(&action.json_data) {
+        Ok(data) => data,
+        Err(_) => Value::Object(Default::default()),
+    };
+
+    keys.extend(vec![
+        format!("code:{}", action.account),
+        format!("receiver:{}", trace.receiver),
+        format!("action:{}", action.name),
+    ]);
+
+    keys.extend(
+        json_data
+            .as_object()
+            .unwrap()
+            .iter()
+            .filter_map(|(key, value)| match value {
+                Value::String(value) => Some(format!("data.{}:{}", key, value)),
+                Value::Number(value) => Some(format!("data.{}:{}", key, value)),
+                Value::Bool(value) => Some(format!("data.{}:{}", key, value)),
+                _ => None,
+            }),
+    );
+
+    keys.extend(
+        action
+            .authorization
+            .iter()
+            .flat_map(|PermissionLevel { actor, permission }| {
+                vec![
+                    format!("auth:{actor}@{permission}"),
+                    format!("auth:{actor}"),
+                ]
+            }),
+    );
+
+    if trace.creator_action_ordinal == 0 {
+        keys.push("input:true".to_string());
+    }
+
+    keys
+}
+
+pub fn action_keys(trace: &ActionTrace) -> Vec<String> {
+    let action = trace.action.as_ref().unwrap();
+    if action.account == "eosio" && action.name == "onblock" {
+        return vec![];
+    }
+    vec![
+        format!("code:{}", action.account),
+        format!("receiver:{}", trace.receiver),
+        format!("action:{}", action.name),
+    ]
+}

antelope-common/src/lib.rs

@@ -1,3 +1,2 @@
-mod actions;
 mod index;
 mod maps;

antelope-common/src/maps.rs

@@ -1,21 +1,22 @@
 use std::collections::HashSet;
 
-use crate::actions::action_keys;
-use substreams::matches_keys_in_parsed_expr;
+use substreams::{errors::Error, matches_keys_in_parsed_expr};
 use substreams_antelope::{
-    pb::{ActionTraces, TransactionTraces},
+    pb::{ActionTrace, ActionTraces, TransactionTraces},
     Block,
 };
 
+use crate::index::{action_keys, action_keys_extra};
+
 #[substreams::handlers::map]
-fn all_transactions(block: Block) -> Result<TransactionTraces, substreams::errors::Error> {
+fn all_transactions(block: Block) -> Result<TransactionTraces, Error> {
     Ok(TransactionTraces {
         transaction_traces: block.into_transaction_traces().collect(),
     })
 }
 
 #[substreams::handlers::map]
-fn all_actions(transactions: TransactionTraces) -> Result<ActionTraces, substreams::errors::Error> {
+fn all_actions(transactions: TransactionTraces) -> Result<ActionTraces, Error> {
     let action_traces = transactions
         .transaction_traces
         .into_iter()
@@ -26,15 +27,44 @@ fn all_actions(transactions: TransactionTraces) -> Result<ActionTraces, substrea
 }
 
 #[substreams::handlers::map]
-fn filtered_actions(
+fn filtered_actions(query: String, actions: ActionTraces) -> Result<ActionTraces, Error> {
+    filter_actions(query, actions, action_keys)
+}
+
+#[substreams::handlers::map]
+fn filtered_actions_extra(query: String, actions: ActionTraces) -> Result<ActionTraces, Error> {
+    filter_actions(query, actions, action_keys_extra)
+}
+
+#[substreams::handlers::map]
+fn filtered_transactions(
+    query: String,
+    transactions: TransactionTraces,
+) -> Result<TransactionTraces, Error> {
+    filter_transactions(query, transactions, action_keys)
+}
+
+#[substreams::handlers::map]
+fn filtered_transactions_extra(
+    query: String,
+    transactions: TransactionTraces,
+) -> Result<TransactionTraces, Error> {
+    filter_transactions(query, transactions, action_keys_extra)
+}
+
+fn filter_actions<F>(
     query: String,
     actions: ActionTraces,
-) -> Result<ActionTraces, substreams::errors::Error> {
+    key_extractor: F,
+) -> Result<ActionTraces, Error>
+where
+    F: Fn(&ActionTrace) -> Vec<String>,
+{
     let action_traces = actions
         .action_traces
         .into_iter()
         .filter(|action| {
-            let keys = action_keys(action);
+            let keys = key_extractor(action);
 
             // will panic if the query is invalid
             matches_keys_in_parsed_expr(&keys, &query).unwrap()
@@ -44,19 +74,22 @@ fn filtered_actions(
     Ok(ActionTraces { action_traces })
 }
 
-#[substreams::handlers::map]
-fn filtered_transactions(
+fn filter_transactions<F>(
     query: String,
     transactions: TransactionTraces,
-) -> Result<TransactionTraces, substreams::errors::Error> {
+    key_extractor: F,
+) -> Result<TransactionTraces, Error>
+where
+    F: Fn(&ActionTrace) -> Vec<String>,
+{
     let transaction_traces = transactions
         .transaction_traces
         .into_iter()
         .filter(|trx| {
             let keys = trx
                 .action_traces
                 .iter()
-                .flat_map(action_keys)
+                .flat_map(&key_extractor)
                 .collect::<HashSet<_>>()
                 .into_iter()
                 .collect::<Vec<_>>();