From 0c97d53338f935c99749e960c14729b156001750 Mon Sep 17 00:00:00 2001 From: Atris Date: Thu, 25 Jul 2024 19:50:02 +0200 Subject: [PATCH 1/4] docs: create pindexer readme and mention pindexer<>pd compat in errors --- crates/bin/pindexer/README.md | 11 ++++++++++ crates/bin/pindexer/src/block.rs | 3 ++- crates/bin/pindexer/src/dex/mod.rs | 32 +++++++++++++++++++----------- crates/bin/pindexer/src/lib.rs | 2 ++ 4 files changed, 35 insertions(+), 13 deletions(-) create mode 100644 crates/bin/pindexer/README.md diff --git a/crates/bin/pindexer/README.md b/crates/bin/pindexer/README.md new file mode 100644 index 0000000000..3e94221e2e --- /dev/null +++ b/crates/bin/pindexer/README.md @@ -0,0 +1,11 @@ +# pindexer + +A indexer that ingests ABCI events emitted by pd and transforms them into useful data. + +## Usage + +1. Follow the setup instructions in cometindex README +2. `cargo run --bin pindexer -- -s "postgresql://localhost:5432/testnet_raw?sslmode=disable" -d "postgresql://localhost:5432/testnet_compiled?sslmode=disable"` + +## pd compatibility + diff --git a/crates/bin/pindexer/src/block.rs b/crates/bin/pindexer/src/block.rs index 6f3aa3b676..8fe4c8538c 100644 --- a/crates/bin/pindexer/src/block.rs +++ b/crates/bin/pindexer/src/block.rs @@ -1,3 +1,4 @@ +use crate::PD_COMPAT; use anyhow::anyhow; use cometindex::{async_trait, sqlx, AppView, ContextualizedEvent, PgTransaction}; use penumbra_proto::{core::component::sct::v1 as pb, event::ProtoEvent}; @@ -52,7 +53,7 @@ CREATE TABLE IF NOT EXISTS block_details ( .bind(i64::try_from(pe.height)?) .bind( DateTime::from_timestamp(timestamp.seconds, u32::try_from(timestamp.nanos)?) - .ok_or(anyhow!("failed to convert timestamp"))?, + .ok_or(anyhow!("failed to convert timestamp." + PD_COMPAT))?, ) .bind(pe.root.unwrap().inner) .execute(dbtx.as_mut()) diff --git a/crates/bin/pindexer/src/dex/mod.rs b/crates/bin/pindexer/src/dex/mod.rs index 18884bbc96..8a7888b030 100644 --- a/crates/bin/pindexer/src/dex/mod.rs +++ b/crates/bin/pindexer/src/dex/mod.rs @@ -9,7 +9,7 @@ use penumbra_proto::{event::ProtoEvent, penumbra::core::component::dex::v1 as pb use sqlx::{PgPool, Postgres, Transaction}; use crate::sql::Sql; -use crate::{AppView, ContextualizedEvent, PgTransaction}; +use crate::{AppView, ContextualizedEvent, PgTransaction, PD_COMPAT}; /// One of the possible events that we care about. #[derive(Clone, Debug)] @@ -142,14 +142,18 @@ impl<'a> TryFrom<&'a ContextualizedEvent> for Event { // Credit x if x == Event::NAMES[0] => { let pe = pb::EventValueCircuitBreakerCredit::from_event(event.as_ref())?; - let asset_id = - AssetId::try_from(pe.asset_id.ok_or(anyhow!("event missing asset_id"))?)?; + let asset_id = AssetId::try_from( + pe.asset_id + .ok_or(anyhow!("event missing asset_id" + PD_COMPAT))?, + )?; let previous_balance = Amount::try_from( pe.previous_balance - .ok_or(anyhow!("event missing previous_balance"))?, + .ok_or(anyhow!("event missing previous_balance" + PD_COMPAT))?, + )?; + let new_balance = Amount::try_from( + pe.new_balance + .ok_or(anyhow!("event missing new_balance" + PD_COMPAT))?, )?; - let new_balance = - Amount::try_from(pe.new_balance.ok_or(anyhow!("event missing new_balance"))?)?; Ok(Self::CircuitBreakerCredit { asset_id, previous_balance, @@ -159,14 +163,18 @@ impl<'a> TryFrom<&'a ContextualizedEvent> for Event { // Debit x if x == Event::NAMES[1] => { let pe = pb::EventValueCircuitBreakerDebit::from_event(event.as_ref())?; - let asset_id = - AssetId::try_from(pe.asset_id.ok_or(anyhow!("event missing asset_id"))?)?; + let asset_id = AssetId::try_from( + pe.asset_id + .ok_or(anyhow!("event missing asset_id" + PD_COMPAT))?, + )?; let previous_balance = Amount::try_from( pe.previous_balance - .ok_or(anyhow!("event missing previous_balance"))?, + .ok_or(anyhow!("event missing previous_balance" + PD_COMPAT))?, + )?; + let new_balance = Amount::try_from( + pe.new_balance + .ok_or(anyhow!("event missing new_balance" + PD_COMPAT))?, )?; - let new_balance = - Amount::try_from(pe.new_balance.ok_or(anyhow!("event missing new_balance"))?)?; Ok(Self::CircuitBreakerDebit { asset_id, previous_balance, @@ -179,7 +187,7 @@ impl<'a> TryFrom<&'a ContextualizedEvent> for Event { let height = pe.height; let execution = pe .swap_execution - .ok_or(anyhow!("missing swap execution"))? + .ok_or(anyhow!("missing swap execution" + PD_COMPAT))? .try_into()?; Ok(Self::ArbExecution { height, execution }) } diff --git a/crates/bin/pindexer/src/lib.rs b/crates/bin/pindexer/src/lib.rs index 9c443724d8..148ba9fdce 100644 --- a/crates/bin/pindexer/src/lib.rs +++ b/crates/bin/pindexer/src/lib.rs @@ -7,3 +7,5 @@ pub mod dex; pub mod shielded_pool; mod sql; pub mod stake; + +pub const PD_COMPAT: &'static str = " Check that your pd and pindexer versions match. See pd compatibility section in README for more information."; From 5f7a9ccbe392e0e59bbbd601684b7ea7f2c3e5bb Mon Sep 17 00:00:00 2001 From: Atris Date: Thu, 25 Jul 2024 20:12:03 +0200 Subject: [PATCH 2/4] docs: create pindexer readme and mention pindexer<>pd compat in errors --- crates/bin/pindexer/src/block.rs | 2 +- crates/bin/pindexer/src/dex/mod.rs | 30 +++++++++++------------------- crates/bin/pindexer/src/lib.rs | 2 +- 3 files changed, 13 insertions(+), 21 deletions(-) diff --git a/crates/bin/pindexer/src/block.rs b/crates/bin/pindexer/src/block.rs index 8fe4c8538c..21ee69dc79 100644 --- a/crates/bin/pindexer/src/block.rs +++ b/crates/bin/pindexer/src/block.rs @@ -53,7 +53,7 @@ CREATE TABLE IF NOT EXISTS block_details ( .bind(i64::try_from(pe.height)?) .bind( DateTime::from_timestamp(timestamp.seconds, u32::try_from(timestamp.nanos)?) - .ok_or(anyhow!("failed to convert timestamp." + PD_COMPAT))?, + .ok_or(anyhow!(format!("failed to convert timestamp. {PD_COMPAT}")))?, ) .bind(pe.root.unwrap().inner) .execute(dbtx.as_mut()) diff --git a/crates/bin/pindexer/src/dex/mod.rs b/crates/bin/pindexer/src/dex/mod.rs index 8a7888b030..dc2c9d85a5 100644 --- a/crates/bin/pindexer/src/dex/mod.rs +++ b/crates/bin/pindexer/src/dex/mod.rs @@ -142,18 +142,14 @@ impl<'a> TryFrom<&'a ContextualizedEvent> for Event { // Credit x if x == Event::NAMES[0] => { let pe = pb::EventValueCircuitBreakerCredit::from_event(event.as_ref())?; - let asset_id = AssetId::try_from( - pe.asset_id - .ok_or(anyhow!("event missing asset_id" + PD_COMPAT))?, - )?; + let asset_id = + AssetId::try_from(pe.asset_id.ok_or(anyhow!("event missing asset_id"))?)?; let previous_balance = Amount::try_from( pe.previous_balance - .ok_or(anyhow!("event missing previous_balance" + PD_COMPAT))?, - )?; - let new_balance = Amount::try_from( - pe.new_balance - .ok_or(anyhow!("event missing new_balance" + PD_COMPAT))?, + .ok_or(anyhow!("event missing previous_balance"))?, )?; + let new_balance = + Amount::try_from(pe.new_balance.ok_or(anyhow!("event missing new_balance"))?)?; Ok(Self::CircuitBreakerCredit { asset_id, previous_balance, @@ -163,18 +159,14 @@ impl<'a> TryFrom<&'a ContextualizedEvent> for Event { // Debit x if x == Event::NAMES[1] => { let pe = pb::EventValueCircuitBreakerDebit::from_event(event.as_ref())?; - let asset_id = AssetId::try_from( - pe.asset_id - .ok_or(anyhow!("event missing asset_id" + PD_COMPAT))?, - )?; + let asset_id = + AssetId::try_from(pe.asset_id.ok_or(anyhow!("event missing asset_id"))?)?; let previous_balance = Amount::try_from( pe.previous_balance - .ok_or(anyhow!("event missing previous_balance" + PD_COMPAT))?, - )?; - let new_balance = Amount::try_from( - pe.new_balance - .ok_or(anyhow!("event missing new_balance" + PD_COMPAT))?, + .ok_or(anyhow!("event missing previous_balance"))?, )?; + let new_balance = + Amount::try_from(pe.new_balance.ok_or(anyhow!("event missing new_balance"))?)?; Ok(Self::CircuitBreakerDebit { asset_id, previous_balance, @@ -187,7 +179,7 @@ impl<'a> TryFrom<&'a ContextualizedEvent> for Event { let height = pe.height; let execution = pe .swap_execution - .ok_or(anyhow!("missing swap execution" + PD_COMPAT))? + .ok_or(anyhow!("missing swap execution"))? .try_into()?; Ok(Self::ArbExecution { height, execution }) } diff --git a/crates/bin/pindexer/src/lib.rs b/crates/bin/pindexer/src/lib.rs index 148ba9fdce..c80951c32d 100644 --- a/crates/bin/pindexer/src/lib.rs +++ b/crates/bin/pindexer/src/lib.rs @@ -8,4 +8,4 @@ pub mod shielded_pool; mod sql; pub mod stake; -pub const PD_COMPAT: &'static str = " Check that your pd and pindexer versions match. See pd compatibility section in README for more information."; +pub const PD_COMPAT: &'static str = "Check that your pd and pindexer versions match. See pd compatibility section in README for more information."; From bf09422a5d32c8ed4d832915c38c0ed58c87984b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=BAc=C3=A1s=20Meier?= Date: Thu, 25 Jul 2024 11:13:05 -0700 Subject: [PATCH 3/4] Update crates/bin/pindexer/src/dex/mod.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lúcás Meier --- crates/bin/pindexer/src/dex/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bin/pindexer/src/dex/mod.rs b/crates/bin/pindexer/src/dex/mod.rs index dc2c9d85a5..18884bbc96 100644 --- a/crates/bin/pindexer/src/dex/mod.rs +++ b/crates/bin/pindexer/src/dex/mod.rs @@ -9,7 +9,7 @@ use penumbra_proto::{event::ProtoEvent, penumbra::core::component::dex::v1 as pb use sqlx::{PgPool, Postgres, Transaction}; use crate::sql::Sql; -use crate::{AppView, ContextualizedEvent, PgTransaction, PD_COMPAT}; +use crate::{AppView, ContextualizedEvent, PgTransaction}; /// One of the possible events that we care about. #[derive(Clone, Debug)] From 75194c9a42d4a76e23c8f8f5d1520abc69fae298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=BAc=C3=A1s=20Meier?= Date: Thu, 25 Jul 2024 11:13:22 -0700 Subject: [PATCH 4/4] Update crates/bin/pindexer/src/lib.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Lúcás Meier --- crates/bin/pindexer/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/bin/pindexer/src/lib.rs b/crates/bin/pindexer/src/lib.rs index c80951c32d..b4e5b3c89e 100644 --- a/crates/bin/pindexer/src/lib.rs +++ b/crates/bin/pindexer/src/lib.rs @@ -8,4 +8,4 @@ pub mod shielded_pool; mod sql; pub mod stake; -pub const PD_COMPAT: &'static str = "Check that your pd and pindexer versions match. See pd compatibility section in README for more information."; +pub(crate) const PD_COMPAT: &'static str = "Check that your pd and pindexer versions match. See pd compatibility section in README for more information.";