From f46b82da0fb4f22170d68e5706745719e2cc860c Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Thu, 14 Nov 2024 01:55:01 +0100 Subject: [PATCH 01/70] Implementation --- .../src/database/onchain_order_events.rs | 7 ++++ crates/database/src/orders.rs | 32 +++++++++++++++---- crates/orderbook/src/database/orders.rs | 19 ++++++++--- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/crates/autopilot/src/database/onchain_order_events.rs b/crates/autopilot/src/database/onchain_order_events.rs index 0c757d63ff..136211f7b6 100644 --- a/crates/autopilot/src/database/onchain_order_events.rs +++ b/crates/autopilot/src/database/onchain_order_events.rs @@ -487,6 +487,8 @@ async fn parse_general_onchain_order_placement_data<'a>( sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), + call_data: order_data.1.call_data.clone(), + verified: quote.data.verified, }), Err(err) => { let err_label = err.to_metrics_label(); @@ -620,6 +622,7 @@ fn convert_onchain_order_placement( true => OrderClass::Limit, false => OrderClass::Market, }, + call_data: String::new(), }; let onchain_order_placement_event = OnchainOrderPlacement { order_uid: ByteArray(order_uid.0), @@ -925,6 +928,7 @@ mod test { buy_token_balance: buy_token_destination_into(expected_order_data.buy_token_balance), full_fee_amount: u256_to_big_decimal(&expected_order_data.fee_amount), cancellation_timestamp: None, + call_data: String::new(), }; assert_eq!(onchain_order_placement, expected_onchain_order_placement); assert_eq!(order, expected_order); @@ -1036,6 +1040,7 @@ mod test { buy_token_balance: buy_token_destination_into(expected_order_data.buy_token_balance), full_fee_amount: u256_to_big_decimal(&U256::zero()), cancellation_timestamp: None, + call_data: String::new(), }; assert_eq!(onchain_order_placement, expected_onchain_order_placement); assert_eq!(order, expected_order); @@ -1187,6 +1192,8 @@ mod test { sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), + call_data: String::new(), + verified: quote.data.verified, }; assert_eq!(result.1, vec![Some(expected_quote)]); assert_eq!( diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index a574dbc882..57542c347c 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -99,6 +99,7 @@ pub struct Order { pub full_fee_amount: BigDecimal, pub cancellation_timestamp: Option>, pub class: OrderClass, + pub call_data: String, } pub async fn insert_orders_and_ignore_conflicts( @@ -142,9 +143,10 @@ INSERT INTO orders ( buy_token_balance, full_fee_amount, cancellation_timestamp, - class + class, + call_data ) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22) "#; pub async fn insert_order_and_ignore_conflicts( @@ -186,6 +188,7 @@ async fn insert_order_execute_sqlx( .bind(&order.full_fee_amount) .bind(order.cancellation_timestamp) .bind(order.class) + .bind(&order.call_data) .execute(ex) .await?; Ok(()) @@ -329,6 +332,8 @@ pub struct Quote { pub sell_amount: BigDecimal, pub buy_amount: BigDecimal, pub solver: Address, + pub call_data: String, + pub verified: bool, } pub async fn insert_quotes(ex: &mut PgConnection, quotes: &[Quote]) -> Result<(), sqlx::Error> { @@ -346,9 +351,11 @@ INSERT INTO order_quotes ( sell_token_price, sell_amount, buy_amount, - solver + solver, + call_data, + verified, ) -VALUES ($1, $2, $3, $4, $5, $6, $7)"#; +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"#; pub async fn insert_quote_and_update_on_conflict( ex: &mut PgConnection, @@ -362,7 +369,7 @@ pub async fn insert_quote_and_update_on_conflict( " ON CONFLICT (order_uid) DO UPDATE SET gas_amount = $2, gas_price = $3, sell_token_price = $4, sell_amount = $5, -buy_amount = $6 +buy_amount = $6, call_data = $8, verified = $9 " ); sqlx::query(QUERY) @@ -373,6 +380,8 @@ buy_amount = $6 .bind("e.sell_amount) .bind("e.buy_amount) .bind(quote.solver) + .bind("e.call_data) + .bind(quote.verified) .execute(ex) .await?; Ok(()) @@ -387,6 +396,8 @@ pub async fn insert_quote(ex: &mut PgConnection, quote: &Quote) -> Result<(), sq .bind("e.sell_amount) .bind("e.buy_amount) .bind(quote.solver) + .bind("e.call_data) + .bind(quote.verified) .execute(ex) .await?; Ok(()) @@ -487,6 +498,7 @@ pub struct FullOrder { pub onchain_placement_error: Option, pub executed_surplus_fee: BigDecimal, pub full_app_data: Option>, + pub call_data: Option, } #[derive(Debug, sqlx::FromRow)] @@ -498,6 +510,7 @@ pub struct FullOrderWithQuote { pub quote_gas_amount: Option, pub quote_gas_price: Option, pub quote_sell_token_price: Option, + pub quote_verified: Option, pub solver: Option
, } @@ -538,7 +551,7 @@ pub const SELECT: &str = r#" o.uid, o.owner, o.creation_timestamp, o.sell_token, o.buy_token, o.sell_amount, o.buy_amount, o.valid_to, o.app_data, o.fee_amount, o.full_fee_amount, o.kind, o.partially_fillable, o.signature, o.receiver, o.signing_scheme, o.settlement_contract, o.sell_token_balance, o.buy_token_balance, -o.class, +o.class, o.call_data, (SELECT COALESCE(SUM(t.buy_amount), 0) FROM trades t WHERE t.order_uid = o.uid) AS sum_buy, (SELECT COALESCE(SUM(t.sell_amount), 0) FROM trades t WHERE t.order_uid = o.uid) AS sum_sell, (SELECT COALESCE(SUM(t.fee_amount), 0) FROM trades t WHERE t.order_uid = o.uid) AS sum_fee, @@ -591,6 +604,7 @@ pub async fn single_full_order_with_quote( ", o_quotes.gas_amount as quote_gas_amount", ", o_quotes.gas_price as quote_gas_price", ", o_quotes.sell_token_price as quote_sell_token_price", + ", o_quotes.verified as quote_verified", ", o_quotes.solver as solver", " FROM ", FROM, " LEFT JOIN order_quotes o_quotes ON o.uid = o_quotes.order_uid", @@ -1201,6 +1215,8 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), + call_data: String::new(), + verified: false, }; insert_quote(&mut db, "e).await.unwrap(); insert_quote_and_update_on_conflict(&mut db, "e) @@ -1261,6 +1277,8 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), + call_data: String::new(), + verified: false, }; insert_quote(&mut db, "e).await.unwrap(); let quote_ = read_quote(&mut db, "e.order_uid) @@ -1287,6 +1305,8 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), + call_data: String::new(), + verified: false, }; insert_quote(&mut db, "e).await.unwrap(); let order_with_quote = single_full_order_with_quote(&mut db, "e.order_uid) diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index 2814664825..394d7babc8 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -100,6 +100,8 @@ impl OrderWithQuote { sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), + call_data: order.metadata.full_app_data.clone().unwrap_or_default(), + verified: quote.data.verified, }), order, } @@ -202,6 +204,7 @@ async fn insert_order(order: &Order, ex: &mut PgConnection) -> Result<(), Insert buy_token_balance: buy_token_destination_into(order.data.buy_token_balance), full_fee_amount: u256_to_big_decimal(&order.metadata.full_fee_amount), cancellation_timestamp: None, + call_data: order.metadata.full_app_data.clone().unwrap_or_default(), }; database::orders::insert_order(ex, &order) @@ -221,18 +224,20 @@ async fn insert_order(order: &Order, ex: &mut PgConnection) -> Result<(), Insert } async fn insert_quote( - uid: &OrderUid, + order: &Order, quote: &Quote, ex: &mut PgConnection, ) -> Result<(), InsertionError> { let quote = database::orders::Quote { - order_uid: ByteArray(uid.0), + order_uid: ByteArray(order.metadata.uid.0), gas_amount: quote.data.fee_parameters.gas_amount, gas_price: quote.data.fee_parameters.gas_price, sell_token_price: quote.data.fee_parameters.sell_token_price, sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), + call_data: order.metadata.full_app_data.clone().unwrap_or_default(), + verified: quote.data.verified, }; database::orders::insert_quote(ex, "e) .await @@ -258,7 +263,7 @@ impl OrderStoring for Postgres { insert_order(&order, &mut ex).await?; if let Some(quote) = quote { - insert_quote(&order.metadata.uid, "e, &mut ex).await?; + insert_quote(&order, "e, &mut ex).await?; } Self::insert_order_app_data(&order, &mut ex).await?; @@ -318,7 +323,7 @@ impl OrderStoring for Postgres { .await?; insert_order(&new_order, ex).await?; if let Some(quote) = new_quote { - insert_quote(&new_order.metadata.uid, "e, ex).await?; + insert_quote(&new_order, "e, ex).await?; } Self::insert_order_app_data(&new_order, ex).await?; @@ -356,12 +361,14 @@ impl OrderStoring for Postgres { let order = orders::single_full_order_with_quote(&mut ex, &ByteArray(uid.0)).await?; order .map(|order_with_quote| { + let call_data = order_with_quote.full_order.call_data.clone().unwrap_or_default(); let quote = match ( order_with_quote.quote_buy_amount, order_with_quote.quote_sell_amount, order_with_quote.quote_gas_amount, order_with_quote.quote_gas_price, order_with_quote.quote_sell_token_price, + order_with_quote.quote_verified, order_with_quote.solver, ) { ( @@ -370,6 +377,7 @@ impl OrderStoring for Postgres { Some(gas_amount), Some(gas_price), Some(sell_token_price), + Some(verified), Some(solver), ) => Some(orders::Quote { order_uid: order_with_quote.full_order.uid, @@ -379,6 +387,8 @@ impl OrderStoring for Postgres { sell_amount, buy_amount, solver, + call_data, + verified }), _ => None, }; @@ -708,6 +718,7 @@ mod tests { onchain_placement_error: None, executed_surplus_fee: Default::default(), full_app_data: Default::default(), + call_data: None, }; // Open - sell (filled - 0%) From d9eb1894f6b829c9176c9fd46ddefda964d9e79f Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 15 Nov 2024 01:38:05 +0100 Subject: [PATCH 02/70] Added call_data to quotes table --- crates/database/src/quotes.rs | 11 ++++++++-- crates/e2e/tests/e2e/quote_verification.rs | 1 + crates/shared/src/event_storing_helpers.rs | 1 + crates/shared/src/order_quoting.rs | 20 +++++++++++++++++++ crates/shared/src/price_estimation.rs | 5 +++-- .../src/price_estimation/competition/mod.rs | 10 +++++----- crates/shared/src/price_estimation/native.rs | 1 + .../shared/src/price_estimation/sanitized.rs | 14 +++++++++++++ .../src/price_estimation/trade_finder.rs | 1 + .../src/price_estimation/trade_verifier.rs | 3 +++ crates/shared/src/trade_finding.rs | 1 + crates/shared/src/trade_finding/external.rs | 2 ++ 12 files changed, 61 insertions(+), 9 deletions(-) diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 41f94e50da..bb651f1a10 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -34,6 +34,7 @@ pub struct Quote { pub expiration_timestamp: DateTime, pub quote_kind: QuoteKind, pub solver: Address, + pub call_data: Option>, } /// Stores the quote and returns the id. The id of the quote parameter is not @@ -51,9 +52,10 @@ INSERT INTO quotes ( order_kind, expiration_timestamp, quote_kind, - solver + solver, + call_data ) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING id "#; let (id,) = sqlx::query_as(QUERY) @@ -68,6 +70,7 @@ RETURNING id .bind(quote.expiration_timestamp) .bind("e.quote_kind) .bind(quote.solver) + .bind("e.call_data) .fetch_one(ex) .await?; Ok(id) @@ -181,6 +184,7 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Standard, solver: ByteArray([1; 20]), + call_data: None, }; let id = save(&mut db, "e).await.unwrap(); quote.id = id; @@ -214,6 +218,7 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Standard, solver: ByteArray([1; 20]), + call_data: None, }; let token_b = ByteArray([2; 20]); @@ -230,6 +235,7 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Standard, solver: ByteArray([2; 20]), + call_data: None, }; // Save two measurements for token_a @@ -401,6 +407,7 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Eip1271OnchainOrder, solver: ByteArray([1; 20]), + call_data: None, }; let id = save(&mut db, "e).await.unwrap(); quote.id = id; diff --git a/crates/e2e/tests/e2e/quote_verification.rs b/crates/e2e/tests/e2e/quote_verification.rs index cc9cf2c132..af34b4c0c5 100644 --- a/crates/e2e/tests/e2e/quote_verification.rs +++ b/crates/e2e/tests/e2e/quote_verification.rs @@ -109,6 +109,7 @@ async fn test_bypass_verification_for_rfq_quotes(web3: Web3) { gas: 225000, solver: H160::from_str("0xe3067c7c27c1038de4e8ad95a83b927d23dfbd99").unwrap(), verified: true, + call_data: None, }; // `tx_origin: 0x0000` is currently used to bypass quote verification due to an diff --git a/crates/shared/src/event_storing_helpers.rs b/crates/shared/src/event_storing_helpers.rs index 1f1938b2c6..a94e81e0a6 100644 --- a/crates/shared/src/event_storing_helpers.rs +++ b/crates/shared/src/event_storing_helpers.rs @@ -25,6 +25,7 @@ pub fn create_quote_row(data: QuoteData) -> DbQuote { expiration_timestamp: data.expiration, quote_kind: data.quote_kind, solver: ByteArray(data.solver.0), + call_data: data.call_data, } } diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index eac107f379..4a31c1243e 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -170,6 +170,7 @@ pub struct QuoteData { pub solver: H160, /// Were we able to verify that this quote is accurate? pub verified: bool, + pub call_data: Option>, } impl TryFrom for QuoteData { @@ -195,6 +196,7 @@ impl TryFrom for QuoteData { // Even if the quote was verified at the time of creation // it might no longer be accurate. verified: false, + call_data: row.call_data, }) } } @@ -442,6 +444,7 @@ impl OrderQuoter { quote_kind, solver: trade_estimate.solver, verified: trade_estimate.verified, + call_data: trade_estimate.call_data, }; Ok(quote) @@ -727,6 +730,7 @@ mod tests { gas: 3, solver: H160([1; 20]), verified: false, + call_data: None, }) } .boxed() @@ -768,6 +772,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, + call_data: None, })) .returning(|_| Ok(1337)); @@ -804,6 +809,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, + call_data: None, }, sell_amount: 70.into(), buy_amount: 29.into(), @@ -862,6 +868,7 @@ mod tests { gas: 3, solver: H160([1; 20]), verified: false, + call_data: None, }) } .boxed() @@ -903,6 +910,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, + call_data: None, })) .returning(|_| Ok(1337)); @@ -939,6 +947,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, + call_data: None, }, sell_amount: 100.into(), buy_amount: 42.into(), @@ -992,6 +1001,7 @@ mod tests { gas: 3, solver: H160([1; 20]), verified: false, + call_data: None, }) } .boxed() @@ -1033,6 +1043,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, + call_data: None, })) .returning(|_| Ok(1337)); @@ -1069,6 +1080,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, + call_data: None, }, sell_amount: 100.into(), buy_amount: 42.into(), @@ -1108,6 +1120,7 @@ mod tests { gas: 200, solver: H160([1; 20]), verified: false, + call_data: None, }) } .boxed() @@ -1179,6 +1192,7 @@ mod tests { gas: 200, solver: H160([1; 20]), verified: false, + call_data: None, }) } .boxed() @@ -1255,6 +1269,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, + call_data: None, })) }); @@ -1288,6 +1303,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, + call_data: None, }, sell_amount: 85.into(), // Allows for "out-of-price" buy amounts. This means that order @@ -1335,6 +1351,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, + call_data: None, })) }); @@ -1368,6 +1385,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, + call_data: None, }, sell_amount: 100.into(), buy_amount: 42.into(), @@ -1416,6 +1434,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, + call_data: None, }, ))) }); @@ -1450,6 +1469,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, + call_data: None, }, sell_amount: 100.into(), buy_amount: 42.into(), diff --git a/crates/shared/src/price_estimation.rs b/crates/shared/src/price_estimation.rs index 6a8fa88674..e7c8db6f62 100644 --- a/crates/shared/src/price_estimation.rs +++ b/crates/shared/src/price_estimation.rs @@ -456,7 +456,7 @@ pub struct Verification { pub buy_token_destination: BuyTokenDestination, } -#[derive(Copy, Clone, Debug, Default, Eq, PartialEq, Deserialize)] +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize)] pub struct Estimate { pub out_amount: U256, /// full gas cost when settling this order alone on gp @@ -465,6 +465,7 @@ pub struct Estimate { pub solver: H160, /// Did we verify the correctness of this estimate's properties? pub verified: bool, + pub call_data: Option>, } impl Estimate { @@ -551,7 +552,7 @@ pub mod mocks { pub struct FakePriceEstimator(pub Estimate); impl PriceEstimating for FakePriceEstimator { fn estimate(&self, _query: Arc) -> BoxFuture<'_, PriceEstimateResult> { - async { Ok(self.0) }.boxed() + async { Ok(self.0.clone()) }.boxed() } } diff --git a/crates/shared/src/price_estimation/competition/mod.rs b/crates/shared/src/price_estimation/competition/mod.rs index 2703e3397a..3800f53ddf 100644 --- a/crates/shared/src/price_estimation/competition/mod.rs +++ b/crates/shared/src/price_estimation/competition/mod.rs @@ -287,17 +287,17 @@ mod tests { }; let first = setup_estimator(vec![ - Ok(estimates[0]), - Ok(estimates[0]), - Ok(estimates[0]), + Ok(estimates[0].clone()), + Ok(estimates[0].clone()), + Ok(estimates[0].clone()), Err(PriceEstimationError::ProtocolInternal(anyhow!("a"))), Err(PriceEstimationError::NoLiquidity), ]); let second = setup_estimator(vec![ Err(PriceEstimationError::ProtocolInternal(anyhow!(""))), - Ok(estimates[1]), - Ok(estimates[1]), + Ok(estimates[1].clone()), + Ok(estimates[1].clone()), Err(PriceEstimationError::ProtocolInternal(anyhow!("b"))), Err(PriceEstimationError::UnsupportedToken { token: H160([0; 20]), diff --git a/crates/shared/src/price_estimation/native.rs b/crates/shared/src/price_estimation/native.rs index e0928ddb8d..910d671cff 100644 --- a/crates/shared/src/price_estimation/native.rs +++ b/crates/shared/src/price_estimation/native.rs @@ -130,6 +130,7 @@ mod tests { gas: 0, solver: H160([1; 20]), verified: false, + call_data: None, }) } .boxed() diff --git a/crates/shared/src/price_estimation/sanitized.rs b/crates/shared/src/price_estimation/sanitized.rs index 29c9f44aff..04195e702a 100644 --- a/crates/shared/src/price_estimation/sanitized.rs +++ b/crates/shared/src/price_estimation/sanitized.rs @@ -67,6 +67,7 @@ impl PriceEstimating for SanitizedPriceEstimator { gas: 0, solver: Default::default(), verified: true, + call_data: None, }; tracing::debug!(?query, ?estimation, "generate trivial price estimation"); return Ok(estimation); @@ -79,6 +80,7 @@ impl PriceEstimating for SanitizedPriceEstimator { gas: GAS_PER_WETH_UNWRAP, solver: Default::default(), verified: true, + call_data: None, }; tracing::debug!(?query, ?estimation, "generate trivial unwrap estimation"); return Ok(estimation); @@ -91,6 +93,7 @@ impl PriceEstimating for SanitizedPriceEstimator { gas: GAS_PER_WETH_WRAP, solver: Default::default(), verified: true, + call_data: None, }; tracing::debug!(?query, ?estimation, "generate trivial wrap estimation"); return Ok(estimation); @@ -185,6 +188,7 @@ mod tests { gas: 100, solver: Default::default(), verified: false, + call_data: None, }), ), // `sanitized_estimator` will replace `buy_token` with `native_token` before querying @@ -206,6 +210,7 @@ mod tests { gas: GAS_PER_WETH_UNWRAP + 100, solver: Default::default(), verified: false, + call_data: None, }), ), // Will cause buffer overflow of gas price in `sanitized_estimator`. @@ -241,6 +246,7 @@ mod tests { gas: GAS_PER_WETH_WRAP + 100, solver: Default::default(), verified: false, + call_data: None, }), ), // Can be estimated by `sanitized_estimator` because `buy_token` and `sell_token` are @@ -259,6 +265,7 @@ mod tests { gas: 0, solver: Default::default(), verified: true, + call_data: None, }), ), // Can be estimated by `sanitized_estimator` because both tokens are the native token. @@ -276,6 +283,7 @@ mod tests { gas: 0, solver: Default::default(), verified: true, + call_data: None, }), ), // Can be estimated by `sanitized_estimator` because it is a native token unwrap. @@ -294,6 +302,7 @@ mod tests { gas: GAS_PER_WETH_UNWRAP, solver: Default::default(), verified: true, + call_data: None, }), ), // Can be estimated by `sanitized_estimator` because it is a native token wrap. @@ -312,6 +321,7 @@ mod tests { gas: GAS_PER_WETH_WRAP, solver: Default::default(), verified: true, + call_data: None, }), ), // Will throw `UnsupportedToken` error in `sanitized_estimator`. @@ -376,6 +386,7 @@ mod tests { gas: 100, solver: Default::default(), verified: false, + call_data: None, }) } .boxed() @@ -391,6 +402,7 @@ mod tests { gas: 100, solver: Default::default(), verified: false, + call_data: None, }) } .boxed() @@ -406,6 +418,7 @@ mod tests { gas: u64::MAX, solver: Default::default(), verified: false, + call_data: None, }) } .boxed() @@ -421,6 +434,7 @@ mod tests { gas: 100, solver: Default::default(), verified: false, + call_data: None, }) } .boxed() diff --git a/crates/shared/src/price_estimation/trade_finder.rs b/crates/shared/src/price_estimation/trade_finder.rs index 8bbe471f36..4f200fe86c 100644 --- a/crates/shared/src/price_estimation/trade_finder.rs +++ b/crates/shared/src/price_estimation/trade_finder.rs @@ -89,6 +89,7 @@ impl Inner { gas: quote.gas_estimate, solver: quote.solver, verified: false, + call_data: quote.call_data, }) } } diff --git a/crates/shared/src/price_estimation/trade_verifier.rs b/crates/shared/src/price_estimation/trade_verifier.rs index 258e928ef3..603686f3f2 100644 --- a/crates/shared/src/price_estimation/trade_verifier.rs +++ b/crates/shared/src/price_estimation/trade_verifier.rs @@ -179,6 +179,7 @@ impl TradeVerifier { gas: trade.gas_estimate.context("no gas estimate")?, solver: trade.solver, verified: true, + call_data: None, }; tracing::warn!( ?estimate, @@ -312,6 +313,7 @@ impl TradeVerifying for TradeVerifier { gas, solver: trade.solver, verified: false, + call_data: None, }; tracing::warn!( ?err, @@ -528,6 +530,7 @@ fn ensure_quote_accuracy( gas: summary.gas_used.as_u64(), solver, verified: true, + call_data: None, }) } diff --git a/crates/shared/src/trade_finding.rs b/crates/shared/src/trade_finding.rs index 3e5e8bfdf4..e4a9d07140 100644 --- a/crates/shared/src/trade_finding.rs +++ b/crates/shared/src/trade_finding.rs @@ -30,6 +30,7 @@ pub struct Quote { pub out_amount: U256, pub gas_estimate: u64, pub solver: H160, + pub call_data: Option>, } /// A trade. diff --git a/crates/shared/src/trade_finding/external.rs b/crates/shared/src/trade_finding/external.rs index 60784e2e4b..24a8c22c0b 100644 --- a/crates/shared/src/trade_finding/external.rs +++ b/crates/shared/src/trade_finding/external.rs @@ -166,10 +166,12 @@ impl TradeFinding for ExternalTradeFinder { .gas_estimate .context("no gas estimate") .map_err(TradeError::Other)?; + let first_interaction = trade.interactions.first().cloned(); Ok(Quote { out_amount: trade.out_amount, gas_estimate, solver: trade.solver, + call_data: first_interaction.map(|interaction| interaction.data), }) } From 2c0e92e06f61c5a945902081f76bad6360d238ae Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 15 Nov 2024 01:57:11 +0100 Subject: [PATCH 03/70] Updated order_quotes table --- .../src/database/onchain_order_events.rs | 7 ++----- crates/database/src/orders.rs | 20 +++++++++---------- crates/orderbook/src/database/orders.rs | 9 ++++----- 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/crates/autopilot/src/database/onchain_order_events.rs b/crates/autopilot/src/database/onchain_order_events.rs index 136211f7b6..77f197802c 100644 --- a/crates/autopilot/src/database/onchain_order_events.rs +++ b/crates/autopilot/src/database/onchain_order_events.rs @@ -487,7 +487,7 @@ async fn parse_general_onchain_order_placement_data<'a>( sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - call_data: order_data.1.call_data.clone(), + call_data: quote.data.call_data, verified: quote.data.verified, }), Err(err) => { @@ -622,7 +622,6 @@ fn convert_onchain_order_placement( true => OrderClass::Limit, false => OrderClass::Market, }, - call_data: String::new(), }; let onchain_order_placement_event = OnchainOrderPlacement { order_uid: ByteArray(order_uid.0), @@ -928,7 +927,6 @@ mod test { buy_token_balance: buy_token_destination_into(expected_order_data.buy_token_balance), full_fee_amount: u256_to_big_decimal(&expected_order_data.fee_amount), cancellation_timestamp: None, - call_data: String::new(), }; assert_eq!(onchain_order_placement, expected_onchain_order_placement); assert_eq!(order, expected_order); @@ -1040,7 +1038,6 @@ mod test { buy_token_balance: buy_token_destination_into(expected_order_data.buy_token_balance), full_fee_amount: u256_to_big_decimal(&U256::zero()), cancellation_timestamp: None, - call_data: String::new(), }; assert_eq!(onchain_order_placement, expected_onchain_order_placement); assert_eq!(order, expected_order); @@ -1192,7 +1189,7 @@ mod test { sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - call_data: String::new(), + call_data: None, verified: quote.data.verified, }; assert_eq!(result.1, vec![Some(expected_quote)]); diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 57542c347c..9b55e196dd 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -99,7 +99,6 @@ pub struct Order { pub full_fee_amount: BigDecimal, pub cancellation_timestamp: Option>, pub class: OrderClass, - pub call_data: String, } pub async fn insert_orders_and_ignore_conflicts( @@ -143,10 +142,9 @@ INSERT INTO orders ( buy_token_balance, full_fee_amount, cancellation_timestamp, - class, - call_data + class ) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21) "#; pub async fn insert_order_and_ignore_conflicts( @@ -188,7 +186,6 @@ async fn insert_order_execute_sqlx( .bind(&order.full_fee_amount) .bind(order.cancellation_timestamp) .bind(order.class) - .bind(&order.call_data) .execute(ex) .await?; Ok(()) @@ -332,7 +329,7 @@ pub struct Quote { pub sell_amount: BigDecimal, pub buy_amount: BigDecimal, pub solver: Address, - pub call_data: String, + pub call_data: Option>, pub verified: bool, } @@ -498,7 +495,6 @@ pub struct FullOrder { pub onchain_placement_error: Option, pub executed_surplus_fee: BigDecimal, pub full_app_data: Option>, - pub call_data: Option, } #[derive(Debug, sqlx::FromRow)] @@ -511,6 +507,7 @@ pub struct FullOrderWithQuote { pub quote_gas_price: Option, pub quote_sell_token_price: Option, pub quote_verified: Option, + pub quote_call_data: Option>>, pub solver: Option
, } @@ -551,7 +548,7 @@ pub const SELECT: &str = r#" o.uid, o.owner, o.creation_timestamp, o.sell_token, o.buy_token, o.sell_amount, o.buy_amount, o.valid_to, o.app_data, o.fee_amount, o.full_fee_amount, o.kind, o.partially_fillable, o.signature, o.receiver, o.signing_scheme, o.settlement_contract, o.sell_token_balance, o.buy_token_balance, -o.class, o.call_data, +o.class, (SELECT COALESCE(SUM(t.buy_amount), 0) FROM trades t WHERE t.order_uid = o.uid) AS sum_buy, (SELECT COALESCE(SUM(t.sell_amount), 0) FROM trades t WHERE t.order_uid = o.uid) AS sum_sell, (SELECT COALESCE(SUM(t.fee_amount), 0) FROM trades t WHERE t.order_uid = o.uid) AS sum_fee, @@ -605,6 +602,7 @@ pub async fn single_full_order_with_quote( ", o_quotes.gas_price as quote_gas_price", ", o_quotes.sell_token_price as quote_sell_token_price", ", o_quotes.verified as quote_verified", + ", o_quotes.call_data as quote_call_data", ", o_quotes.solver as solver", " FROM ", FROM, " LEFT JOIN order_quotes o_quotes ON o.uid = o_quotes.order_uid", @@ -1215,7 +1213,7 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), - call_data: String::new(), + call_data: None, verified: false, }; insert_quote(&mut db, "e).await.unwrap(); @@ -1277,7 +1275,7 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), - call_data: String::new(), + call_data: None, verified: false, }; insert_quote(&mut db, "e).await.unwrap(); @@ -1305,7 +1303,7 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), - call_data: String::new(), + call_data: None, verified: false, }; insert_quote(&mut db, "e).await.unwrap(); diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index 394d7babc8..09ffc88190 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -100,7 +100,7 @@ impl OrderWithQuote { sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - call_data: order.metadata.full_app_data.clone().unwrap_or_default(), + call_data: quote.data.call_data, verified: quote.data.verified, }), order, @@ -204,7 +204,6 @@ async fn insert_order(order: &Order, ex: &mut PgConnection) -> Result<(), Insert buy_token_balance: buy_token_destination_into(order.data.buy_token_balance), full_fee_amount: u256_to_big_decimal(&order.metadata.full_fee_amount), cancellation_timestamp: None, - call_data: order.metadata.full_app_data.clone().unwrap_or_default(), }; database::orders::insert_order(ex, &order) @@ -236,7 +235,7 @@ async fn insert_quote( sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - call_data: order.metadata.full_app_data.clone().unwrap_or_default(), + call_data: quote.data.call_data.clone(), verified: quote.data.verified, }; database::orders::insert_quote(ex, "e) @@ -361,7 +360,6 @@ impl OrderStoring for Postgres { let order = orders::single_full_order_with_quote(&mut ex, &ByteArray(uid.0)).await?; order .map(|order_with_quote| { - let call_data = order_with_quote.full_order.call_data.clone().unwrap_or_default(); let quote = match ( order_with_quote.quote_buy_amount, order_with_quote.quote_sell_amount, @@ -369,6 +367,7 @@ impl OrderStoring for Postgres { order_with_quote.quote_gas_price, order_with_quote.quote_sell_token_price, order_with_quote.quote_verified, + order_with_quote.quote_call_data, order_with_quote.solver, ) { ( @@ -378,6 +377,7 @@ impl OrderStoring for Postgres { Some(gas_price), Some(sell_token_price), Some(verified), + Some(call_data), Some(solver), ) => Some(orders::Quote { order_uid: order_with_quote.full_order.uid, @@ -718,7 +718,6 @@ mod tests { onchain_placement_error: None, executed_surplus_fee: Default::default(), full_app_data: Default::default(), - call_data: None, }; // Open - sell (filled - 0%) From 006a873fc035570deefc960421b591331a5c60b0 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 15 Nov 2024 02:06:14 +0100 Subject: [PATCH 04/70] Added comments --- crates/shared/src/order_quoting.rs | 1 + crates/shared/src/price_estimation.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index 4a31c1243e..c9dc798981 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -170,6 +170,7 @@ pub struct QuoteData { pub solver: H160, /// Were we able to verify that this quote is accurate? pub verified: bool, + /// Data provided by the solver in response to /quote request. pub call_data: Option>, } diff --git a/crates/shared/src/price_estimation.rs b/crates/shared/src/price_estimation.rs index e7c8db6f62..b27a9befdc 100644 --- a/crates/shared/src/price_estimation.rs +++ b/crates/shared/src/price_estimation.rs @@ -465,6 +465,7 @@ pub struct Estimate { pub solver: H160, /// Did we verify the correctness of this estimate's properties? pub verified: bool, + /// Data provided by the solver in response to /quote request. pub call_data: Option>, } From 7d2bb015bd97fdd0d7960f333b77e3f57f068af8 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 15 Nov 2024 02:22:17 +0100 Subject: [PATCH 05/70] Added db migration --- database/README.md | 3 +++ .../V075__add_call_data_to_quotes_and_order_quotes.sql | 8 ++++++++ 2 files changed, 11 insertions(+) create mode 100644 database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql diff --git a/database/README.md b/database/README.md index 0a9e8a0bcb..fbe8ee61a9 100644 --- a/database/README.md +++ b/database/README.md @@ -238,6 +238,8 @@ Quotes that an order was created with. These quotes get stored persistently and sell\_amount | numeric | not null | sell\_amount of the quote used to create the order with buy\_amount | numeric | not null | buy\_amount of the quote used to create the order with solver | bytea | not null | public address of the solver that provided this quote + call\_data | bytea | | call\_data provided by solver in response to the /quote request + verified | boolean | | information if quote was verified Indexes: - PRIMARY KEY: btree(`order_uid`) @@ -339,6 +341,7 @@ Stores quotes in order to determine whether it makes sense to allow a user to cr id | bigint | not null | unique identifier of this quote quote\_kind | [enum](#quotekind) | not null | quotekind for which this quote is considered valid solver | bytea | not null | public address of the solver that provided this quote + call\_data | bytea | | call\_data provided by solver in response to the /quote request Indexes: - PRIMARY KEY: btree(`id`) diff --git a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql new file mode 100644 index 0000000000..ccb293afe2 --- /dev/null +++ b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql @@ -0,0 +1,8 @@ +-- Step 1: Add a new column to the quotes table +ALTER TABLE quotes + ADD COLUMN call_data bytea; + +-- Step 2: Add two new columns to the order_quotes table +ALTER TABLE order_quotes + ADD COLUMN call_data bytea, + ADD COLUMN verified boolean; From 994725755ef4252a1e7b88248d3a194b81c61c25 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 15 Nov 2024 02:25:05 +0100 Subject: [PATCH 06/70] Fixed formatting --- crates/database/src/orders.rs | 2 +- crates/orderbook/src/database/orders.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 9b55e196dd..27096dc68a 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -329,7 +329,7 @@ pub struct Quote { pub sell_amount: BigDecimal, pub buy_amount: BigDecimal, pub solver: Address, - pub call_data: Option>, + pub call_data: Option>, pub verified: bool, } diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index 09ffc88190..93c9aa0c3f 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -388,7 +388,7 @@ impl OrderStoring for Postgres { buy_amount, solver, call_data, - verified + verified, }), _ => None, }; From 692d94eda974080c08ea5d56c34a06928c4acde0 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 15 Nov 2024 14:51:32 +0100 Subject: [PATCH 07/70] Fixed sql query --- crates/database/src/orders.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 27096dc68a..c7555c38ff 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -350,7 +350,7 @@ INSERT INTO order_quotes ( buy_amount, solver, call_data, - verified, + verified ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"#; From 6fb1ae72ef1ea3c5e0155df6e0f0aadcb88e8d53 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 15 Nov 2024 22:17:12 +0100 Subject: [PATCH 08/70] Updated db querying for order with quote --- crates/database/src/orders.rs | 2 +- crates/orderbook/src/database/orders.rs | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index c7555c38ff..88d180be2e 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -507,7 +507,7 @@ pub struct FullOrderWithQuote { pub quote_gas_price: Option, pub quote_sell_token_price: Option, pub quote_verified: Option, - pub quote_call_data: Option>>, + pub quote_call_data: Option>, pub solver: Option
, } diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index 93c9aa0c3f..ac999e5678 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -367,7 +367,6 @@ impl OrderStoring for Postgres { order_with_quote.quote_gas_price, order_with_quote.quote_sell_token_price, order_with_quote.quote_verified, - order_with_quote.quote_call_data, order_with_quote.solver, ) { ( @@ -377,7 +376,6 @@ impl OrderStoring for Postgres { Some(gas_price), Some(sell_token_price), Some(verified), - Some(call_data), Some(solver), ) => Some(orders::Quote { order_uid: order_with_quote.full_order.uid, @@ -387,7 +385,7 @@ impl OrderStoring for Postgres { sell_amount, buy_amount, solver, - call_data, + call_data: order_with_quote.quote_call_data, verified, }), _ => None, From a8f349f4d5cc759afc41c00d9d4e895e42d16886 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 15 Nov 2024 22:37:19 +0100 Subject: [PATCH 09/70] Added test --- crates/orderbook/src/database/orders.rs | 45 +++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index ac999e5678..edffc2c872 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -676,6 +676,7 @@ mod tests { signature::{Signature, SigningScheme}, }, primitive_types::U256, + shared::order_quoting::QuoteData, std::sync::atomic::{AtomicI64, Ordering}, }; @@ -1178,4 +1179,48 @@ mod tests { u256_to_big_decimal("e.buy_amount) ); } + + #[tokio::test] + #[ignore] + async fn postgres_insert_orders_with_call_data_and_verified() { + let db = Postgres::new("postgresql://").unwrap(); + database::clear_DANGER(&db.pool).await.unwrap(); + + let uid = OrderUid([0x42; 56]); + let call_data = vec![1]; + let order = Order { + data: OrderData { + valid_to: u32::MAX, + ..Default::default() + }, + metadata: OrderMetadata { + uid, + ..Default::default() + }, + ..Default::default() + }; + + let quote = Quote { + id: Some(5), + sell_amount: U256::from(1), + buy_amount: U256::from(2), + data: QuoteData { + call_data: Some(call_data.clone()), + verified: true, + ..Default::default() + }, + ..Default::default() + }; + db.insert_order(&order, Some(quote)).await.unwrap(); + + let single_order_with_quote = db.single_order_with_quote(&uid).await.unwrap().unwrap(); + assert_eq!(single_order_with_quote.order, order); + assert_eq!( + single_order_with_quote.quote.clone().unwrap().call_data, + Some(call_data) + ); + assert!( + single_order_with_quote.quote.unwrap().verified, + ); + } } From 67c20ffedfa1d4639d5e9fbf3747750628bc22e7 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 15 Nov 2024 22:41:59 +0100 Subject: [PATCH 10/70] Fixed formatting --- crates/orderbook/src/database/orders.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index edffc2c872..d9daa2bffe 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -1219,8 +1219,6 @@ mod tests { single_order_with_quote.quote.clone().unwrap().call_data, Some(call_data) ); - assert!( - single_order_with_quote.quote.unwrap().verified, - ); + assert!(single_order_with_quote.quote.unwrap().verified,); } } From e9026a4022506ab08610a5f734ceafae6847d0f6 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 15 Nov 2024 23:22:35 +0100 Subject: [PATCH 11/70] Updated tests --- crates/shared/src/order_quoting.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index c9dc798981..247662739d 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -1270,7 +1270,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - call_data: None, + call_data: Some(vec![1]), })) }); @@ -1304,7 +1304,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - call_data: None, + call_data: Some(vec![1]), }, sell_amount: 85.into(), // Allows for "out-of-price" buy amounts. This means that order @@ -1434,8 +1434,8 @@ mod tests { expiration: now + chrono::Duration::seconds(10), quote_kind: QuoteKind::Standard, solver: H160([1; 20]), - verified: false, - call_data: None, + verified: true, + call_data: Some(vec![1]), }, ))) }); @@ -1469,8 +1469,8 @@ mod tests { expiration: now + chrono::Duration::seconds(10), quote_kind: QuoteKind::Standard, solver: H160([1; 20]), - verified: false, - call_data: None, + verified: true, + call_data: Some(vec![1]), }, sell_amount: 100.into(), buy_amount: 42.into(), From 66717673f6b84d5c94c668e81621073bfdbf88e9 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Thu, 21 Nov 2024 10:38:51 +0100 Subject: [PATCH 12/70] Added verified column to quotes table --- crates/database/src/quotes.rs | 11 +++++++++-- crates/shared/src/event_storing_helpers.rs | 1 + database/README.md | 1 + ...V075__add_call_data_to_quotes_and_order_quotes.sql | 3 ++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index bb651f1a10..455be0f3be 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -35,6 +35,7 @@ pub struct Quote { pub quote_kind: QuoteKind, pub solver: Address, pub call_data: Option>, + pub verified: bool, } /// Stores the quote and returns the id. The id of the quote parameter is not @@ -53,9 +54,10 @@ INSERT INTO quotes ( expiration_timestamp, quote_kind, solver, - call_data + call_data, + verified ) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING id "#; let (id,) = sqlx::query_as(QUERY) @@ -71,6 +73,7 @@ RETURNING id .bind("e.quote_kind) .bind(quote.solver) .bind("e.call_data) + .bind(quote.verified) .fetch_one(ex) .await?; Ok(id) @@ -185,6 +188,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: ByteArray([1; 20]), call_data: None, + verified: false, }; let id = save(&mut db, "e).await.unwrap(); quote.id = id; @@ -219,6 +223,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: ByteArray([1; 20]), call_data: None, + verified: false, }; let token_b = ByteArray([2; 20]); @@ -236,6 +241,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: ByteArray([2; 20]), call_data: None, + verified: false, }; // Save two measurements for token_a @@ -408,6 +414,7 @@ mod tests { quote_kind: QuoteKind::Eip1271OnchainOrder, solver: ByteArray([1; 20]), call_data: None, + verified: false, }; let id = save(&mut db, "e).await.unwrap(); quote.id = id; diff --git a/crates/shared/src/event_storing_helpers.rs b/crates/shared/src/event_storing_helpers.rs index a94e81e0a6..e2f681488c 100644 --- a/crates/shared/src/event_storing_helpers.rs +++ b/crates/shared/src/event_storing_helpers.rs @@ -26,6 +26,7 @@ pub fn create_quote_row(data: QuoteData) -> DbQuote { quote_kind: data.quote_kind, solver: ByteArray(data.solver.0), call_data: data.call_data, + verified: data.verified, } } diff --git a/database/README.md b/database/README.md index fbe8ee61a9..171561a6a6 100644 --- a/database/README.md +++ b/database/README.md @@ -342,6 +342,7 @@ Stores quotes in order to determine whether it makes sense to allow a user to cr quote\_kind | [enum](#quotekind) | not null | quotekind for which this quote is considered valid solver | bytea | not null | public address of the solver that provided this quote call\_data | bytea | | call\_data provided by solver in response to the /quote request + verified | boolean | | information if quote was verified Indexes: - PRIMARY KEY: btree(`id`) diff --git a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql index ccb293afe2..743e795e36 100644 --- a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql +++ b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql @@ -1,6 +1,7 @@ -- Step 1: Add a new column to the quotes table ALTER TABLE quotes - ADD COLUMN call_data bytea; + ADD COLUMN call_data bytea, + ADD COLUMN verified boolean; -- Step 2: Add two new columns to the order_quotes table ALTER TABLE order_quotes From 086e579cc2c6dc71711935a8a05be5bcd3440c87 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Sat, 23 Nov 2024 01:18:53 +0100 Subject: [PATCH 13/70] Added quotes_interactions table --- crates/autopilot/src/database/auction.rs | 12 ++- .../src/database/onchain_order_events.rs | 2 - crates/database/src/orders.rs | 11 +-- crates/database/src/quotes.rs | 87 +++++++++++++++++-- crates/e2e/tests/e2e/quote_verification.rs | 2 +- crates/orderbook/src/database/orders.rs | 4 - crates/orderbook/src/database/quotes.rs | 12 ++- crates/shared/src/event_storing_helpers.rs | 29 ++++++- crates/shared/src/order_quoting.rs | 40 ++++----- crates/shared/src/price_estimation.rs | 9 +- .../shared/src/price_estimation/sanitized.rs | 28 +++--- .../src/price_estimation/trade_finder.rs | 2 +- .../src/price_estimation/trade_verifier.rs | 14 +-- crates/shared/src/trade_finding.rs | 18 +++- crates/shared/src/trade_finding/external.rs | 3 +- ...d_call_data_to_quotes_and_order_quotes.sql | 16 +++- 16 files changed, 208 insertions(+), 81 deletions(-) diff --git a/crates/autopilot/src/database/auction.rs b/crates/autopilot/src/database/auction.rs index 30c4b037e4..8e254483fa 100644 --- a/crates/autopilot/src/database/auction.rs +++ b/crates/autopilot/src/database/auction.rs @@ -8,7 +8,11 @@ use { num::ToPrimitive, shared::{ db_order_conversions::full_order_into_model_order, - event_storing_helpers::{create_db_search_parameters, create_quote_row}, + event_storing_helpers::{ + create_db_search_parameters, + create_quote_interactions_insert_data, + create_quote_row, + }, order_quoting::{QuoteData, QuoteSearchParameters, QuoteStoring}, }, std::{collections::HashMap, ops::DerefMut}, @@ -23,8 +27,12 @@ impl QuoteStoring for Postgres { .start_timer(); let mut ex = self.pool.acquire().await?; - let row = create_quote_row(data); + let row = create_quote_row(&data); let id = database::quotes::save(&mut ex, &row).await?; + if !data.interactions.is_empty() { + let interactions = create_quote_interactions_insert_data(id, &data); + database::quotes::insert_quote_interactions(&mut ex, &interactions).await?; + } Ok(id) } diff --git a/crates/autopilot/src/database/onchain_order_events.rs b/crates/autopilot/src/database/onchain_order_events.rs index 77f197802c..ec7cb4ee59 100644 --- a/crates/autopilot/src/database/onchain_order_events.rs +++ b/crates/autopilot/src/database/onchain_order_events.rs @@ -487,7 +487,6 @@ async fn parse_general_onchain_order_placement_data<'a>( sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - call_data: quote.data.call_data, verified: quote.data.verified, }), Err(err) => { @@ -1189,7 +1188,6 @@ mod test { sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - call_data: None, verified: quote.data.verified, }; assert_eq!(result.1, vec![Some(expected_quote)]); diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 88d180be2e..b8db7c1da6 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -329,7 +329,6 @@ pub struct Quote { pub sell_amount: BigDecimal, pub buy_amount: BigDecimal, pub solver: Address, - pub call_data: Option>, pub verified: bool, } @@ -349,10 +348,9 @@ INSERT INTO order_quotes ( sell_amount, buy_amount, solver, - call_data, verified ) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"#; +VALUES ($1, $2, $3, $4, $5, $6, $7, $8)"#; pub async fn insert_quote_and_update_on_conflict( ex: &mut PgConnection, @@ -366,7 +364,7 @@ pub async fn insert_quote_and_update_on_conflict( " ON CONFLICT (order_uid) DO UPDATE SET gas_amount = $2, gas_price = $3, sell_token_price = $4, sell_amount = $5, -buy_amount = $6, call_data = $8, verified = $9 +buy_amount = $6, verified = $8 " ); sqlx::query(QUERY) @@ -377,7 +375,6 @@ buy_amount = $6, call_data = $8, verified = $9 .bind("e.sell_amount) .bind("e.buy_amount) .bind(quote.solver) - .bind("e.call_data) .bind(quote.verified) .execute(ex) .await?; @@ -393,7 +390,6 @@ pub async fn insert_quote(ex: &mut PgConnection, quote: &Quote) -> Result<(), sq .bind("e.sell_amount) .bind("e.buy_amount) .bind(quote.solver) - .bind("e.call_data) .bind(quote.verified) .execute(ex) .await?; @@ -1213,7 +1209,6 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), - call_data: None, verified: false, }; insert_quote(&mut db, "e).await.unwrap(); @@ -1275,7 +1270,6 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), - call_data: None, verified: false, }; insert_quote(&mut db, "e).await.unwrap(); @@ -1303,7 +1297,6 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), - call_data: None, verified: false, }; insert_quote(&mut db, "e).await.unwrap(); diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 455be0f3be..6c9b2eb7d2 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -34,7 +34,6 @@ pub struct Quote { pub expiration_timestamp: DateTime, pub quote_kind: QuoteKind, pub solver: Address, - pub call_data: Option>, pub verified: bool, } @@ -54,10 +53,9 @@ INSERT INTO quotes ( expiration_timestamp, quote_kind, solver, - call_data, verified ) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) RETURNING id "#; let (id,) = sqlx::query_as(QUERY) @@ -72,7 +70,6 @@ RETURNING id .bind(quote.expiration_timestamp) .bind("e.quote_kind) .bind(quote.solver) - .bind("e.call_data) .bind(quote.verified) .fetch_one(ex) .await?; @@ -150,6 +147,52 @@ WHERE expiration_timestamp < $1 .map(|_| ()) } +/// One row in the `quotes_interactions` table. +#[derive(Clone, Debug, PartialEq, sqlx::FromRow)] +pub struct QuoteInteraction { + pub id: QuoteId, + pub index: i64, + pub target: Address, + pub value: BigDecimal, + pub call_data: Vec, +} + +/// Stores interactions provided by the solver for quote. +pub async fn insert_quote_interaction( + ex: &mut PgConnection, + quote_interaction: &QuoteInteraction, +) -> Result<(), sqlx::Error> { + const QUERY: &str = r#" +INSERT INTO quotes_interactions ( + order_uid, + index + target, + value, + call_data +) +VALUES ($1, $2, $3, $4) + "#; + sqlx::query(QUERY) + .bind(quote_interaction.id) + .bind(quote_interaction.index) + .bind(quote_interaction.target) + .bind("e_interaction.value) + .bind("e_interaction.call_data) + .execute(ex) + .await?; + Ok(()) +} + +pub async fn insert_quote_interactions( + ex: &mut PgConnection, + quote_interactions: &[QuoteInteraction], +) -> Result<(), sqlx::Error> { + for interaction in quote_interactions { + insert_quote_interaction(ex, interaction).await?; + } + Ok(()) +} + #[cfg(test)] mod tests { use { @@ -187,7 +230,6 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Standard, solver: ByteArray([1; 20]), - call_data: None, verified: false, }; let id = save(&mut db, "e).await.unwrap(); @@ -222,7 +264,6 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Standard, solver: ByteArray([1; 20]), - call_data: None, verified: false, }; @@ -240,7 +281,6 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Standard, solver: ByteArray([2; 20]), - call_data: None, verified: false, }; @@ -413,7 +453,6 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Eip1271OnchainOrder, solver: ByteArray([1; 20]), - call_data: None, verified: false, }; let id = save(&mut db, "e).await.unwrap(); @@ -436,4 +475,36 @@ mod tests { search_a.quote_kind = QuoteKind::Standard; assert_eq!(find(&mut db, &search_a).await.unwrap(), None,); } + + #[tokio::test] + #[ignore] + async fn postgres_insert_quote_interaction_by_id() { + let mut db = PgConnection::connect("postgresql://").await.unwrap(); + let mut db = db.begin().await.unwrap(); + crate::clear_DANGER_(&mut db).await.unwrap(); + + let quote_interaction = QuoteInteraction { + id: Default::default(), + index: Default::default(), + target: ByteArray([1; 20]), + value: 2.into(), + call_data: vec![3; 20], + }; + insert_quote_interaction(&mut db, "e_interaction) + .await + .unwrap(); + + const QUERY: &str = r#" + SELECT * + FROM quotes_interactions + WHERE order_uid = $1 + "#; + let interaction: Option = sqlx::query_as(QUERY) + .bind(quote_interaction.id) + .fetch_optional(&mut db as &mut PgConnection) + .await + .unwrap(); + + assert_eq!(interaction.unwrap(), quote_interaction); + } } diff --git a/crates/e2e/tests/e2e/quote_verification.rs b/crates/e2e/tests/e2e/quote_verification.rs index af34b4c0c5..2c94e604d3 100644 --- a/crates/e2e/tests/e2e/quote_verification.rs +++ b/crates/e2e/tests/e2e/quote_verification.rs @@ -109,7 +109,7 @@ async fn test_bypass_verification_for_rfq_quotes(web3: Web3) { gas: 225000, solver: H160::from_str("0xe3067c7c27c1038de4e8ad95a83b927d23dfbd99").unwrap(), verified: true, - call_data: None, + interactions: vec![], }; // `tx_origin: 0x0000` is currently used to bypass quote verification due to an diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index d9daa2bffe..887cfab122 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -100,7 +100,6 @@ impl OrderWithQuote { sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - call_data: quote.data.call_data, verified: quote.data.verified, }), order, @@ -235,7 +234,6 @@ async fn insert_quote( sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - call_data: quote.data.call_data.clone(), verified: quote.data.verified, }; database::orders::insert_quote(ex, "e) @@ -385,7 +383,6 @@ impl OrderStoring for Postgres { sell_amount, buy_amount, solver, - call_data: order_with_quote.quote_call_data, verified, }), _ => None, @@ -1205,7 +1202,6 @@ mod tests { sell_amount: U256::from(1), buy_amount: U256::from(2), data: QuoteData { - call_data: Some(call_data.clone()), verified: true, ..Default::default() }, diff --git a/crates/orderbook/src/database/quotes.rs b/crates/orderbook/src/database/quotes.rs index 0cb149cc86..99a8bca3d0 100644 --- a/crates/orderbook/src/database/quotes.rs +++ b/crates/orderbook/src/database/quotes.rs @@ -4,7 +4,11 @@ use { chrono::{DateTime, Utc}, model::quote::QuoteId, shared::{ - event_storing_helpers::{create_db_search_parameters, create_quote_row}, + event_storing_helpers::{ + create_db_search_parameters, + create_quote_interactions_insert_data, + create_quote_row, + }, order_quoting::{QuoteData, QuoteSearchParameters, QuoteStoring}, }, }; @@ -18,8 +22,12 @@ impl QuoteStoring for Postgres { .start_timer(); let mut ex = self.pool.acquire().await?; - let row = create_quote_row(data); + let row = create_quote_row(&data); let id = database::quotes::save(&mut ex, &row).await?; + if !data.interactions.is_empty() { + let interactions = create_quote_interactions_insert_data(id, &data); + database::quotes::insert_quote_interactions(&mut ex, &interactions).await?; + } Ok(id) } diff --git a/crates/shared/src/event_storing_helpers.rs b/crates/shared/src/event_storing_helpers.rs index e2f681488c..28f829391f 100644 --- a/crates/shared/src/event_storing_helpers.rs +++ b/crates/shared/src/event_storing_helpers.rs @@ -6,12 +6,17 @@ use { chrono::{DateTime, Utc}, database::{ byte_array::ByteArray, - quotes::{Quote as DbQuote, QuoteSearchParameters as DbQuoteSearchParameters}, + quotes::{ + Quote as DbQuote, + QuoteId, + QuoteInteraction as DbQuoteInteraction, + QuoteSearchParameters as DbQuoteSearchParameters, + }, }, number::conversions::u256_to_big_decimal, }; -pub fn create_quote_row(data: QuoteData) -> DbQuote { +pub fn create_quote_row(data: &QuoteData) -> DbQuote { DbQuote { id: Default::default(), sell_token: ByteArray(data.sell_token.0), @@ -23,13 +28,29 @@ pub fn create_quote_row(data: QuoteData) -> DbQuote { sell_token_price: data.fee_parameters.sell_token_price, order_kind: order_kind_into(data.kind), expiration_timestamp: data.expiration, - quote_kind: data.quote_kind, + quote_kind: data.quote_kind.clone(), solver: ByteArray(data.solver.0), - call_data: data.call_data, verified: data.verified, } } +pub fn create_quote_interactions_insert_data( + id: QuoteId, + data: &QuoteData, +) -> Vec { + data.interactions + .iter() + .enumerate() + .map(|(index, interaction)| DbQuoteInteraction { + id, + index: index as i64, + target: ByteArray(interaction.target.0), + value: u256_to_big_decimal(&interaction.value), + call_data: interaction.call_data.clone(), + }) + .collect() +} + pub fn create_db_search_parameters( params: QuoteSearchParameters, expiration: DateTime, diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index 247662739d..b494edc72b 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -171,7 +171,7 @@ pub struct QuoteData { /// Were we able to verify that this quote is accurate? pub verified: bool, /// Data provided by the solver in response to /quote request. - pub call_data: Option>, + pub interactions: Vec, } impl TryFrom for QuoteData { @@ -197,7 +197,7 @@ impl TryFrom for QuoteData { // Even if the quote was verified at the time of creation // it might no longer be accurate. verified: false, - call_data: row.call_data, + interactions: vec![], }) } } @@ -445,7 +445,7 @@ impl OrderQuoter { quote_kind, solver: trade_estimate.solver, verified: trade_estimate.verified, - call_data: trade_estimate.call_data, + interactions: trade_estimate.interactions, }; Ok(quote) @@ -731,7 +731,7 @@ mod tests { gas: 3, solver: H160([1; 20]), verified: false, - call_data: None, + interactions: vec![], }) } .boxed() @@ -773,7 +773,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - call_data: None, + interactions: vec![], })) .returning(|_| Ok(1337)); @@ -810,7 +810,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - call_data: None, + interactions: vec![], }, sell_amount: 70.into(), buy_amount: 29.into(), @@ -869,7 +869,7 @@ mod tests { gas: 3, solver: H160([1; 20]), verified: false, - call_data: None, + interactions: vec![], }) } .boxed() @@ -911,7 +911,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - call_data: None, + interactions: vec![], })) .returning(|_| Ok(1337)); @@ -948,7 +948,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - call_data: None, + interactions: vec![], }, sell_amount: 100.into(), buy_amount: 42.into(), @@ -1002,7 +1002,7 @@ mod tests { gas: 3, solver: H160([1; 20]), verified: false, - call_data: None, + interactions: vec![], }) } .boxed() @@ -1044,7 +1044,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - call_data: None, + interactions: vec![], })) .returning(|_| Ok(1337)); @@ -1081,7 +1081,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - call_data: None, + interactions: vec![], }, sell_amount: 100.into(), buy_amount: 42.into(), @@ -1121,7 +1121,7 @@ mod tests { gas: 200, solver: H160([1; 20]), verified: false, - call_data: None, + interactions: vec![], }) } .boxed() @@ -1193,7 +1193,7 @@ mod tests { gas: 200, solver: H160([1; 20]), verified: false, - call_data: None, + interactions: vec![], }) } .boxed() @@ -1270,7 +1270,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - call_data: Some(vec![1]), + interactions: vec![], })) }); @@ -1304,7 +1304,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - call_data: Some(vec![1]), + interactions: vec![], }, sell_amount: 85.into(), // Allows for "out-of-price" buy amounts. This means that order @@ -1352,7 +1352,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - call_data: None, + interactions: vec![], })) }); @@ -1386,7 +1386,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - call_data: None, + interactions: vec![], }, sell_amount: 100.into(), buy_amount: 42.into(), @@ -1435,7 +1435,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: true, - call_data: Some(vec![1]), + interactions: vec![], }, ))) }); @@ -1470,7 +1470,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: true, - call_data: Some(vec![1]), + interactions: vec![], }, sell_amount: 100.into(), buy_amount: 42.into(), diff --git a/crates/shared/src/price_estimation.rs b/crates/shared/src/price_estimation.rs index b27a9befdc..7155d14bd8 100644 --- a/crates/shared/src/price_estimation.rs +++ b/crates/shared/src/price_estimation.rs @@ -8,7 +8,10 @@ use { ethcontract::{H160, U256}, futures::future::BoxFuture, itertools::Itertools, - model::order::{BuyTokenDestination, OrderKind, SellTokenSource}, + model::{ + interaction::InteractionData, + order::{BuyTokenDestination, OrderKind, SellTokenSource}, + }, num::BigRational, number::nonzero::U256 as NonZeroU256, rate_limit::{RateLimiter, Strategy}, @@ -465,8 +468,8 @@ pub struct Estimate { pub solver: H160, /// Did we verify the correctness of this estimate's properties? pub verified: bool, - /// Data provided by the solver in response to /quote request. - pub call_data: Option>, + /// Interactions provided by the solver in response to /quote request. + pub interactions: Vec, } impl Estimate { diff --git a/crates/shared/src/price_estimation/sanitized.rs b/crates/shared/src/price_estimation/sanitized.rs index 04195e702a..7181b28baf 100644 --- a/crates/shared/src/price_estimation/sanitized.rs +++ b/crates/shared/src/price_estimation/sanitized.rs @@ -67,7 +67,7 @@ impl PriceEstimating for SanitizedPriceEstimator { gas: 0, solver: Default::default(), verified: true, - call_data: None, + interactions: vec![], }; tracing::debug!(?query, ?estimation, "generate trivial price estimation"); return Ok(estimation); @@ -80,7 +80,7 @@ impl PriceEstimating for SanitizedPriceEstimator { gas: GAS_PER_WETH_UNWRAP, solver: Default::default(), verified: true, - call_data: None, + interactions: vec![], }; tracing::debug!(?query, ?estimation, "generate trivial unwrap estimation"); return Ok(estimation); @@ -93,7 +93,7 @@ impl PriceEstimating for SanitizedPriceEstimator { gas: GAS_PER_WETH_WRAP, solver: Default::default(), verified: true, - call_data: None, + interactions: vec![], }; tracing::debug!(?query, ?estimation, "generate trivial wrap estimation"); return Ok(estimation); @@ -188,7 +188,7 @@ mod tests { gas: 100, solver: Default::default(), verified: false, - call_data: None, + interactions: vec![], }), ), // `sanitized_estimator` will replace `buy_token` with `native_token` before querying @@ -210,7 +210,7 @@ mod tests { gas: GAS_PER_WETH_UNWRAP + 100, solver: Default::default(), verified: false, - call_data: None, + interactions: vec![], }), ), // Will cause buffer overflow of gas price in `sanitized_estimator`. @@ -246,7 +246,7 @@ mod tests { gas: GAS_PER_WETH_WRAP + 100, solver: Default::default(), verified: false, - call_data: None, + interactions: vec![], }), ), // Can be estimated by `sanitized_estimator` because `buy_token` and `sell_token` are @@ -265,7 +265,7 @@ mod tests { gas: 0, solver: Default::default(), verified: true, - call_data: None, + interactions: vec![], }), ), // Can be estimated by `sanitized_estimator` because both tokens are the native token. @@ -283,7 +283,7 @@ mod tests { gas: 0, solver: Default::default(), verified: true, - call_data: None, + interactions: vec![], }), ), // Can be estimated by `sanitized_estimator` because it is a native token unwrap. @@ -302,7 +302,7 @@ mod tests { gas: GAS_PER_WETH_UNWRAP, solver: Default::default(), verified: true, - call_data: None, + interactions: vec![], }), ), // Can be estimated by `sanitized_estimator` because it is a native token wrap. @@ -321,7 +321,7 @@ mod tests { gas: GAS_PER_WETH_WRAP, solver: Default::default(), verified: true, - call_data: None, + interactions: vec![], }), ), // Will throw `UnsupportedToken` error in `sanitized_estimator`. @@ -386,7 +386,7 @@ mod tests { gas: 100, solver: Default::default(), verified: false, - call_data: None, + interactions: vec![], }) } .boxed() @@ -402,7 +402,7 @@ mod tests { gas: 100, solver: Default::default(), verified: false, - call_data: None, + interactions: vec![], }) } .boxed() @@ -418,7 +418,7 @@ mod tests { gas: u64::MAX, solver: Default::default(), verified: false, - call_data: None, + interactions: vec![], }) } .boxed() @@ -434,7 +434,7 @@ mod tests { gas: 100, solver: Default::default(), verified: false, - call_data: None, + interactions: vec![], }) } .boxed() diff --git a/crates/shared/src/price_estimation/trade_finder.rs b/crates/shared/src/price_estimation/trade_finder.rs index 4f200fe86c..d35d6751d3 100644 --- a/crates/shared/src/price_estimation/trade_finder.rs +++ b/crates/shared/src/price_estimation/trade_finder.rs @@ -89,7 +89,7 @@ impl Inner { gas: quote.gas_estimate, solver: quote.solver, verified: false, - call_data: quote.call_data, + interactions: quote.interactions, }) } } diff --git a/crates/shared/src/price_estimation/trade_verifier.rs b/crates/shared/src/price_estimation/trade_verifier.rs index 603686f3f2..44aae65816 100644 --- a/crates/shared/src/price_estimation/trade_verifier.rs +++ b/crates/shared/src/price_estimation/trade_verifier.rs @@ -179,7 +179,7 @@ impl TradeVerifier { gas: trade.gas_estimate.context("no gas estimate")?, solver: trade.solver, verified: true, - call_data: None, + interactions: crate::trade_finding::map_interactions_data(&trade.interactions), }; tracing::warn!( ?estimate, @@ -231,7 +231,7 @@ impl TradeVerifier { "verified quote", ); - ensure_quote_accuracy(&self.quote_inaccuracy_limit, query, trade.solver, &summary) + ensure_quote_accuracy(&self.quote_inaccuracy_limit, query, &trade, &summary) } /// Configures all the state overrides that are needed to mock the given @@ -313,7 +313,9 @@ impl TradeVerifying for TradeVerifier { gas, solver: trade.solver, verified: false, - call_data: None, + interactions: crate::trade_finding::map_interactions_data( + &trade.interactions, + ), }; tracing::warn!( ?err, @@ -510,7 +512,7 @@ impl SettleOutput { fn ensure_quote_accuracy( inaccuracy_limit: &BigRational, query: &PriceQuery, - solver: H160, + trade: &Trade, summary: &SettleOutput, ) -> Result { // amounts verified by the simulation @@ -528,9 +530,9 @@ fn ensure_quote_accuracy( Ok(Estimate { out_amount: summary.out_amount, gas: summary.gas_used.as_u64(), - solver, + solver: trade.solver, verified: true, - call_data: None, + interactions: crate::trade_finding::map_interactions_data(&trade.interactions), }) } diff --git a/crates/shared/src/trade_finding.rs b/crates/shared/src/trade_finding.rs index e4a9d07140..7cd6c672e9 100644 --- a/crates/shared/src/trade_finding.rs +++ b/crates/shared/src/trade_finding.rs @@ -30,7 +30,7 @@ pub struct Quote { pub out_amount: U256, pub gas_estimate: u64, pub solver: H160, - pub call_data: Option>, + pub interactions: Vec, } /// A trade. @@ -76,6 +76,14 @@ impl Interaction { pub fn encode(&self) -> EncodedInteraction { (self.target, self.value, Bytes(self.data.clone())) } + + pub fn to_interaction_data(&self) -> InteractionData { + InteractionData { + target: self.target, + value: self.value, + call_data: self.data.clone(), + } + } } impl From for Interaction { @@ -142,3 +150,11 @@ impl Clone for TradeError { pub fn map_interactions(interactions: &[InteractionData]) -> Vec { interactions.iter().cloned().map(Into::into).collect() } + +pub fn map_interactions_data(interactions: &[Interaction]) -> Vec { + interactions + .iter() + .cloned() + .map(|i| i.to_interaction_data()) + .collect() +} diff --git a/crates/shared/src/trade_finding/external.rs b/crates/shared/src/trade_finding/external.rs index 24a8c22c0b..72e738e66a 100644 --- a/crates/shared/src/trade_finding/external.rs +++ b/crates/shared/src/trade_finding/external.rs @@ -166,12 +166,11 @@ impl TradeFinding for ExternalTradeFinder { .gas_estimate .context("no gas estimate") .map_err(TradeError::Other)?; - let first_interaction = trade.interactions.first().cloned(); Ok(Quote { out_amount: trade.out_amount, gas_estimate, solver: trade.solver, - call_data: first_interaction.map(|interaction| interaction.data), + interactions: crate::trade_finding::map_interactions_data(&trade.interactions), }) } diff --git a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql index 743e795e36..8d4d2c717f 100644 --- a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql +++ b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql @@ -1,9 +1,21 @@ -- Step 1: Add a new column to the quotes table ALTER TABLE quotes - ADD COLUMN call_data bytea, ADD COLUMN verified boolean; -- Step 2: Add two new columns to the order_quotes table ALTER TABLE order_quotes - ADD COLUMN call_data bytea, ADD COLUMN verified boolean; + +-- Step 3: Create table with quote interactions +CREATE TABLE quotes_interactions ( + order_uid bytea NOT NULL, + index int NOT NULL, + target bytea NOT NULL, + value numeric(78,0) NOT NULL, + call_data bytea, + + PRIMARY KEY (order_uid, index) +); + +-- Get a specific quote's interactions. +CREATE INDEX quote_uid_interactions ON quotes_interactions USING HASH (order_uid); From bca89ac7e4caa16135e498b88c3537f96be65272 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Sat, 23 Nov 2024 01:29:32 +0100 Subject: [PATCH 14/70] Fixed tests compilation --- crates/orderbook/src/database/orders.rs | 5 ----- crates/shared/src/event_storing_helpers.rs | 2 +- crates/shared/src/price_estimation/native.rs | 2 +- .../shared/src/price_estimation/trade_verifier.rs | 14 +++++++------- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index 887cfab122..04825b43b7 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -1184,7 +1184,6 @@ mod tests { database::clear_DANGER(&db.pool).await.unwrap(); let uid = OrderUid([0x42; 56]); - let call_data = vec![1]; let order = Order { data: OrderData { valid_to: u32::MAX, @@ -1211,10 +1210,6 @@ mod tests { let single_order_with_quote = db.single_order_with_quote(&uid).await.unwrap().unwrap(); assert_eq!(single_order_with_quote.order, order); - assert_eq!( - single_order_with_quote.quote.clone().unwrap().call_data, - Some(call_data) - ); assert!(single_order_with_quote.quote.unwrap().verified,); } } diff --git a/crates/shared/src/event_storing_helpers.rs b/crates/shared/src/event_storing_helpers.rs index 28f829391f..496d2f1732 100644 --- a/crates/shared/src/event_storing_helpers.rs +++ b/crates/shared/src/event_storing_helpers.rs @@ -43,7 +43,7 @@ pub fn create_quote_interactions_insert_data( .enumerate() .map(|(index, interaction)| DbQuoteInteraction { id, - index: index as i64, + index: index.try_into().unwrap(), target: ByteArray(interaction.target.0), value: u256_to_big_decimal(&interaction.value), call_data: interaction.call_data.clone(), diff --git a/crates/shared/src/price_estimation/native.rs b/crates/shared/src/price_estimation/native.rs index 910d671cff..6901ce3072 100644 --- a/crates/shared/src/price_estimation/native.rs +++ b/crates/shared/src/price_estimation/native.rs @@ -130,7 +130,7 @@ mod tests { gas: 0, solver: H160([1; 20]), verified: false, - call_data: None, + interactions: vec![], }) } .boxed() diff --git a/crates/shared/src/price_estimation/trade_verifier.rs b/crates/shared/src/price_estimation/trade_verifier.rs index 44aae65816..609af16626 100644 --- a/crates/shared/src/price_estimation/trade_verifier.rs +++ b/crates/shared/src/price_estimation/trade_verifier.rs @@ -231,7 +231,7 @@ impl TradeVerifier { "verified quote", ); - ensure_quote_accuracy(&self.quote_inaccuracy_limit, query, &trade, &summary) + ensure_quote_accuracy(&self.quote_inaccuracy_limit, query, trade, &summary) } /// Configures all the state overrides that are needed to mock the given @@ -581,11 +581,11 @@ mod tests { sell_tokens_lost: BigRational::from_integer(500.into()), }; - let estimate = ensure_quote_accuracy(&low_threshold, &query, H160::zero(), &sell_more); + let estimate = ensure_quote_accuracy(&low_threshold, &query, &Default::default(), &sell_more); assert!(matches!(estimate, Err(Error::TooInaccurate))); // passes with slightly higher tolerance - let estimate = ensure_quote_accuracy(&high_threshold, &query, H160::zero(), &sell_more); + let estimate = ensure_quote_accuracy(&high_threshold, &query, &Default::default(), &sell_more); assert!(estimate.is_ok()); let pay_out_more = SettleOutput { @@ -595,11 +595,11 @@ mod tests { sell_tokens_lost: BigRational::from_integer(0.into()), }; - let estimate = ensure_quote_accuracy(&low_threshold, &query, H160::zero(), &pay_out_more); + let estimate = ensure_quote_accuracy(&low_threshold, &query, &Default::default(), &pay_out_more); assert!(matches!(estimate, Err(Error::TooInaccurate))); // passes with slightly higher tolerance - let estimate = ensure_quote_accuracy(&high_threshold, &query, H160::zero(), &pay_out_more); + let estimate = ensure_quote_accuracy(&high_threshold, &query, &Default::default(), &pay_out_more); assert!(estimate.is_ok()); let sell_less = SettleOutput { @@ -609,7 +609,7 @@ mod tests { sell_tokens_lost: BigRational::from_integer((-500).into()), }; // Ending up with surplus in the buffers is always fine - let estimate = ensure_quote_accuracy(&low_threshold, &query, H160::zero(), &sell_less); + let estimate = ensure_quote_accuracy(&low_threshold, &query, &Default::default(), &sell_less); assert!(estimate.is_ok()); let pay_out_less = SettleOutput { @@ -619,7 +619,7 @@ mod tests { sell_tokens_lost: BigRational::from_integer(0.into()), }; // Ending up with surplus in the buffers is always fine - let estimate = ensure_quote_accuracy(&low_threshold, &query, H160::zero(), &pay_out_less); + let estimate = ensure_quote_accuracy(&low_threshold, &query, &Default::default(), &pay_out_less); assert!(estimate.is_ok()); } } From 7c6983c5bf49db49e9627ae6e6975fb73c1632f3 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Sat, 23 Nov 2024 01:33:44 +0100 Subject: [PATCH 15/70] Fixed formatting --- .../src/price_estimation/trade_verifier.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/crates/shared/src/price_estimation/trade_verifier.rs b/crates/shared/src/price_estimation/trade_verifier.rs index 609af16626..fcefb25cc3 100644 --- a/crates/shared/src/price_estimation/trade_verifier.rs +++ b/crates/shared/src/price_estimation/trade_verifier.rs @@ -581,11 +581,13 @@ mod tests { sell_tokens_lost: BigRational::from_integer(500.into()), }; - let estimate = ensure_quote_accuracy(&low_threshold, &query, &Default::default(), &sell_more); + let estimate = + ensure_quote_accuracy(&low_threshold, &query, &Default::default(), &sell_more); assert!(matches!(estimate, Err(Error::TooInaccurate))); // passes with slightly higher tolerance - let estimate = ensure_quote_accuracy(&high_threshold, &query, &Default::default(), &sell_more); + let estimate = + ensure_quote_accuracy(&high_threshold, &query, &Default::default(), &sell_more); assert!(estimate.is_ok()); let pay_out_more = SettleOutput { @@ -595,11 +597,13 @@ mod tests { sell_tokens_lost: BigRational::from_integer(0.into()), }; - let estimate = ensure_quote_accuracy(&low_threshold, &query, &Default::default(), &pay_out_more); + let estimate = + ensure_quote_accuracy(&low_threshold, &query, &Default::default(), &pay_out_more); assert!(matches!(estimate, Err(Error::TooInaccurate))); // passes with slightly higher tolerance - let estimate = ensure_quote_accuracy(&high_threshold, &query, &Default::default(), &pay_out_more); + let estimate = + ensure_quote_accuracy(&high_threshold, &query, &Default::default(), &pay_out_more); assert!(estimate.is_ok()); let sell_less = SettleOutput { @@ -609,7 +613,8 @@ mod tests { sell_tokens_lost: BigRational::from_integer((-500).into()), }; // Ending up with surplus in the buffers is always fine - let estimate = ensure_quote_accuracy(&low_threshold, &query, &Default::default(), &sell_less); + let estimate = + ensure_quote_accuracy(&low_threshold, &query, &Default::default(), &sell_less); assert!(estimate.is_ok()); let pay_out_less = SettleOutput { @@ -619,7 +624,8 @@ mod tests { sell_tokens_lost: BigRational::from_integer(0.into()), }; // Ending up with surplus in the buffers is always fine - let estimate = ensure_quote_accuracy(&low_threshold, &query, &Default::default(), &pay_out_less); + let estimate = + ensure_quote_accuracy(&low_threshold, &query, &Default::default(), &pay_out_less); assert!(estimate.is_ok()); } } From a847647067f12021876f20a5454bc1b8843aa99e Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Sat, 23 Nov 2024 01:37:47 +0100 Subject: [PATCH 16/70] Updated orders database tables --- crates/database/src/orders.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index b8db7c1da6..09cc5fb4c1 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -503,7 +503,6 @@ pub struct FullOrderWithQuote { pub quote_gas_price: Option, pub quote_sell_token_price: Option, pub quote_verified: Option, - pub quote_call_data: Option>, pub solver: Option
, } @@ -598,7 +597,6 @@ pub async fn single_full_order_with_quote( ", o_quotes.gas_price as quote_gas_price", ", o_quotes.sell_token_price as quote_sell_token_price", ", o_quotes.verified as quote_verified", - ", o_quotes.call_data as quote_call_data", ", o_quotes.solver as solver", " FROM ", FROM, " LEFT JOIN order_quotes o_quotes ON o.uid = o_quotes.order_uid", From 1868aaeb94276cd2232ba0657c7b6c4bfd98e8df Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Mon, 25 Nov 2024 15:44:57 +0100 Subject: [PATCH 17/70] Fixed sql --- crates/database/src/quotes.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 6c9b2eb7d2..87586df6aa 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -165,12 +165,12 @@ pub async fn insert_quote_interaction( const QUERY: &str = r#" INSERT INTO quotes_interactions ( order_uid, - index + index, target, value, call_data ) -VALUES ($1, $2, $3, $4) +VALUES ($1, $2, $3, $4, $5) "#; sqlx::query(QUERY) .bind(quote_interaction.id) From 507f6aecb2e8dabda38144e6b4c0ef2034a8f377 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Mon, 25 Nov 2024 22:27:52 +0100 Subject: [PATCH 18/70] Update --- crates/autopilot/src/database/auction.rs | 2 +- crates/database/src/quotes.rs | 14 +++++++------- crates/orderbook/src/database/quotes.rs | 2 +- crates/shared/src/event_storing_helpers.rs | 17 ++++++++++------- ...add_call_data_to_quotes_and_order_quotes.sql | 6 +++--- 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/crates/autopilot/src/database/auction.rs b/crates/autopilot/src/database/auction.rs index 8e254483fa..fffcccd46e 100644 --- a/crates/autopilot/src/database/auction.rs +++ b/crates/autopilot/src/database/auction.rs @@ -30,7 +30,7 @@ impl QuoteStoring for Postgres { let row = create_quote_row(&data); let id = database::quotes::save(&mut ex, &row).await?; if !data.interactions.is_empty() { - let interactions = create_quote_interactions_insert_data(id, &data); + let interactions = create_quote_interactions_insert_data(id, &data)?; database::quotes::insert_quote_interactions(&mut ex, &interactions).await?; } Ok(id) diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 87586df6aa..d816056a95 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -150,8 +150,8 @@ WHERE expiration_timestamp < $1 /// One row in the `quotes_interactions` table. #[derive(Clone, Debug, PartialEq, sqlx::FromRow)] pub struct QuoteInteraction { - pub id: QuoteId, - pub index: i64, + pub quote_id: QuoteId, + pub index: i32, pub target: Address, pub value: BigDecimal, pub call_data: Vec, @@ -164,7 +164,7 @@ pub async fn insert_quote_interaction( ) -> Result<(), sqlx::Error> { const QUERY: &str = r#" INSERT INTO quotes_interactions ( - order_uid, + quote_id, index, target, value, @@ -173,7 +173,7 @@ INSERT INTO quotes_interactions ( VALUES ($1, $2, $3, $4, $5) "#; sqlx::query(QUERY) - .bind(quote_interaction.id) + .bind(quote_interaction.quote_id) .bind(quote_interaction.index) .bind(quote_interaction.target) .bind("e_interaction.value) @@ -484,7 +484,7 @@ mod tests { crate::clear_DANGER_(&mut db).await.unwrap(); let quote_interaction = QuoteInteraction { - id: Default::default(), + quote_id: Default::default(), index: Default::default(), target: ByteArray([1; 20]), value: 2.into(), @@ -497,10 +497,10 @@ mod tests { const QUERY: &str = r#" SELECT * FROM quotes_interactions - WHERE order_uid = $1 + WHERE quote_id = $1 "#; let interaction: Option = sqlx::query_as(QUERY) - .bind(quote_interaction.id) + .bind(quote_interaction.quote_id) .fetch_optional(&mut db as &mut PgConnection) .await .unwrap(); diff --git a/crates/orderbook/src/database/quotes.rs b/crates/orderbook/src/database/quotes.rs index 99a8bca3d0..e702b9e415 100644 --- a/crates/orderbook/src/database/quotes.rs +++ b/crates/orderbook/src/database/quotes.rs @@ -25,7 +25,7 @@ impl QuoteStoring for Postgres { let row = create_quote_row(&data); let id = database::quotes::save(&mut ex, &row).await?; if !data.interactions.is_empty() { - let interactions = create_quote_interactions_insert_data(id, &data); + let interactions = create_quote_interactions_insert_data(id, &data)?; database::quotes::insert_quote_interactions(&mut ex, &interactions).await?; } Ok(id) diff --git a/crates/shared/src/event_storing_helpers.rs b/crates/shared/src/event_storing_helpers.rs index 496d2f1732..3f2f160ca6 100644 --- a/crates/shared/src/event_storing_helpers.rs +++ b/crates/shared/src/event_storing_helpers.rs @@ -3,6 +3,7 @@ use { db_order_conversions::order_kind_into, order_quoting::{quote_kind_from_signing_scheme, QuoteData, QuoteSearchParameters}, }, + anyhow::Result, chrono::{DateTime, Utc}, database::{ byte_array::ByteArray, @@ -37,16 +38,18 @@ pub fn create_quote_row(data: &QuoteData) -> DbQuote { pub fn create_quote_interactions_insert_data( id: QuoteId, data: &QuoteData, -) -> Vec { +) -> Result> { data.interactions .iter() .enumerate() - .map(|(index, interaction)| DbQuoteInteraction { - id, - index: index.try_into().unwrap(), - target: ByteArray(interaction.target.0), - value: u256_to_big_decimal(&interaction.value), - call_data: interaction.call_data.clone(), + .map(|(index, interaction)| { + Ok(DbQuoteInteraction { + quote_id: id, + index: index.try_into()?, + target: ByteArray(interaction.target.0), + value: u256_to_big_decimal(&interaction.value), + call_data: interaction.call_data.clone(), + }) }) .collect() } diff --git a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql index 8d4d2c717f..9384296a96 100644 --- a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql +++ b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql @@ -8,14 +8,14 @@ ALTER TABLE order_quotes -- Step 3: Create table with quote interactions CREATE TABLE quotes_interactions ( - order_uid bytea NOT NULL, + quote_id bigint NOT NULL, index int NOT NULL, target bytea NOT NULL, value numeric(78,0) NOT NULL, call_data bytea, - PRIMARY KEY (order_uid, index) + PRIMARY KEY (quote_id, index) ); -- Get a specific quote's interactions. -CREATE INDEX quote_uid_interactions ON quotes_interactions USING HASH (order_uid); +CREATE INDEX quote_id_interactions ON quotes_interactions USING HASH (quote_id); From aff04681a0135d9a043f6ae906f3174bd3dbde45 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Mon, 25 Nov 2024 23:51:11 +0100 Subject: [PATCH 19/70] Added removing quote_interactions on quote remove --- crates/database/src/quotes.rs | 108 +++++++++++++++++++++--- crates/orderbook/src/database/orders.rs | 12 +++ 2 files changed, 107 insertions(+), 13 deletions(-) diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index d816056a95..c7cf295701 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -4,6 +4,7 @@ use { sqlx::{ types::chrono::{DateTime, Utc}, PgConnection, + Row, }, }; @@ -139,12 +140,17 @@ pub async fn remove_expired_quotes( const QUERY: &str = r#" DELETE FROM quotes WHERE expiration_timestamp < $1 +RETURNING id "#; - sqlx::query(QUERY) + let deleted_ids = sqlx::query(QUERY) .bind(max_expiry) - .execute(ex) - .await - .map(|_| ()) + .fetch_all(&mut *ex) + .await? + .iter() + .map(|row| row.get(0)) + .collect::>(); + + delete_quote_interactions(ex, &deleted_ids).await } /// One row in the `quotes_interactions` table. @@ -193,6 +199,31 @@ pub async fn insert_quote_interactions( Ok(()) } +pub async fn get_quote_interactions( + ex: &mut PgConnection, + quote_id: QuoteId, +) -> Result, sqlx::Error> { + const QUERY: &str = r#" +SELECT * +FROM quotes_interactions +WHERE quote_id = $1 + "#; + sqlx::query_as(QUERY).bind(quote_id).fetch_all(ex).await +} + +pub async fn delete_quote_interactions( + ex: &mut PgConnection, + quote_ids: &[QuoteId], +) -> Result<(), sqlx::Error> { + const QUERY: &str = r#" +DELETE FROM quotes_interactions +WHERE quote_id = ANY($1) + "#; + + sqlx::query(QUERY).bind("e_ids).execute(ex).await?; + Ok(()) +} + #[cfg(test)] mod tests { use { @@ -494,17 +525,68 @@ mod tests { .await .unwrap(); - const QUERY: &str = r#" - SELECT * - FROM quotes_interactions - WHERE quote_id = $1 - "#; - let interaction: Option = sqlx::query_as(QUERY) - .bind(quote_interaction.quote_id) - .fetch_optional(&mut db as &mut PgConnection) + let interactions = get_quote_interactions(&mut db, quote_interaction.quote_id) + .await + .unwrap(); + assert_eq!(*interactions.first().unwrap(), quote_interaction); + } + + #[tokio::test] + #[ignore] + async fn postgres_removed_quote_interactions_by_id() { + let mut db = PgConnection::connect("postgresql://").await.unwrap(); + let mut db = db.begin().await.unwrap(); + crate::clear_DANGER_(&mut db).await.unwrap(); + + let now = low_precision_now(); + let quote = Quote { + id: Default::default(), + sell_token: ByteArray([1; 20]), + buy_token: ByteArray([2; 20]), + sell_amount: 3.into(), + buy_amount: 4.into(), + gas_amount: 5., + gas_price: 6., + sell_token_price: 7., + order_kind: OrderKind::Sell, + expiration_timestamp: now, + quote_kind: QuoteKind::Standard, + solver: ByteArray([1; 20]), + verified: false, + }; + // store quote in database + let id = save(&mut db, "e).await.unwrap(); + + let quote_interactions = [ + QuoteInteraction { + quote_id: id, + index: 0, + target: ByteArray([1; 20]), + value: 2.into(), + call_data: vec![3; 20], + }, + QuoteInteraction { + quote_id: id, + index: 1, + target: ByteArray([1; 20]), + value: 2.into(), + call_data: vec![3; 20], + }, + ]; + // store interactions for the quote in database + insert_quote_interactions(&mut db, "e_interactions) + .await + .unwrap(); + + let interactions = get_quote_interactions(&mut db, id).await.unwrap(); + assert_eq!(interactions.len(), 2); + + // remove quote using expired functino call, should also remove interactions + remove_expired_quotes(&mut db, now + Duration::seconds(30)) .await .unwrap(); - assert_eq!(interaction.unwrap(), quote_interaction); + let interactions = get_quote_interactions(&mut db, id).await.unwrap(); + assert!(interactions.is_empty()); } } diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index 04825b43b7..c524702fe8 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -1202,6 +1202,18 @@ mod tests { buy_amount: U256::from(2), data: QuoteData { verified: true, + interactions: vec![ + InteractionData { + target: H160([1; 20]), + value: U256::from(100), + call_data: vec![1, 20], + }, + InteractionData { + target: H160([2; 20]), + value: U256::from(10), + call_data: vec![2, 20], + }, + ], ..Default::default() }, ..Default::default() From 189222b68ab778e9b851e60766522a7235c8843a Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Tue, 26 Nov 2024 13:42:10 +0100 Subject: [PATCH 20/70] Added order_quotes_interactions table --- crates/database/src/lib.rs | 2 + crates/database/src/orders.rs | 44 +++++++++++++++++++ ...d_call_data_to_quotes_and_order_quotes.sql | 14 ++++++ 3 files changed, 60 insertions(+) diff --git a/crates/database/src/lib.rs b/crates/database/src/lib.rs index ec30cd557d..a2577aa2e9 100644 --- a/crates/database/src/lib.rs +++ b/crates/database/src/lib.rs @@ -72,6 +72,8 @@ pub const TABLES: &[&str] = &[ "auction_participants", "app_data", "jit_orders", + "quotes_interactions", + "order_quotes_interactions", ]; /// The names of potentially big volume tables we use in the db. diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 09cc5fb4c1..6106890f0b 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -449,6 +449,50 @@ AND cancellation_timestamp IS NULL .map(|_| ()) } +/// One row in the `order_quotes_interactions` table. +#[derive(Clone, Default, Debug, PartialEq, sqlx::FromRow)] +pub struct OrderQuoteInteraction { + pub order_uid: OrderUid, + pub index: i32, + pub target: Address, + pub value: BigDecimal, + pub call_data: Vec, +} + +pub async fn insert_order_quote_interaction( + ex: &mut PgConnection, + quote_interaction: &OrderQuoteInteraction, +) -> Result<(), sqlx::Error> { + const INSERT_ORDER_QUOTES_INTERACTION_QUERY: &str = r#" + INSERT INTO order_quotes_interactions ( + order_uid, + index, + target, + value, + call_data + ) + VALUES ($1, $2, $3, $4, $5)"#; + sqlx::query(INSERT_ORDER_QUOTES_INTERACTION_QUERY) + .bind(quote_interaction.order_uid) + .bind(quote_interaction.index) + .bind(quote_interaction.target) + .bind("e_interaction.value) + .bind("e_interaction.call_data) + .execute(ex) + .await?; + Ok(()) +} + +pub async fn insert_order_quote_interactions( + ex: &mut PgConnection, + quote_interactions: &[OrderQuoteInteraction], +) -> Result<(), sqlx::Error> { + for interactions in quote_interactions { + insert_order_quote_interaction(ex, interactions).await?; + } + Ok(()) +} + /// Interactions are read as arrays of their fields: target, value, data. /// This is done as sqlx does not support reading arrays of more complicated /// types than just one field. The pre_ and post_interaction's data of diff --git a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql index 9384296a96..904f64d787 100644 --- a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql +++ b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql @@ -19,3 +19,17 @@ CREATE TABLE quotes_interactions ( -- Get a specific quote's interactions. CREATE INDEX quote_id_interactions ON quotes_interactions USING HASH (quote_id); + +-- Step 3: Create table with quote interactions +CREATE TABLE order_quotes_interactions ( + order_uid bytea NOT NULL, + index int NOT NULL, + target bytea NOT NULL, + value numeric(78,0) NOT NULL, + call_data bytea, + + PRIMARY KEY (order_uid, index) +); + +-- Get a specific order's interactions. +CREATE INDEX order_uid_interactions ON orders_quotes_interactions USING HASH (order_uid); From 7a0cdc3c1b5907425def8b733f64b65c23fb4e38 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Tue, 26 Nov 2024 19:48:53 +0100 Subject: [PATCH 21/70] Added filling order quote interaction table --- crates/database/src/orders.rs | 44 ++++++++++++++++++++++++- crates/database/src/quotes.rs | 2 +- crates/orderbook/src/api/post_order.rs | 4 +++ crates/orderbook/src/database/orders.rs | 27 ++++++++++++--- crates/orderbook/src/orderbook.rs | 3 ++ 5 files changed, 74 insertions(+), 6 deletions(-) diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 6106890f0b..5a1665b75c 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -471,7 +471,18 @@ pub async fn insert_order_quote_interaction( value, call_data ) - VALUES ($1, $2, $3, $4, $5)"#; + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (order_uid, index) DO UPDATE SET + ( + target, + value, + call_data + ) = ( + EXCLUDED.target, + EXCLUDED.value, + EXCLUDED.call_data + ) + "#; sqlx::query(INSERT_ORDER_QUOTES_INTERACTION_QUERY) .bind(quote_interaction.order_uid) .bind(quote_interaction.index) @@ -2195,4 +2206,35 @@ mod tests { ] ); } + + #[tokio::test] + #[ignore] + async fn postgres_insert_order_quote_interaction() { + let mut db = PgConnection::connect("postgresql://").await.unwrap(); + let mut db = db.begin().await.unwrap(); + crate::clear_DANGER_(&mut db).await.unwrap(); + + let interaction = OrderQuoteInteraction { + order_uid: Default::default(), + index: Default::default(), + target: ByteArray([1; 20]), + value: 2.into(), + call_data: vec![3; 20], + }; + insert_order_quote_interaction(&mut db, &interaction) + .await + .unwrap(); + + const QUERY: &str = r#" + SELECT * FROM order_quotes_interactions + WHERE order_uid = $1 + "#; + + let interactions: Vec = sqlx::query_as(QUERY) + .bind(&interaction.order_uid) + .fetch_all(&mut db as &mut PgConnection) + .await + .unwrap(); + assert_eq!(*interactions.first().unwrap(), interaction); + } } diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index c7cf295701..959e886f69 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -220,7 +220,7 @@ DELETE FROM quotes_interactions WHERE quote_id = ANY($1) "#; - sqlx::query(QUERY).bind("e_ids).execute(ex).await?; + sqlx::query(QUERY).bind(quote_ids).execute(ex).await?; Ok(()) } diff --git a/crates/orderbook/src/api/post_order.rs b/crates/orderbook/src/api/post_order.rs index 2208bff6fd..cf0c768e68 100644 --- a/crates/orderbook/src/api/post_order.rs +++ b/crates/orderbook/src/api/post_order.rs @@ -262,6 +262,10 @@ impl IntoWarpReply for AddOrderError { super::error("InvalidReplacement", err.to_string()), StatusCode::UNAUTHORIZED, ), + AddOrderError::QuoteInteractionWrongIndex => { + tracing::error!("QuoteInteractionWrongIndex"); + crate::api::internal_error_reply() + } } } } diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index c524702fe8..c5d3ef48e3 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -7,7 +7,7 @@ use { database::{ byte_array::ByteArray, order_events::{insert_order_event, OrderEvent, OrderEventLabel}, - orders::{self, FullOrder, OrderKind as DbOrderKind}, + orders::{self, FullOrder, OrderKind as DbOrderKind, OrderQuoteInteraction}, }, ethcontract::H256, futures::{stream::TryStreamExt, FutureExt, StreamExt}, @@ -226,7 +226,7 @@ async fn insert_quote( quote: &Quote, ex: &mut PgConnection, ) -> Result<(), InsertionError> { - let quote = database::orders::Quote { + let dbquote = database::orders::Quote { order_uid: ByteArray(order.metadata.uid.0), gas_amount: quote.data.fee_parameters.gas_amount, gas_price: quote.data.fee_parameters.gas_price, @@ -236,10 +236,29 @@ async fn insert_quote( solver: ByteArray(quote.data.solver.0), verified: quote.data.verified, }; - database::orders::insert_quote(ex, "e) + database::orders::insert_quote(ex, &dbquote) .await .map_err(InsertionError::DbError)?; - Ok(()) + + let dbinteractions = quote + .data + .interactions + .iter() + .enumerate() + .map(|(idx, interaction)| { + Ok(OrderQuoteInteraction { + order_uid: dbquote.order_uid, + index: idx.try_into()?, + target: ByteArray(interaction.target.0), + value: u256_to_big_decimal(&interaction.value), + call_data: interaction.call_data.clone(), + }) + }) + .collect::>>() + .map_err(|_| InsertionError::IndexConversionFailed)?; + database::orders::insert_order_quote_interactions(ex, dbinteractions.as_slice()) + .await + .map_err(InsertionError::DbError) } #[async_trait::async_trait] diff --git a/crates/orderbook/src/orderbook.rs b/crates/orderbook/src/orderbook.rs index fc637a1c2f..74163d17cb 100644 --- a/crates/orderbook/src/orderbook.rs +++ b/crates/orderbook/src/orderbook.rs @@ -141,6 +141,8 @@ pub enum AddOrderError { provided: String, existing: String, }, + #[error("unable to convert type usize to i32 for one of the quote interactions")] + QuoteInteractionWrongIndex, } impl AddOrderError { @@ -161,6 +163,7 @@ impl AddOrderError { s.into_owned() }, }, + InsertionError::IndexConversionFailed => AddOrderError::QuoteInteractionWrongIndex, } } } From ae1115af49e87d33c4044e26f7a1f7ea16201eec Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Tue, 26 Nov 2024 23:21:42 +0100 Subject: [PATCH 22/70] Fixed clippy warning --- crates/database/src/orders.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 5a1665b75c..e49e6c2a67 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -2231,7 +2231,7 @@ mod tests { "#; let interactions: Vec = sqlx::query_as(QUERY) - .bind(&interaction.order_uid) + .bind(interaction.order_uid) .fetch_all(&mut db as &mut PgConnection) .await .unwrap(); From 33a4cc40bd2addb3dca577b98dff5ac316fb829c Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Tue, 26 Nov 2024 23:25:35 +0100 Subject: [PATCH 23/70] Fixed missing variant --- crates/orderbook/src/database/orders.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index c5d3ef48e3..d49af28291 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -113,6 +113,8 @@ pub enum InsertionError { DbError(sqlx::Error), /// Full app data to be inserted doesn't match existing. AppDataMismatch(Vec), + /// Type conversion (usize to i32) of sequence index of order quote interaction failed. + IndexConversionFailed, } impl From for InsertionError { From c62ac2dc246547c6e2ecdd9f118373a019229e91 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Wed, 27 Nov 2024 09:47:12 +0100 Subject: [PATCH 24/70] Returning quote interactions in QuoteStoring implementation --- crates/autopilot/src/database/auction.rs | 48 +++++++++++++++++++++--- crates/orderbook/src/database/orders.rs | 3 +- crates/orderbook/src/database/quotes.rs | 48 +++++++++++++++++++++--- 3 files changed, 88 insertions(+), 11 deletions(-) diff --git a/crates/autopilot/src/database/auction.rs b/crates/autopilot/src/database/auction.rs index fffcccd46e..1dedce31da 100644 --- a/crates/autopilot/src/database/auction.rs +++ b/crates/autopilot/src/database/auction.rs @@ -4,8 +4,10 @@ use { anyhow::{Context, Result}, chrono::{DateTime, Utc}, futures::{StreamExt, TryStreamExt}, - model::{order::Order, quote::QuoteId}, + model::{interaction::InteractionData, order::Order, quote::QuoteId}, num::ToPrimitive, + number::conversions::big_decimal_to_u256, + primitive_types::H160, shared::{ db_order_conversions::full_order_into_model_order, event_storing_helpers::{ @@ -44,7 +46,25 @@ impl QuoteStoring for Postgres { let mut ex = self.pool.acquire().await?; let quote = database::quotes::get(&mut ex, id).await?; - quote.map(TryFrom::try_from).transpose() + let quote_interactions = database::quotes::get_quote_interactions(&mut ex, id) + .await? + .iter() + .map(|data| { + Ok(InteractionData { + target: H160(data.target.0), + value: big_decimal_to_u256(&data.value) + .context("quote interaction value is not a valid U256")?, + call_data: data.call_data.clone(), + }) + }) + .collect::>>()?; + Ok(quote + .map(QuoteData::try_from) + .transpose()? + .map(|mut quote_data| { + quote_data.interactions = quote_interactions; + quote_data + })) } async fn find( @@ -62,9 +82,27 @@ impl QuoteStoring for Postgres { let quote = database::quotes::find(&mut ex, ¶ms) .await .context("failed finding quote by parameters")?; - quote - .map(|quote| Ok((quote.id, quote.try_into()?))) - .transpose() + if let Some(quote) = quote { + let quote_id = quote.id; + let quote_interactions = database::quotes::get_quote_interactions(&mut ex, quote_id) + .await? + .iter() + .map(|data| { + Ok(InteractionData { + target: H160(data.target.0), + value: big_decimal_to_u256(&data.value) + .context("quote interaction value is not a valid U256")?, + call_data: data.call_data.clone(), + }) + }) + .collect::>>()?; + + let mut quote_data = QuoteData::try_from(quote)?; + quote_data.interactions = quote_interactions; + Ok(Some((quote_id, quote_data))) + } else { + Ok(None) + } } } diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index d49af28291..ad412a3145 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -113,7 +113,8 @@ pub enum InsertionError { DbError(sqlx::Error), /// Full app data to be inserted doesn't match existing. AppDataMismatch(Vec), - /// Type conversion (usize to i32) of sequence index of order quote interaction failed. + /// Type conversion (usize to i32) of sequence index of order quote + /// interaction failed. IndexConversionFailed, } diff --git a/crates/orderbook/src/database/quotes.rs b/crates/orderbook/src/database/quotes.rs index e702b9e415..8d66416e5a 100644 --- a/crates/orderbook/src/database/quotes.rs +++ b/crates/orderbook/src/database/quotes.rs @@ -2,7 +2,9 @@ use { super::Postgres, anyhow::{Context, Result}, chrono::{DateTime, Utc}, - model::quote::QuoteId, + model::{interaction::InteractionData, quote::QuoteId}, + number::conversions::big_decimal_to_u256, + primitive_types::H160, shared::{ event_storing_helpers::{ create_db_search_parameters, @@ -39,7 +41,25 @@ impl QuoteStoring for Postgres { let mut ex = self.pool.acquire().await?; let quote = database::quotes::get(&mut ex, id).await?; - quote.map(TryFrom::try_from).transpose() + let quote_interactions = database::quotes::get_quote_interactions(&mut ex, id) + .await? + .iter() + .map(|data| { + Ok(InteractionData { + target: H160(data.target.0), + value: big_decimal_to_u256(&data.value) + .context("quote interaction value is not a valid U256")?, + call_data: data.call_data.clone(), + }) + }) + .collect::>>()?; + Ok(quote + .map(QuoteData::try_from) + .transpose()? + .map(|mut quote_data| { + quote_data.interactions = quote_interactions; + quote_data + })) } async fn find( @@ -57,8 +77,26 @@ impl QuoteStoring for Postgres { let quote = database::quotes::find(&mut ex, ¶ms) .await .context("failed finding quote by parameters")?; - quote - .map(|quote| Ok((quote.id, quote.try_into()?))) - .transpose() + if let Some(quote) = quote { + let quote_id = quote.id; + let quote_interactions = database::quotes::get_quote_interactions(&mut ex, quote_id) + .await? + .iter() + .map(|data| { + Ok(InteractionData { + target: H160(data.target.0), + value: big_decimal_to_u256(&data.value) + .context("quote interaction value is not a valid U256")?, + call_data: data.call_data.clone(), + }) + }) + .collect::>>()?; + + let mut quote_data = QuoteData::try_from(quote)?; + quote_data.interactions = quote_interactions; + Ok(Some((quote_id, quote_data))) + } else { + Ok(None) + } } } From dc3626909865bed3a3ca57d903c463437e01f116 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Wed, 27 Nov 2024 10:28:04 +0100 Subject: [PATCH 25/70] Added waiting for db migrations before tests --- .github/workflows/pull-request.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 0475fe7ecf..c313a22472 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -91,6 +91,7 @@ jobs: with: file: docker-compose.yaml up-opts: -d db migrations + - run: docker wait services-migrations-1 - run: cargo nextest run postgres -p orderbook -p database -p autopilot --test-threads 1 --run-ignored ignored-only test-local-node: From 888e576a45b9c4f650e59f2a538c4f1562d0df0d Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Wed, 27 Nov 2024 11:03:00 +0100 Subject: [PATCH 26/70] Added db dependency to migrations docker container --- docker-compose.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index cdc9721cd0..a6fe60d7c3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -32,6 +32,8 @@ services: - type: bind source: ./database/sql/ target: /flyway/sql + depends_on: + - db volumes: postgres: From ec9eae025856fb0233ef4581ab6276a536f6b909 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Wed, 27 Nov 2024 11:21:15 +0100 Subject: [PATCH 27/70] Temporarily disabled restarts on migration --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index a6fe60d7c3..179577bf34 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -24,7 +24,7 @@ services: build: context: . target: migrations - restart: on-failure + restart: no command: migrate environment: FLYWAY_URL: jdbc:postgresql://db/?user=$USER&password= From 0210d3c9debd01dd7d74bede2c8a64f769ff3057 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Wed, 27 Nov 2024 11:32:47 +0100 Subject: [PATCH 28/70] Reverted yaml file changes --- .github/workflows/pull-request.yaml | 1 - docker-compose.yaml | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index c313a22472..0475fe7ecf 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -91,7 +91,6 @@ jobs: with: file: docker-compose.yaml up-opts: -d db migrations - - run: docker wait services-migrations-1 - run: cargo nextest run postgres -p orderbook -p database -p autopilot --test-threads 1 --run-ignored ignored-only test-local-node: diff --git a/docker-compose.yaml b/docker-compose.yaml index 179577bf34..cdc9721cd0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -24,7 +24,7 @@ services: build: context: . target: migrations - restart: no + restart: on-failure command: migrate environment: FLYWAY_URL: jdbc:postgresql://db/?user=$USER&password= @@ -32,8 +32,6 @@ services: - type: bind source: ./database/sql/ target: /flyway/sql - depends_on: - - db volumes: postgres: From 77fc40665cfde90665b641439e882f868690b16c Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Wed, 27 Nov 2024 11:33:06 +0100 Subject: [PATCH 29/70] Fixed migration script --- database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql index 904f64d787..15be7380ad 100644 --- a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql +++ b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql @@ -32,4 +32,4 @@ CREATE TABLE order_quotes_interactions ( ); -- Get a specific order's interactions. -CREATE INDEX order_uid_interactions ON orders_quotes_interactions USING HASH (order_uid); +CREATE INDEX order_uid_interactions ON order_quotes_interactions USING HASH (order_uid); From 69dd7faeee9ec9abebfd32697b60ba4b48b6234a Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Wed, 27 Nov 2024 17:15:27 +0100 Subject: [PATCH 30/70] Fixed test --- crates/e2e/tests/e2e/quote_verification.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/e2e/tests/e2e/quote_verification.rs b/crates/e2e/tests/e2e/quote_verification.rs index 2c94e604d3..b870598cf8 100644 --- a/crates/e2e/tests/e2e/quote_verification.rs +++ b/crates/e2e/tests/e2e/quote_verification.rs @@ -3,6 +3,7 @@ use { ethcontract::H160, ethrpc::Web3, model::{ + interaction::InteractionData, order::{BuyTokenDestination, OrderKind, SellTokenSource}, quote::{OrderQuoteRequest, OrderQuoteSide, SellAmount}, }, @@ -109,7 +110,11 @@ async fn test_bypass_verification_for_rfq_quotes(web3: Web3) { gas: 225000, solver: H160::from_str("0xe3067c7c27c1038de4e8ad95a83b927d23dfbd99").unwrap(), verified: true, - interactions: vec![], + interactions: vec![InteractionData { + target: H160::from_str("0xdef1c0ded9bec7f1a1670819833240f027b25eff").unwrap(), + value: 0.into(), + call_data: hex::decode("aa77476c000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000002260fac5e5542a773aa44fbcfedf7c193bc2c599000000000000000000000000000000000000000000000000e357b42c3a9d8ccf0000000000000000000000000000000000000000000000000000000004d0e79e000000000000000000000000a69babef1ca67a37ffaf7a485dfff3382056e78c0000000000000000000000009008d19f58aabd9ed0d60971565aa8510560ab41000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000066360af101ffffffffffffffffffffffffffffffffffffff0f3f47f166360a8d0000003f0000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000001c66b3383f287dd9c85ad90e7c5a576ea4ba1bdf5a001d794a9afa379e6b2517b47e487a1aef32e75af432cbdbd301ada42754eaeac21ec4ca744afd92732f47540000000000000000000000000000000000000000000000000000000004d0c80f").unwrap() + }], }; // `tx_origin: 0x0000` is currently used to bypass quote verification due to an From 757029a5c492dbfb7f0cd576a2ac649a597b8f07 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Thu, 28 Nov 2024 09:44:58 +0100 Subject: [PATCH 31/70] Updated new table names --- crates/database/src/lib.rs | 4 ++-- crates/database/src/orders.rs | 10 +++++----- crates/database/src/quotes.rs | 8 ++++---- .../V075__add_call_data_to_quotes_and_order_quotes.sql | 8 ++++---- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/crates/database/src/lib.rs b/crates/database/src/lib.rs index a2577aa2e9..8b59d60243 100644 --- a/crates/database/src/lib.rs +++ b/crates/database/src/lib.rs @@ -72,8 +72,8 @@ pub const TABLES: &[&str] = &[ "auction_participants", "app_data", "jit_orders", - "quotes_interactions", - "order_quotes_interactions", + "quote_interactions", + "order_quote_interactions", ]; /// The names of potentially big volume tables we use in the db. diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index e49e6c2a67..2749cc86fc 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -449,7 +449,7 @@ AND cancellation_timestamp IS NULL .map(|_| ()) } -/// One row in the `order_quotes_interactions` table. +/// One row in the `order_quote_interactions` table. #[derive(Clone, Default, Debug, PartialEq, sqlx::FromRow)] pub struct OrderQuoteInteraction { pub order_uid: OrderUid, @@ -463,8 +463,8 @@ pub async fn insert_order_quote_interaction( ex: &mut PgConnection, quote_interaction: &OrderQuoteInteraction, ) -> Result<(), sqlx::Error> { - const INSERT_ORDER_QUOTES_INTERACTION_QUERY: &str = r#" - INSERT INTO order_quotes_interactions ( + const INSERT_ORDER_QUOTE_INTERACTION_QUERY: &str = r#" + INSERT INTO order_quote_interactions ( order_uid, index, target, @@ -483,7 +483,7 @@ pub async fn insert_order_quote_interaction( EXCLUDED.call_data ) "#; - sqlx::query(INSERT_ORDER_QUOTES_INTERACTION_QUERY) + sqlx::query(INSERT_ORDER_QUOTE_INTERACTION_QUERY) .bind(quote_interaction.order_uid) .bind(quote_interaction.index) .bind(quote_interaction.target) @@ -2226,7 +2226,7 @@ mod tests { .unwrap(); const QUERY: &str = r#" - SELECT * FROM order_quotes_interactions + SELECT * FROM order_quote_interactions WHERE order_uid = $1 "#; diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 959e886f69..fe15fd68ac 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -153,7 +153,7 @@ RETURNING id delete_quote_interactions(ex, &deleted_ids).await } -/// One row in the `quotes_interactions` table. +/// One row in the `quote_interactions` table. #[derive(Clone, Debug, PartialEq, sqlx::FromRow)] pub struct QuoteInteraction { pub quote_id: QuoteId, @@ -169,7 +169,7 @@ pub async fn insert_quote_interaction( quote_interaction: &QuoteInteraction, ) -> Result<(), sqlx::Error> { const QUERY: &str = r#" -INSERT INTO quotes_interactions ( +INSERT INTO quote_interactions ( quote_id, index, target, @@ -205,7 +205,7 @@ pub async fn get_quote_interactions( ) -> Result, sqlx::Error> { const QUERY: &str = r#" SELECT * -FROM quotes_interactions +FROM quote_interactions WHERE quote_id = $1 "#; sqlx::query_as(QUERY).bind(quote_id).fetch_all(ex).await @@ -216,7 +216,7 @@ pub async fn delete_quote_interactions( quote_ids: &[QuoteId], ) -> Result<(), sqlx::Error> { const QUERY: &str = r#" -DELETE FROM quotes_interactions +DELETE FROM quote_interactions WHERE quote_id = ANY($1) "#; diff --git a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql index 15be7380ad..e7b25a75c2 100644 --- a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql +++ b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql @@ -7,7 +7,7 @@ ALTER TABLE order_quotes ADD COLUMN verified boolean; -- Step 3: Create table with quote interactions -CREATE TABLE quotes_interactions ( +CREATE TABLE quote_interactions ( quote_id bigint NOT NULL, index int NOT NULL, target bytea NOT NULL, @@ -18,10 +18,10 @@ CREATE TABLE quotes_interactions ( ); -- Get a specific quote's interactions. -CREATE INDEX quote_id_interactions ON quotes_interactions USING HASH (quote_id); +CREATE INDEX quote_id_interactions ON quote_interactions USING HASH (quote_id); -- Step 3: Create table with quote interactions -CREATE TABLE order_quotes_interactions ( +CREATE TABLE order_quote_interactions ( order_uid bytea NOT NULL, index int NOT NULL, target bytea NOT NULL, @@ -32,4 +32,4 @@ CREATE TABLE order_quotes_interactions ( ); -- Get a specific order's interactions. -CREATE INDEX order_uid_interactions ON order_quotes_interactions USING HASH (order_uid); +CREATE INDEX order_uid_interactions ON order_quote_interactions USING HASH (order_uid); From 085c5304b6c3e5b225c912df0eb0e5fb0028fccf Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Thu, 28 Nov 2024 10:16:13 +0100 Subject: [PATCH 32/70] Updated db readme & migration script --- database/README.md | 37 ++++++++++++++++++- ...d_call_data_to_quotes_and_order_quotes.sql | 6 +-- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/database/README.md b/database/README.md index 171561a6a6..6933fe29f6 100644 --- a/database/README.md +++ b/database/README.md @@ -238,7 +238,6 @@ Quotes that an order was created with. These quotes get stored persistently and sell\_amount | numeric | not null | sell\_amount of the quote used to create the order with buy\_amount | numeric | not null | buy\_amount of the quote used to create the order with solver | bytea | not null | public address of the solver that provided this quote - call\_data | bytea | | call\_data provided by solver in response to the /quote request verified | boolean | | information if quote was verified Indexes: @@ -276,6 +275,23 @@ Column | Type | Nullable | Details Indexes: - PRIMARY KEY: btree(`uid`) +### order\_quote\_interactions + +Solver responding to quote request provides a list of interactions which need to be executed to fulfill the quote. When order is created +these interactions are copied from quote. These interactions are stored persistently and can be used to audit auction winning order. + + Colmun | Type | Nullable | Details +--------------------|---------|----------|-------- + order\_uid | bytea | not null | order that this interaction belongs to + index | integer | not null | sequence number of the interaction for that order + target | bytea | not null | address of the smart contract this interaction should call + value | numeric | not null | amount of ETH this interaction should send to the smart contract + call\_data | bytea | not null | call data that contains the function selector and the bytes passed to it + +Indexes: +- PRIMARY KEY: composite key(`order_uid`, `index`) +- order\_uid\_interactions: hash(`order_uid`) + ### fee_policies Contains all relevant data of fee policies applied to orders during auctions. @@ -341,13 +357,30 @@ Stores quotes in order to determine whether it makes sense to allow a user to cr id | bigint | not null | unique identifier of this quote quote\_kind | [enum](#quotekind) | not null | quotekind for which this quote is considered valid solver | bytea | not null | public address of the solver that provided this quote - call\_data | bytea | | call\_data provided by solver in response to the /quote request verified | boolean | | information if quote was verified Indexes: - PRIMARY KEY: btree(`id`) - quotes\_token\_expiration: btree (`sell_token`, `buy_token`, `expiration_timestamp` DESC) +### quote\_interactions + +Solver responding to quote request provides a list of interactions which need to be executed to fulfill the quote. This table stores +these interactions. Later, when order is created basing on this quote, interactions are copied to `order_quote_interactions`. +Data in this table is removed together with the referenced quote, usually at quote expiration. + + Colmun | Type | Nullable | Details +--------------------|---------|----------|-------- + quote_id | bigint | not null | quote that this interaction belongs to + index | integer | not null | sequence number of the interaction for that quote + target | bytea | not null | address of the smart contract this interaction should call + value | numeric | not null | amount of ETH this interaction should send to the smart contract + call\_data | bytea | not null | call data that contains the function selector and the bytes passed to it + +Indexes: +- PRIMARY KEY: composite key(`quote_id`, `index`) +- quote\_id\_interactions: hash(`quote_id`) + ### proposed\_solutions All solutions reported by solvers, that were part of a solver competition. A solver competition can have more than one winner. diff --git a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql index e7b25a75c2..5c41e0ed47 100644 --- a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql +++ b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql @@ -12,7 +12,7 @@ CREATE TABLE quote_interactions ( index int NOT NULL, target bytea NOT NULL, value numeric(78,0) NOT NULL, - call_data bytea, + call_data bytea NOT NULL, PRIMARY KEY (quote_id, index) ); @@ -20,13 +20,13 @@ CREATE TABLE quote_interactions ( -- Get a specific quote's interactions. CREATE INDEX quote_id_interactions ON quote_interactions USING HASH (quote_id); --- Step 3: Create table with quote interactions +-- Step 4: Create table with quote interactions for order CREATE TABLE order_quote_interactions ( order_uid bytea NOT NULL, index int NOT NULL, target bytea NOT NULL, value numeric(78,0) NOT NULL, - call_data bytea, + call_data bytea NOT NULL, PRIMARY KEY (order_uid, index) ); From 67cd65c9afaed6bcd89ba2a0cb9faab7d732896e Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Thu, 28 Nov 2024 23:28:18 +0100 Subject: [PATCH 33/70] Code cleanup --- crates/autopilot/src/database/auction.rs | 46 +++++++++---------- crates/orderbook/src/database/mod.rs | 24 +++++++++- crates/orderbook/src/database/quotes.rs | 31 ++----------- .../src/price_estimation/trade_verifier.rs | 10 ++-- crates/shared/src/trade_finding/external.rs | 11 ++++- ...d_call_data_to_quotes_and_order_quotes.sql | 2 + 6 files changed, 62 insertions(+), 62 deletions(-) diff --git a/crates/autopilot/src/database/auction.rs b/crates/autopilot/src/database/auction.rs index 1dedce31da..90b2cbdb3a 100644 --- a/crates/autopilot/src/database/auction.rs +++ b/crates/autopilot/src/database/auction.rs @@ -17,6 +17,7 @@ use { }, order_quoting::{QuoteData, QuoteSearchParameters, QuoteStoring}, }, + sqlx::PgConnection, std::{collections::HashMap, ops::DerefMut}, }; @@ -46,18 +47,7 @@ impl QuoteStoring for Postgres { let mut ex = self.pool.acquire().await?; let quote = database::quotes::get(&mut ex, id).await?; - let quote_interactions = database::quotes::get_quote_interactions(&mut ex, id) - .await? - .iter() - .map(|data| { - Ok(InteractionData { - target: H160(data.target.0), - value: big_decimal_to_u256(&data.value) - .context("quote interaction value is not a valid U256")?, - call_data: data.call_data.clone(), - }) - }) - .collect::>>()?; + let quote_interactions = Self::get_quote_interactions(&mut ex, id).await?; Ok(quote .map(QuoteData::try_from) .transpose()? @@ -84,19 +74,7 @@ impl QuoteStoring for Postgres { .context("failed finding quote by parameters")?; if let Some(quote) = quote { let quote_id = quote.id; - let quote_interactions = database::quotes::get_quote_interactions(&mut ex, quote_id) - .await? - .iter() - .map(|data| { - Ok(InteractionData { - target: H160(data.target.0), - value: big_decimal_to_u256(&data.value) - .context("quote interaction value is not a valid U256")?, - call_data: data.call_data.clone(), - }) - }) - .collect::>>()?; - + let quote_interactions = Self::get_quote_interactions(&mut ex, quote_id).await?; let mut quote_data = QuoteData::try_from(quote)?; quote_data.interactions = quote_interactions; Ok(Some((quote_id, quote_data))) @@ -157,4 +135,22 @@ impl Postgres { let id = database::auction::replace_auction(&mut ex, &data).await?; Ok(id) } + + async fn get_quote_interactions( + ex: &mut PgConnection, + quote_id: QuoteId, + ) -> Result> { + database::quotes::get_quote_interactions(ex, quote_id) + .await? + .iter() + .map(|data| { + Ok(InteractionData { + target: H160(data.target.0), + value: big_decimal_to_u256(&data.value) + .context("quote interaction value is not a valid U256")?, + call_data: data.call_data.clone(), + }) + }) + .collect::>>() + } } diff --git a/crates/orderbook/src/database/mod.rs b/crates/orderbook/src/database/mod.rs index 25d10f8c17..7fe2aa0ec1 100644 --- a/crates/orderbook/src/database/mod.rs +++ b/crates/orderbook/src/database/mod.rs @@ -10,9 +10,11 @@ pub mod trades; use { crate::database::orders::InsertionError, - anyhow::Result, + anyhow::{Context, Result}, database::byte_array::ByteArray, - model::order::Order, + model::{interaction::InteractionData, order::Order, quote::QuoteId}, + number::conversions::big_decimal_to_u256, + primitive_types::H160, sqlx::{PgConnection, PgPool}, }; @@ -53,6 +55,24 @@ impl Postgres { } Ok(()) } + + async fn get_quote_interactions( + ex: &mut PgConnection, + quote_id: QuoteId, + ) -> Result> { + database::quotes::get_quote_interactions(ex, quote_id) + .await? + .iter() + .map(|data| { + Ok(InteractionData { + target: H160(data.target.0), + value: big_decimal_to_u256(&data.value) + .context("quote interaction value is not a valid U256")?, + call_data: data.call_data.clone(), + }) + }) + .collect::>>() + } } #[derive(prometheus_metric_storage::MetricStorage)] diff --git a/crates/orderbook/src/database/quotes.rs b/crates/orderbook/src/database/quotes.rs index 8d66416e5a..27be954fcc 100644 --- a/crates/orderbook/src/database/quotes.rs +++ b/crates/orderbook/src/database/quotes.rs @@ -2,9 +2,7 @@ use { super::Postgres, anyhow::{Context, Result}, chrono::{DateTime, Utc}, - model::{interaction::InteractionData, quote::QuoteId}, - number::conversions::big_decimal_to_u256, - primitive_types::H160, + model::quote::QuoteId, shared::{ event_storing_helpers::{ create_db_search_parameters, @@ -41,18 +39,8 @@ impl QuoteStoring for Postgres { let mut ex = self.pool.acquire().await?; let quote = database::quotes::get(&mut ex, id).await?; - let quote_interactions = database::quotes::get_quote_interactions(&mut ex, id) - .await? - .iter() - .map(|data| { - Ok(InteractionData { - target: H160(data.target.0), - value: big_decimal_to_u256(&data.value) - .context("quote interaction value is not a valid U256")?, - call_data: data.call_data.clone(), - }) - }) - .collect::>>()?; + let quote_interactions = Self::get_quote_interactions(&mut ex, id).await?; + Ok(quote .map(QuoteData::try_from) .transpose()? @@ -79,18 +67,7 @@ impl QuoteStoring for Postgres { .context("failed finding quote by parameters")?; if let Some(quote) = quote { let quote_id = quote.id; - let quote_interactions = database::quotes::get_quote_interactions(&mut ex, quote_id) - .await? - .iter() - .map(|data| { - Ok(InteractionData { - target: H160(data.target.0), - value: big_decimal_to_u256(&data.value) - .context("quote interaction value is not a valid U256")?, - call_data: data.call_data.clone(), - }) - }) - .collect::>>()?; + let quote_interactions = Self::get_quote_interactions(&mut ex, quote_id).await?; let mut quote_data = QuoteData::try_from(quote)?; quote_data.interactions = quote_interactions; diff --git a/crates/shared/src/price_estimation/trade_verifier.rs b/crates/shared/src/price_estimation/trade_verifier.rs index fcefb25cc3..fe9e98fcae 100644 --- a/crates/shared/src/price_estimation/trade_verifier.rs +++ b/crates/shared/src/price_estimation/trade_verifier.rs @@ -5,7 +5,7 @@ use { code_simulation::CodeSimulating, encoded_settlement::{encode_trade, EncodedSettlement}, interaction::EncodedInteraction, - trade_finding::{Interaction, Trade}, + trade_finding::{map_interactions_data, Interaction, Trade}, }, anyhow::{Context, Result}, contracts::{ @@ -179,7 +179,7 @@ impl TradeVerifier { gas: trade.gas_estimate.context("no gas estimate")?, solver: trade.solver, verified: true, - interactions: crate::trade_finding::map_interactions_data(&trade.interactions), + interactions: map_interactions_data(&trade.interactions), }; tracing::warn!( ?estimate, @@ -313,9 +313,7 @@ impl TradeVerifying for TradeVerifier { gas, solver: trade.solver, verified: false, - interactions: crate::trade_finding::map_interactions_data( - &trade.interactions, - ), + interactions: map_interactions_data(&trade.interactions), }; tracing::warn!( ?err, @@ -532,7 +530,7 @@ fn ensure_quote_accuracy( gas: summary.gas_used.as_u64(), solver: trade.solver, verified: true, - interactions: crate::trade_finding::map_interactions_data(&trade.interactions), + interactions: map_interactions_data(&trade.interactions), }) } diff --git a/crates/shared/src/trade_finding/external.rs b/crates/shared/src/trade_finding/external.rs index 72e738e66a..47e844a96d 100644 --- a/crates/shared/src/trade_finding/external.rs +++ b/crates/shared/src/trade_finding/external.rs @@ -4,7 +4,14 @@ use { crate::{ price_estimation::{PriceEstimationError, Query}, request_sharing::{BoxRequestSharing, RequestSharing}, - trade_finding::{Interaction, Quote, Trade, TradeError, TradeFinding}, + trade_finding::{ + map_interactions_data, + Interaction, + Quote, + Trade, + TradeError, + TradeFinding, + }, }, anyhow::{anyhow, Context}, ethrpc::block_stream::CurrentBlockWatcher, @@ -170,7 +177,7 @@ impl TradeFinding for ExternalTradeFinder { out_amount: trade.out_amount, gas_estimate, solver: trade.solver, - interactions: crate::trade_finding::map_interactions_data(&trade.interactions), + interactions: map_interactions_data(&trade.interactions), }) } diff --git a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql index 5c41e0ed47..0d0f50576b 100644 --- a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql +++ b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql @@ -1,3 +1,5 @@ +-- This migration script is reversible. + -- Step 1: Add a new column to the quotes table ALTER TABLE quotes ADD COLUMN verified boolean; From 39cade5bf7a55cb95551c5adad6d24285a8b97ae Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Thu, 28 Nov 2024 23:36:04 +0100 Subject: [PATCH 34/70] Updated tests --- crates/database/src/orders.rs | 43 +++++++++++++++++++++++++ crates/database/src/quotes.rs | 2 +- crates/orderbook/src/database/orders.rs | 2 +- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 2749cc86fc..72d0bf2d3a 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -2237,4 +2237,47 @@ mod tests { .unwrap(); assert_eq!(*interactions.first().unwrap(), interaction); } + + #[tokio::test] + #[ignore] + async fn postgres_insert_order_quote_interaction_on_conflict() { + let mut db = PgConnection::connect("postgresql://").await.unwrap(); + let mut db = db.begin().await.unwrap(); + crate::clear_DANGER_(&mut db).await.unwrap(); + + let order_uid = Default::default(); + let interaction1 = OrderQuoteInteraction { + order_uid, + index: Default::default(), + target: ByteArray([1; 20]), + value: 2.into(), + call_data: vec![3; 20], + }; + insert_order_quote_interaction(&mut db, &interaction1) + .await + .unwrap(); + + let interaction2 = OrderQuoteInteraction { + order_uid, + index: Default::default(), + target: ByteArray([4; 20]), + value: 4.into(), + call_data: vec![5; 20], + }; + insert_order_quote_interaction(&mut db, &interaction2) + .await + .unwrap(); + + const QUERY: &str = r#" + SELECT * FROM order_quote_interactions + WHERE order_uid = $1 + "#; + + let interactions: Vec = sqlx::query_as(QUERY) + .bind(order_uid) + .fetch_all(&mut db as &mut PgConnection) + .await + .unwrap(); + assert_eq!(*interactions.first().unwrap(), interaction2); + } } diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index fe15fd68ac..6d6cd1a47d 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -509,7 +509,7 @@ mod tests { #[tokio::test] #[ignore] - async fn postgres_insert_quote_interaction_by_id() { + async fn postgres_insert_quote_interaction() { let mut db = PgConnection::connect("postgresql://").await.unwrap(); let mut db = db.begin().await.unwrap(); crate::clear_DANGER_(&mut db).await.unwrap(); diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index ad412a3145..bbc00c70ac 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -1201,7 +1201,7 @@ mod tests { #[tokio::test] #[ignore] - async fn postgres_insert_orders_with_call_data_and_verified() { + async fn postgres_insert_orders_with_interactions_and_verified() { let db = Postgres::new("postgresql://").unwrap(); database::clear_DANGER(&db.pool).await.unwrap(); From 52399018573b4379bb9a44f6db677774ad6fa20b Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 29 Nov 2024 14:34:57 +0100 Subject: [PATCH 35/70] Added transaction for storing interactions --- crates/autopilot/src/database/auction.rs | 9 ++++++--- crates/orderbook/src/database/orders.rs | 14 ++++++++------ crates/orderbook/src/database/quotes.rs | 8 ++++++-- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/crates/autopilot/src/database/auction.rs b/crates/autopilot/src/database/auction.rs index 90b2cbdb3a..8bf5887027 100644 --- a/crates/autopilot/src/database/auction.rs +++ b/crates/autopilot/src/database/auction.rs @@ -17,7 +17,7 @@ use { }, order_quoting::{QuoteData, QuoteSearchParameters, QuoteStoring}, }, - sqlx::PgConnection, + sqlx::{Acquire, PgConnection}, std::{collections::HashMap, ops::DerefMut}, }; @@ -31,11 +31,14 @@ impl QuoteStoring for Postgres { let mut ex = self.pool.acquire().await?; let row = create_quote_row(&data); - let id = database::quotes::save(&mut ex, &row).await?; + + let mut transaction = ex.begin().await?; + let id = database::quotes::save(&mut transaction, &row).await?; if !data.interactions.is_empty() { let interactions = create_quote_interactions_insert_data(id, &data)?; - database::quotes::insert_quote_interactions(&mut ex, &interactions).await?; + database::quotes::insert_quote_interactions(&mut transaction, &interactions).await?; } + transaction.commit().await.context("commit")?; Ok(id) } diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index bbc00c70ac..2d9e522aca 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -239,10 +239,6 @@ async fn insert_quote( solver: ByteArray(quote.data.solver.0), verified: quote.data.verified, }; - database::orders::insert_quote(ex, &dbquote) - .await - .map_err(InsertionError::DbError)?; - let dbinteractions = quote .data .interactions @@ -259,9 +255,15 @@ async fn insert_quote( }) .collect::>>() .map_err(|_| InsertionError::IndexConversionFailed)?; - database::orders::insert_order_quote_interactions(ex, dbinteractions.as_slice()) + + let mut transaction = ex.begin().await?; + database::orders::insert_quote(&mut transaction, &dbquote) .await - .map_err(InsertionError::DbError) + .map_err(InsertionError::DbError)?; + database::orders::insert_order_quote_interactions(&mut transaction, dbinteractions.as_slice()) + .await + .map_err(InsertionError::DbError)?; + transaction.commit().await.map_err(InsertionError::DbError) } #[async_trait::async_trait] diff --git a/crates/orderbook/src/database/quotes.rs b/crates/orderbook/src/database/quotes.rs index 27be954fcc..acc37fe52e 100644 --- a/crates/orderbook/src/database/quotes.rs +++ b/crates/orderbook/src/database/quotes.rs @@ -11,6 +11,7 @@ use { }, order_quoting::{QuoteData, QuoteSearchParameters, QuoteStoring}, }, + sqlx::Acquire, }; #[async_trait::async_trait] @@ -23,11 +24,14 @@ impl QuoteStoring for Postgres { let mut ex = self.pool.acquire().await?; let row = create_quote_row(&data); - let id = database::quotes::save(&mut ex, &row).await?; + + let mut transaction = ex.begin().await?; + let id = database::quotes::save(&mut transaction, &row).await?; if !data.interactions.is_empty() { let interactions = create_quote_interactions_insert_data(id, &data)?; - database::quotes::insert_quote_interactions(&mut ex, &interactions).await?; + database::quotes::insert_quote_interactions(&mut transaction, &interactions).await?; } + transaction.commit().await.context("commit")?; Ok(id) } From 5aac1dc6c91403e52f92abc617156b8f564f5dbb Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 29 Nov 2024 15:12:00 +0100 Subject: [PATCH 36/70] Updated removing of expired quote interactions --- crates/database/src/lib.rs | 3 ++- crates/database/src/quotes.rs | 27 +++---------------- ...d_call_data_to_quotes_and_order_quotes.sql | 3 ++- 3 files changed, 8 insertions(+), 25 deletions(-) diff --git a/crates/database/src/lib.rs b/crates/database/src/lib.rs index 8b59d60243..48e6496ef6 100644 --- a/crates/database/src/lib.rs +++ b/crates/database/src/lib.rs @@ -87,7 +87,8 @@ pub fn all_tables() -> impl Iterator { #[allow(non_snake_case)] pub async fn clear_DANGER_(ex: &mut PgTransaction<'_>) -> sqlx::Result<()> { for table in all_tables() { - ex.execute(format!("TRUNCATE {table};").as_str()).await?; + ex.execute(format!("TRUNCATE {table} CASCADE;").as_str()) + .await?; } Ok(()) } diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 6d6cd1a47d..24a62b04d2 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -4,7 +4,6 @@ use { sqlx::{ types::chrono::{DateTime, Utc}, PgConnection, - Row, }, }; @@ -140,17 +139,12 @@ pub async fn remove_expired_quotes( const QUERY: &str = r#" DELETE FROM quotes WHERE expiration_timestamp < $1 -RETURNING id "#; - let deleted_ids = sqlx::query(QUERY) + sqlx::query(QUERY) .bind(max_expiry) - .fetch_all(&mut *ex) - .await? - .iter() - .map(|row| row.get(0)) - .collect::>(); - - delete_quote_interactions(ex, &deleted_ids).await + .execute(ex) + .await + .map(|_| ()) } /// One row in the `quote_interactions` table. @@ -211,19 +205,6 @@ WHERE quote_id = $1 sqlx::query_as(QUERY).bind(quote_id).fetch_all(ex).await } -pub async fn delete_quote_interactions( - ex: &mut PgConnection, - quote_ids: &[QuoteId], -) -> Result<(), sqlx::Error> { - const QUERY: &str = r#" -DELETE FROM quote_interactions -WHERE quote_id = ANY($1) - "#; - - sqlx::query(QUERY).bind(quote_ids).execute(ex).await?; - Ok(()) -} - #[cfg(test)] mod tests { use { diff --git a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql index 0d0f50576b..bbb19aed3b 100644 --- a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql +++ b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql @@ -16,7 +16,8 @@ CREATE TABLE quote_interactions ( value numeric(78,0) NOT NULL, call_data bytea NOT NULL, - PRIMARY KEY (quote_id, index) + PRIMARY KEY (quote_id, index), + FOREIGN KEY (quote_id) REFERENCES quotes(id) ON DELETE CASCADE ); -- Get a specific quote's interactions. From 23b21828b279a1b252b3853494a256f83689e495 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 29 Nov 2024 15:17:33 +0100 Subject: [PATCH 37/70] Removed updated on conflict from insert order quote interaction --- crates/database/src/orders.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 72d0bf2d3a..440090aa76 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -472,16 +472,6 @@ pub async fn insert_order_quote_interaction( call_data ) VALUES ($1, $2, $3, $4, $5) - ON CONFLICT (order_uid, index) DO UPDATE SET - ( - target, - value, - call_data - ) = ( - EXCLUDED.target, - EXCLUDED.value, - EXCLUDED.call_data - ) "#; sqlx::query(INSERT_ORDER_QUOTE_INTERACTION_QUERY) .bind(quote_interaction.order_uid) From f4f040c82095df107e4b97295029798ef7ca07ee Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 29 Nov 2024 15:32:54 +0100 Subject: [PATCH 38/70] Updated test --- crates/database/src/quotes.rs | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 24a62b04d2..01685a7e06 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -491,12 +491,33 @@ mod tests { #[tokio::test] #[ignore] async fn postgres_insert_quote_interaction() { - let mut db = PgConnection::connect("postgresql://").await.unwrap(); + let mut db = + PgConnection::connect("postgresql://127.0.0.1:5432/?user=postgres&password=123") + .await + .unwrap(); let mut db = db.begin().await.unwrap(); crate::clear_DANGER_(&mut db).await.unwrap(); + let quote = Quote { + id: Default::default(), + sell_token: ByteArray([1; 20]), + buy_token: ByteArray([2; 20]), + sell_amount: 3.into(), + buy_amount: 4.into(), + gas_amount: 5., + gas_price: 6., + sell_token_price: 7., + order_kind: OrderKind::Sell, + expiration_timestamp: low_precision_now(), + quote_kind: QuoteKind::Standard, + solver: ByteArray([1; 20]), + verified: false, + }; + // store quote in database + let id = save(&mut db, "e).await.unwrap(); + let quote_interaction = QuoteInteraction { - quote_id: Default::default(), + quote_id: id, index: Default::default(), target: ByteArray([1; 20]), value: 2.into(), @@ -515,7 +536,10 @@ mod tests { #[tokio::test] #[ignore] async fn postgres_removed_quote_interactions_by_id() { - let mut db = PgConnection::connect("postgresql://").await.unwrap(); + let mut db = + PgConnection::connect("postgresql://127.0.0.1:5432/?user=postgres&password=123") + .await + .unwrap(); let mut db = db.begin().await.unwrap(); crate::clear_DANGER_(&mut db).await.unwrap(); From 8848b73a82d03d9ef932f206ef05c3f9d3305069 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 29 Nov 2024 15:37:57 +0100 Subject: [PATCH 39/70] Updated db readme --- database/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/database/README.md b/database/README.md index 6933fe29f6..e86fa5b34b 100644 --- a/database/README.md +++ b/database/README.md @@ -229,7 +229,7 @@ Indexes: Quotes that an order was created with. These quotes get stored persistently and can be used to evaluate how accurate the quoted fee predicted the execution cost that actually happened on-chain. - Colmun | Type | Nullable | Details + Column | Type | Nullable | Details --------------------|---------|----------|-------- order\_uid | bytea | not null | order that this quote belongs to gas\_amount | double | not null | estimated gas used by the quote used to create this order with @@ -280,7 +280,7 @@ Indexes: Solver responding to quote request provides a list of interactions which need to be executed to fulfill the quote. When order is created these interactions are copied from quote. These interactions are stored persistently and can be used to audit auction winning order. - Colmun | Type | Nullable | Details + Column | Type | Nullable | Details --------------------|---------|----------|-------- order\_uid | bytea | not null | order that this interaction belongs to index | integer | not null | sequence number of the interaction for that order @@ -369,7 +369,7 @@ Solver responding to quote request provides a list of interactions which need to these interactions. Later, when order is created basing on this quote, interactions are copied to `order_quote_interactions`. Data in this table is removed together with the referenced quote, usually at quote expiration. - Colmun | Type | Nullable | Details + Column | Type | Nullable | Details --------------------|---------|----------|-------- quote_id | bigint | not null | quote that this interaction belongs to index | integer | not null | sequence number of the interaction for that quote @@ -379,6 +379,7 @@ Data in this table is removed together with the referenced quote, usually at quo Indexes: - PRIMARY KEY: composite key(`quote_id`, `index`) +- FOREIGN KEY: `quote_id` into `quotes(id)` table - quote\_id\_interactions: hash(`quote_id`) ### proposed\_solutions From 0afb9ad2b857c4cea3724764b2557bbff83f5ded Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 29 Nov 2024 17:08:27 +0100 Subject: [PATCH 40/70] Ensuring repeated db inserts uses transaction --- crates/database/src/orders.rs | 22 +++++++--------------- crates/database/src/quotes.rs | 4 ++-- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 440090aa76..b8ad30b3ac 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -5,6 +5,7 @@ use { Address, AppId, OrderUid, + PgTransaction, TransactionHash, }, futures::stream::BoxStream, @@ -485,7 +486,7 @@ pub async fn insert_order_quote_interaction( } pub async fn insert_order_quote_interactions( - ex: &mut PgConnection, + ex: &mut PgTransaction<'_>, quote_interactions: &[OrderQuoteInteraction], ) -> Result<(), sqlx::Error> { for interactions in quote_interactions { @@ -2231,7 +2232,10 @@ mod tests { #[tokio::test] #[ignore] async fn postgres_insert_order_quote_interaction_on_conflict() { - let mut db = PgConnection::connect("postgresql://").await.unwrap(); + let mut db = + PgConnection::connect("postgresql://127.0.0.1:5432/?user=postgres&password=123") + .await + .unwrap(); let mut db = db.begin().await.unwrap(); crate::clear_DANGER_(&mut db).await.unwrap(); @@ -2256,18 +2260,6 @@ mod tests { }; insert_order_quote_interaction(&mut db, &interaction2) .await - .unwrap(); - - const QUERY: &str = r#" - SELECT * FROM order_quote_interactions - WHERE order_uid = $1 - "#; - - let interactions: Vec = sqlx::query_as(QUERY) - .bind(order_uid) - .fetch_all(&mut db as &mut PgConnection) - .await - .unwrap(); - assert_eq!(*interactions.first().unwrap(), interaction2); + .expect_err("Inserting interaction for the same key should fail."); } } diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 01685a7e06..9262c4801e 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -1,5 +1,5 @@ use { - crate::{orders::OrderKind, Address}, + crate::{orders::OrderKind, Address, PgTransaction}, bigdecimal::BigDecimal, sqlx::{ types::chrono::{DateTime, Utc}, @@ -184,7 +184,7 @@ VALUES ($1, $2, $3, $4, $5) } pub async fn insert_quote_interactions( - ex: &mut PgConnection, + ex: &mut PgTransaction<'_>, quote_interactions: &[QuoteInteraction], ) -> Result<(), sqlx::Error> { for interaction in quote_interactions { From 2f610ae0dcb0dc2b9930cc0e662a0e86646652db Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 29 Nov 2024 17:20:09 +0100 Subject: [PATCH 41/70] Fixed tests --- crates/database/src/orders.rs | 5 +---- crates/database/src/quotes.rs | 10 ++-------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index b8ad30b3ac..d4c67b9cad 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -2232,10 +2232,7 @@ mod tests { #[tokio::test] #[ignore] async fn postgres_insert_order_quote_interaction_on_conflict() { - let mut db = - PgConnection::connect("postgresql://127.0.0.1:5432/?user=postgres&password=123") - .await - .unwrap(); + let mut db = PgConnection::connect("postgresql://").await.unwrap(); let mut db = db.begin().await.unwrap(); crate::clear_DANGER_(&mut db).await.unwrap(); diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 9262c4801e..06cfaf4637 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -491,10 +491,7 @@ mod tests { #[tokio::test] #[ignore] async fn postgres_insert_quote_interaction() { - let mut db = - PgConnection::connect("postgresql://127.0.0.1:5432/?user=postgres&password=123") - .await - .unwrap(); + let mut db = PgConnection::connect("postgresql://").await.unwrap(); let mut db = db.begin().await.unwrap(); crate::clear_DANGER_(&mut db).await.unwrap(); @@ -536,10 +533,7 @@ mod tests { #[tokio::test] #[ignore] async fn postgres_removed_quote_interactions_by_id() { - let mut db = - PgConnection::connect("postgresql://127.0.0.1:5432/?user=postgres&password=123") - .await - .unwrap(); + let mut db = PgConnection::connect("postgresql://").await.unwrap(); let mut db = db.begin().await.unwrap(); crate::clear_DANGER_(&mut db).await.unwrap(); From 99796453032bc2e76f5b0f1397ebf6c8ba2df574 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 29 Nov 2024 23:36:15 +0100 Subject: [PATCH 42/70] Optimized db queries using join --- crates/autopilot/src/database/auction.rs | 52 +++------- crates/database/src/quotes.rs | 123 +++++++++++++++++++++++ crates/orderbook/src/database/mod.rs | 24 +---- crates/orderbook/src/database/quotes.rs | 26 ++--- crates/shared/src/order_quoting.rs | 39 +++++++ 5 files changed, 188 insertions(+), 76 deletions(-) diff --git a/crates/autopilot/src/database/auction.rs b/crates/autopilot/src/database/auction.rs index 8bf5887027..ced6c0eeb9 100644 --- a/crates/autopilot/src/database/auction.rs +++ b/crates/autopilot/src/database/auction.rs @@ -4,10 +4,8 @@ use { anyhow::{Context, Result}, chrono::{DateTime, Utc}, futures::{StreamExt, TryStreamExt}, - model::{interaction::InteractionData, order::Order, quote::QuoteId}, + model::{order::Order, quote::QuoteId}, num::ToPrimitive, - number::conversions::big_decimal_to_u256, - primitive_types::H160, shared::{ db_order_conversions::full_order_into_model_order, event_storing_helpers::{ @@ -17,7 +15,7 @@ use { }, order_quoting::{QuoteData, QuoteSearchParameters, QuoteStoring}, }, - sqlx::{Acquire, PgConnection}, + sqlx::Acquire, std::{collections::HashMap, ops::DerefMut}, }; @@ -49,15 +47,10 @@ impl QuoteStoring for Postgres { .start_timer(); let mut ex = self.pool.acquire().await?; - let quote = database::quotes::get(&mut ex, id).await?; - let quote_interactions = Self::get_quote_interactions(&mut ex, id).await?; - Ok(quote - .map(QuoteData::try_from) - .transpose()? - .map(|mut quote_data| { - quote_data.interactions = quote_interactions; - quote_data - })) + + let query_result = database::quotes::get_quote_with_interactions(&mut ex, id).await?; + + Ok(query_result.map(QuoteData::try_from).transpose()?) } async fn find( @@ -72,15 +65,16 @@ impl QuoteStoring for Postgres { let mut ex = self.pool.acquire().await?; let params = create_db_search_parameters(params, expiration); - let quote = database::quotes::find(&mut ex, ¶ms) + + let query_result = database::quotes::find_quote_with_interactions(&mut ex, ¶ms) .await .context("failed finding quote by parameters")?; - if let Some(quote) = quote { - let quote_id = quote.id; - let quote_interactions = Self::get_quote_interactions(&mut ex, quote_id).await?; - let mut quote_data = QuoteData::try_from(quote)?; - quote_data.interactions = quote_interactions; - Ok(Some((quote_id, quote_data))) + + if let Some(query_result) = query_result { + Ok(Some(( + query_result.0.id, + QuoteData::try_from(query_result)?, + ))) } else { Ok(None) } @@ -138,22 +132,4 @@ impl Postgres { let id = database::auction::replace_auction(&mut ex, &data).await?; Ok(id) } - - async fn get_quote_interactions( - ex: &mut PgConnection, - quote_id: QuoteId, - ) -> Result> { - database::quotes::get_quote_interactions(ex, quote_id) - .await? - .iter() - .map(|data| { - Ok(InteractionData { - target: H160(data.target.0), - value: big_decimal_to_u256(&data.value) - .context("quote interaction value is not a valid U256")?, - call_data: data.call_data.clone(), - }) - }) - .collect::>>() - } } diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 06cfaf4637..21b22dd1b1 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -132,6 +132,129 @@ LIMIT 1 .await } +/// Internal structure used for getting Quote with Interactions in one SQL +/// query. +#[derive(Clone, sqlx::FromRow)] +struct QuoteWithInteraction { + pub id: QuoteId, + pub sell_token: Address, + pub buy_token: Address, + pub sell_amount: BigDecimal, + pub buy_amount: BigDecimal, + pub gas_amount: f64, + pub gas_price: f64, + pub sell_token_price: f64, + pub order_kind: OrderKind, + pub expiration_timestamp: DateTime, + pub quote_kind: QuoteKind, + pub solver: Address, + pub verified: bool, + pub index: i32, + pub target: Address, + pub value: BigDecimal, + pub call_data: Vec, +} + +impl QuoteWithInteraction { + fn to_quote(&self) -> Quote { + Quote { + id: self.id, + sell_token: self.sell_token, + buy_token: self.buy_token, + sell_amount: self.sell_amount.clone(), + buy_amount: self.buy_amount.clone(), + gas_amount: self.gas_amount, + gas_price: self.gas_price, + sell_token_price: self.sell_token_price, + order_kind: self.order_kind, + expiration_timestamp: self.expiration_timestamp, + quote_kind: self.quote_kind.clone(), + solver: self.solver, + verified: self.verified, + } + } + + fn to_interaction(&self) -> QuoteInteraction { + QuoteInteraction { + quote_id: self.id, + index: self.index, + target: self.target, + value: self.value.clone(), + call_data: self.call_data.clone(), + } + } + + fn to_quote_with_interactions(items: &[QuoteWithInteraction]) -> Option { + if let Some(first_item) = items.first() { + // Due to use of sql join, all rows has same quote items. + let quote = first_item.to_quote(); + let interactions = items + .iter() + .map(|item| item.to_interaction()) + .collect::>(); + Some((quote, interactions)) + } else { + None + } + } +} + +pub type QuoteWithInteractions = (Quote, Vec); + +pub async fn get_quote_with_interactions( + ex: &mut PgConnection, + id: QuoteId, +) -> Result, sqlx::Error> { + const QUERY: &str = r#" + SELECT q.*, i.index, i.target, i.value, i.call_data FROM quotes q + JOIN quote_interactions i ON quote_id = id + WHERE id = $1 + "#; + + Ok(QuoteWithInteraction::to_quote_with_interactions( + &sqlx::query_as(QUERY).bind(id).fetch_all(ex).await?, + )) +} + +pub async fn find_quote_with_interactions( + ex: &mut PgConnection, + params: &QuoteSearchParameters, +) -> Result, sqlx::Error> { + const QUERY: &str = r#" +SELECT quotes.*, i.index, i.target, i.value, i.call_data +FROM quotes +JOIN quote_interactions i +ON i.quote_id = id +WHERE + sell_token = $1 AND + buy_token = $2 AND + ( + (order_kind = 'sell' AND sell_amount = $3) OR + (order_kind = 'sell' AND sell_amount = $4) OR + (order_kind = 'buy' AND buy_amount = $5) + ) AND + order_kind = $6 AND + expiration_timestamp >= $7 AND + quote_kind = $8 +ORDER BY gas_amount * gas_price * sell_token_price ASC +LIMIT 1 + "#; + + let result: Vec = sqlx::query_as(QUERY) + .bind(params.sell_token) + .bind(params.buy_token) + .bind(¶ms.sell_amount_0) + .bind(¶ms.sell_amount_1) + .bind(¶ms.buy_amount) + .bind(params.kind) + .bind(params.expiration) + .bind(¶ms.quote_kind) + .fetch_all(ex) + .await?; + + Ok(QuoteWithInteraction::to_quote_with_interactions(&result)) +} + pub async fn remove_expired_quotes( ex: &mut PgConnection, max_expiry: DateTime, diff --git a/crates/orderbook/src/database/mod.rs b/crates/orderbook/src/database/mod.rs index 7fe2aa0ec1..25d10f8c17 100644 --- a/crates/orderbook/src/database/mod.rs +++ b/crates/orderbook/src/database/mod.rs @@ -10,11 +10,9 @@ pub mod trades; use { crate::database::orders::InsertionError, - anyhow::{Context, Result}, + anyhow::Result, database::byte_array::ByteArray, - model::{interaction::InteractionData, order::Order, quote::QuoteId}, - number::conversions::big_decimal_to_u256, - primitive_types::H160, + model::order::Order, sqlx::{PgConnection, PgPool}, }; @@ -55,24 +53,6 @@ impl Postgres { } Ok(()) } - - async fn get_quote_interactions( - ex: &mut PgConnection, - quote_id: QuoteId, - ) -> Result> { - database::quotes::get_quote_interactions(ex, quote_id) - .await? - .iter() - .map(|data| { - Ok(InteractionData { - target: H160(data.target.0), - value: big_decimal_to_u256(&data.value) - .context("quote interaction value is not a valid U256")?, - call_data: data.call_data.clone(), - }) - }) - .collect::>>() - } } #[derive(prometheus_metric_storage::MetricStorage)] diff --git a/crates/orderbook/src/database/quotes.rs b/crates/orderbook/src/database/quotes.rs index acc37fe52e..a93961fe66 100644 --- a/crates/orderbook/src/database/quotes.rs +++ b/crates/orderbook/src/database/quotes.rs @@ -42,16 +42,10 @@ impl QuoteStoring for Postgres { .start_timer(); let mut ex = self.pool.acquire().await?; - let quote = database::quotes::get(&mut ex, id).await?; - let quote_interactions = Self::get_quote_interactions(&mut ex, id).await?; - Ok(quote - .map(QuoteData::try_from) - .transpose()? - .map(|mut quote_data| { - quote_data.interactions = quote_interactions; - quote_data - })) + let query_result = database::quotes::get_quote_with_interactions(&mut ex, id).await?; + + Ok(query_result.map(QuoteData::try_from).transpose()?) } async fn find( @@ -66,16 +60,16 @@ impl QuoteStoring for Postgres { let mut ex = self.pool.acquire().await?; let params = create_db_search_parameters(params, expiration); - let quote = database::quotes::find(&mut ex, ¶ms) + + let query_result = database::quotes::find_quote_with_interactions(&mut ex, ¶ms) .await .context("failed finding quote by parameters")?; - if let Some(quote) = quote { - let quote_id = quote.id; - let quote_interactions = Self::get_quote_interactions(&mut ex, quote_id).await?; - let mut quote_data = QuoteData::try_from(quote)?; - quote_data.interactions = quote_interactions; - Ok(Some((quote_id, quote_data))) + if let Some(query_result) = query_result { + Ok(Some(( + query_result.0.id, + QuoteData::try_from(query_result)?, + ))) } else { Ok(None) } diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index b494edc72b..f98a369bb1 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -202,6 +202,45 @@ impl TryFrom for QuoteData { } } +impl TryFrom for QuoteData { + type Error = anyhow::Error; + + fn try_from(input: database::quotes::QuoteWithInteractions) -> Result { + Ok(QuoteData { + sell_token: H160(input.0.sell_token.0), + buy_token: H160(input.0.buy_token.0), + quoted_sell_amount: big_decimal_to_u256(&input.0.sell_amount) + .context("quoted sell amount is not a valid U256")?, + quoted_buy_amount: big_decimal_to_u256(&input.0.buy_amount) + .context("quoted buy amount is not a valid U256")?, + fee_parameters: FeeParameters { + gas_amount: input.0.gas_amount, + gas_price: input.0.gas_price, + sell_token_price: input.0.sell_token_price, + }, + kind: order_kind_from(input.0.order_kind), + expiration: input.0.expiration_timestamp, + quote_kind: input.0.quote_kind, + solver: H160(input.0.solver.0), + // Even if the quote was verified at the time of creation + // it might no longer be accurate. + verified: false, + interactions: input + .1 + .iter() + .map(|data| { + Ok(InteractionData { + target: H160(data.target.0), + value: big_decimal_to_u256(&data.value) + .context("quote interaction value is not a valid U256")?, + call_data: data.call_data.clone(), + }) + }) + .collect::>>()?, + }) + } +} + #[mockall::automock] #[async_trait::async_trait] pub trait OrderQuoting: Send + Sync { From 9a7f510b925399549dbe2248c54b9ee053495447 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Mon, 2 Dec 2024 08:08:20 +0100 Subject: [PATCH 43/70] Added test --- crates/database/src/quotes.rs | 62 +++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 21b22dd1b1..4d034d97a1 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -711,4 +711,66 @@ mod tests { let interactions = get_quote_interactions(&mut db, id).await.unwrap(); assert!(interactions.is_empty()); } + + #[tokio::test] + #[ignore] + async fn postgres_get_quote_with_interactions_by_id() { + let mut db = PgConnection::connect("postgresql://").await.unwrap(); + let mut db = db.begin().await.unwrap(); + crate::clear_DANGER_(&mut db).await.unwrap(); + + let now = low_precision_now(); + let mut quote = Quote { + id: Default::default(), + sell_token: ByteArray([1; 20]), + buy_token: ByteArray([2; 20]), + sell_amount: 3.into(), + buy_amount: 4.into(), + gas_amount: 5., + gas_price: 6., + sell_token_price: 7., + order_kind: OrderKind::Sell, + expiration_timestamp: now, + quote_kind: QuoteKind::Standard, + solver: ByteArray([1; 20]), + verified: false, + }; + // store quote in database + let id = save(&mut db, "e).await.unwrap(); + quote.id = id; + + let quote_interactions = [ + QuoteInteraction { + quote_id: id, + index: 0, + target: ByteArray([1; 20]), + value: 2.into(), + call_data: vec![3; 20], + }, + QuoteInteraction { + quote_id: id, + index: 1, + target: ByteArray([1; 20]), + value: 2.into(), + call_data: vec![3; 20], + }, + ]; + // store interactions for the quote in database + insert_quote_interactions(&mut db, "e_interactions) + .await + .unwrap(); + + let (returned_quote, interactions) = get_quote_with_interactions(&mut db, id) + .await + .unwrap() + .unwrap(); + assert_eq!(returned_quote, quote); + assert_eq!(interactions.len(), 2); + for i in [0, 1] { + assert_eq!( + interactions.iter().find(|val| val.index == i).unwrap(), + "e_interactions[i as usize] + ); + } + } } From d8ff00b11d218528e3696788c44343359e705b80 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Mon, 2 Dec 2024 08:46:48 +0100 Subject: [PATCH 44/70] Small refactorings --- crates/autopilot/src/database/auction.rs | 11 +++-------- crates/database/src/quotes.rs | 20 +++++++++----------- crates/orderbook/src/database/quotes.rs | 11 +++-------- 3 files changed, 15 insertions(+), 27 deletions(-) diff --git a/crates/autopilot/src/database/auction.rs b/crates/autopilot/src/database/auction.rs index ced6c0eeb9..997b19286d 100644 --- a/crates/autopilot/src/database/auction.rs +++ b/crates/autopilot/src/database/auction.rs @@ -70,14 +70,9 @@ impl QuoteStoring for Postgres { .await .context("failed finding quote by parameters")?; - if let Some(query_result) = query_result { - Ok(Some(( - query_result.0.id, - QuoteData::try_from(query_result)?, - ))) - } else { - Ok(None) - } + query_result + .map(|query_result| Ok((query_result.0.id, query_result.try_into()?))) + .transpose() } } diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 4d034d97a1..71b23da55b 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -185,17 +185,15 @@ impl QuoteWithInteraction { } fn to_quote_with_interactions(items: &[QuoteWithInteraction]) -> Option { - if let Some(first_item) = items.first() { - // Due to use of sql join, all rows has same quote items. - let quote = first_item.to_quote(); - let interactions = items - .iter() - .map(|item| item.to_interaction()) - .collect::>(); - Some((quote, interactions)) - } else { - None - } + items.first().map(|first_item| { + ( + first_item.to_quote(), + items + .iter() + .map(|item| item.to_interaction()) + .collect::>(), + ) + }) } } diff --git a/crates/orderbook/src/database/quotes.rs b/crates/orderbook/src/database/quotes.rs index a93961fe66..ba28cdea59 100644 --- a/crates/orderbook/src/database/quotes.rs +++ b/crates/orderbook/src/database/quotes.rs @@ -65,13 +65,8 @@ impl QuoteStoring for Postgres { .await .context("failed finding quote by parameters")?; - if let Some(query_result) = query_result { - Ok(Some(( - query_result.0.id, - QuoteData::try_from(query_result)?, - ))) - } else { - Ok(None) - } + query_result + .map(|query_result| Ok((query_result.0.id, query_result.try_into()?))) + .transpose() } } From 726376a308f9f8ac046474ccc33711988c4c4e32 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Mon, 2 Dec 2024 12:25:53 +0100 Subject: [PATCH 45/70] Function rename --- crates/database/src/quotes.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 71b23da55b..745e560ffc 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -184,7 +184,9 @@ impl QuoteWithInteraction { } } - fn to_quote_with_interactions(items: &[QuoteWithInteraction]) -> Option { + fn build_quote_with_interactions( + items: &[QuoteWithInteraction], + ) -> Option { items.first().map(|first_item| { ( first_item.to_quote(), @@ -209,7 +211,7 @@ pub async fn get_quote_with_interactions( WHERE id = $1 "#; - Ok(QuoteWithInteraction::to_quote_with_interactions( + Ok(QuoteWithInteraction::build_quote_with_interactions( &sqlx::query_as(QUERY).bind(id).fetch_all(ex).await?, )) } @@ -250,7 +252,7 @@ LIMIT 1 .fetch_all(ex) .await?; - Ok(QuoteWithInteraction::to_quote_with_interactions(&result)) + Ok(QuoteWithInteraction::build_quote_with_interactions(&result)) } pub async fn remove_expired_quotes( From 1e23f0d7e404118a81dfa6f21b84e82befbcddeb Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Mon, 2 Dec 2024 23:41:03 +0100 Subject: [PATCH 46/70] Using array_agg function for getting quote with interactions from db --- crates/autopilot/src/database/auction.rs | 2 +- crates/database/src/quotes.rs | 234 +++++++++++++++-------- crates/orderbook/src/database/quotes.rs | 2 +- crates/shared/src/order_quoting.rs | 24 +-- 4 files changed, 167 insertions(+), 95 deletions(-) diff --git a/crates/autopilot/src/database/auction.rs b/crates/autopilot/src/database/auction.rs index 997b19286d..fff4e65afc 100644 --- a/crates/autopilot/src/database/auction.rs +++ b/crates/autopilot/src/database/auction.rs @@ -71,7 +71,7 @@ impl QuoteStoring for Postgres { .context("failed finding quote by parameters")?; query_result - .map(|query_result| Ok((query_result.0.id, query_result.try_into()?))) + .map(|query_result| Ok((query_result.id, query_result.try_into()?))) .transpose() } } diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 745e560ffc..b708ff2d3f 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -4,6 +4,7 @@ use { sqlx::{ types::chrono::{DateTime, Utc}, PgConnection, + Row, }, }; @@ -37,6 +38,27 @@ pub struct Quote { pub verified: bool, } +impl Quote { + pub fn to_quote_with_interactions(&self) -> QuoteWithInteractions { + QuoteWithInteractions { + id: self.id, + sell_token: self.sell_token, + buy_token: self.buy_token, + sell_amount: self.sell_amount.clone(), + buy_amount: self.buy_amount.clone(), + gas_amount: self.gas_amount, + gas_price: self.gas_price, + sell_token_price: self.sell_token_price, + order_kind: self.order_kind, + expiration_timestamp: self.expiration_timestamp, + quote_kind: self.quote_kind.clone(), + solver: self.solver, + verified: self.verified, + interactions: vec![], + } + } +} + /// Stores the quote and returns the id. The id of the quote parameter is not /// used. pub async fn save(ex: &mut PgConnection, quote: &Quote) -> Result { @@ -132,10 +154,8 @@ LIMIT 1 .await } -/// Internal structure used for getting Quote with Interactions in one SQL -/// query. -#[derive(Clone, sqlx::FromRow)] -struct QuoteWithInteraction { +#[derive(Clone, Debug, PartialEq)] +pub struct QuoteWithInteractions { pub id: QuoteId, pub sell_token: Address, pub buy_token: Address, @@ -149,71 +169,65 @@ struct QuoteWithInteraction { pub quote_kind: QuoteKind, pub solver: Address, pub verified: bool, - pub index: i32, - pub target: Address, - pub value: BigDecimal, - pub call_data: Vec, + pub interactions: Vec, } -impl QuoteWithInteraction { - fn to_quote(&self) -> Quote { - Quote { - id: self.id, - sell_token: self.sell_token, - buy_token: self.buy_token, - sell_amount: self.sell_amount.clone(), - buy_amount: self.buy_amount.clone(), - gas_amount: self.gas_amount, - gas_price: self.gas_price, - sell_token_price: self.sell_token_price, - order_kind: self.order_kind, - expiration_timestamp: self.expiration_timestamp, - quote_kind: self.quote_kind.clone(), - solver: self.solver, - verified: self.verified, - } - } - - fn to_interaction(&self) -> QuoteInteraction { - QuoteInteraction { - quote_id: self.id, - index: self.index, - target: self.target, - value: self.value.clone(), - call_data: self.call_data.clone(), - } - } +impl sqlx::FromRow<'_, sqlx::postgres::PgRow> for QuoteWithInteractions { + fn from_row(row: &sqlx::postgres::PgRow) -> sqlx::Result { + let id = row.get("id"); + + let interactions = match row + .get::, BigDecimal, Vec)>>, &str>( + "interactions", + ) { + None => vec![], + Some(col) => col + .into_iter() + .map(|(index, target, value, call_data)| QuoteInteraction { + quote_id: id, + index, + target, + value, + call_data, + }) + .collect(), + }; - fn build_quote_with_interactions( - items: &[QuoteWithInteraction], - ) -> Option { - items.first().map(|first_item| { - ( - first_item.to_quote(), - items - .iter() - .map(|item| item.to_interaction()) - .collect::>(), - ) + Ok(QuoteWithInteractions { + id, + sell_token: row.get("sell_token"), + buy_token: row.get("buy_token"), + sell_amount: row.get("sell_amount"), + buy_amount: row.get("buy_amount"), + gas_amount: row.get("gas_amount"), + gas_price: row.get("gas_price"), + sell_token_price: row.get("sell_token_price"), + order_kind: row.get("order_kind"), + expiration_timestamp: row.get("expiration_timestamp"), + quote_kind: row.get("quote_kind"), + solver: row.get("solver"), + verified: row.get("verified"), + interactions, }) } } -pub type QuoteWithInteractions = (Quote, Vec); - pub async fn get_quote_with_interactions( ex: &mut PgConnection, id: QuoteId, ) -> Result, sqlx::Error> { - const QUERY: &str = r#" - SELECT q.*, i.index, i.target, i.value, i.call_data FROM quotes q - JOIN quote_interactions i ON quote_id = id - WHERE id = $1 - "#; - - Ok(QuoteWithInteraction::build_quote_with_interactions( - &sqlx::query_as(QUERY).bind(id).fetch_all(ex).await?, - )) + sqlx::query_as( + r#" + SELECT q.*, array_agg((i.index, i.target, i.value, i.call_data)) FILTER (WHERE i.quote_id IS NOT NULL) as "interactions" + FROM quotes q + LEFT JOIN quote_interactions i ON i.quote_id = q.id + WHERE q.id = $1 + GROUP BY q.id + "#, + ) + .bind(id) + .fetch_optional(ex) + .await } pub async fn find_quote_with_interactions( @@ -221,10 +235,9 @@ pub async fn find_quote_with_interactions( params: &QuoteSearchParameters, ) -> Result, sqlx::Error> { const QUERY: &str = r#" -SELECT quotes.*, i.index, i.target, i.value, i.call_data +SELECT quotes.*, array_agg((i.index, i.target, i.value, i.call_data)) FILTER (WHERE i.quote_id IS NOT NULL) as "interactions" FROM quotes -JOIN quote_interactions i -ON i.quote_id = id +LEFT JOIN quote_interactions i ON i.quote_id = id WHERE sell_token = $1 AND buy_token = $2 AND @@ -236,11 +249,12 @@ WHERE order_kind = $6 AND expiration_timestamp >= $7 AND quote_kind = $8 +GROUP BY id ORDER BY gas_amount * gas_price * sell_token_price ASC LIMIT 1 "#; - let result: Vec = sqlx::query_as(QUERY) + sqlx::query_as(QUERY) .bind(params.sell_token) .bind(params.buy_token) .bind(¶ms.sell_amount_0) @@ -249,10 +263,8 @@ LIMIT 1 .bind(params.kind) .bind(params.expiration) .bind(¶ms.quote_kind) - .fetch_all(ex) - .await?; - - Ok(QuoteWithInteraction::build_quote_with_interactions(&result)) + .fetch_optional(ex) + .await } pub async fn remove_expired_quotes( @@ -527,11 +539,14 @@ mod tests { quote_kind: QuoteKind::Standard, }; assert_eq!( - find(&mut db, &search_b).await.unwrap().unwrap(), - quotes_b[0], + find_quote_with_interactions(&mut db, &search_b) + .await + .unwrap() + .unwrap(), + quotes_b[0].to_quote_with_interactions(), ); assert_eq!( - find( + find_quote_with_interactions( &mut db, &QuoteSearchParameters { expiration: now + Duration::seconds(30), @@ -545,7 +560,7 @@ mod tests { // Token B has no reading for wrong filter assert_eq!( - find( + find_quote_with_interactions( &mut db, &QuoteSearchParameters { buy_amount: 99.into(), @@ -561,8 +576,18 @@ mod tests { remove_expired_quotes(&mut db, now + Duration::seconds(120)) .await .unwrap(); - assert_eq!(find(&mut db, &search_a).await.unwrap(), None); - assert_eq!(find(&mut db, &search_b).await.unwrap(), None); + assert_eq!( + find_quote_with_interactions(&mut db, &search_a) + .await + .unwrap(), + None + ); + assert_eq!( + find_quote_with_interactions(&mut db, &search_b) + .await + .unwrap(), + None + ); } #[tokio::test] @@ -606,9 +631,20 @@ mod tests { quote_kind: quote.quote_kind.clone(), }; - assert_eq!(find(&mut db, &search_a).await.unwrap().unwrap(), quote,); + assert_eq!( + find_quote_with_interactions(&mut db, &search_a) + .await + .unwrap() + .unwrap(), + quote.to_quote_with_interactions(), + ); search_a.quote_kind = QuoteKind::Standard; - assert_eq!(find(&mut db, &search_a).await.unwrap(), None,); + assert_eq!( + find_quote_with_interactions(&mut db, &search_a) + .await + .unwrap(), + None, + ); } #[tokio::test] @@ -735,9 +771,15 @@ mod tests { solver: ByteArray([1; 20]), verified: false, }; + let mut quote2 = quote.clone(); + let mut quote3 = quote.clone(); // store quote in database let id = save(&mut db, "e).await.unwrap(); + let id2 = save(&mut db, "e2).await.unwrap(); + let id3 = save(&mut db, "e3).await.unwrap(); quote.id = id; + quote2.id = id2; + quote3.id = id3; let quote_interactions = [ QuoteInteraction { @@ -750,9 +792,16 @@ mod tests { QuoteInteraction { quote_id: id, index: 1, - target: ByteArray([1; 20]), - value: 2.into(), - call_data: vec![3; 20], + target: ByteArray([4; 20]), + value: 5.into(), + call_data: vec![6; 20], + }, + QuoteInteraction { + quote_id: id2, + index: 0, + target: ByteArray([7; 20]), + value: 8.into(), + call_data: vec![9; 20], }, ]; // store interactions for the quote in database @@ -760,17 +809,40 @@ mod tests { .await .unwrap(); - let (returned_quote, interactions) = get_quote_with_interactions(&mut db, id) + let returned_quote: QuoteWithInteractions = get_quote_with_interactions(&mut db, id) .await .unwrap() .unwrap(); - assert_eq!(returned_quote, quote); - assert_eq!(interactions.len(), 2); + + assert_eq!(returned_quote.id, quote.id); + assert_eq!(returned_quote.interactions.len(), 2); for i in [0, 1] { assert_eq!( - interactions.iter().find(|val| val.index == i).unwrap(), + returned_quote + .interactions + .iter() + .find(|val| val.index == i) + .unwrap(), "e_interactions[i as usize] ); } + + let returned_quote2: QuoteWithInteractions = get_quote_with_interactions(&mut db, id2) + .await + .unwrap() + .unwrap(); + assert_eq!(returned_quote2.id, quote2.id); + assert_eq!(returned_quote2.interactions.len(), 1); + assert_eq!( + returned_quote2.interactions.first().unwrap(), + "e_interactions[2] + ); + + let returned_quote3: QuoteWithInteractions = get_quote_with_interactions(&mut db, id3) + .await + .unwrap() + .unwrap(); + assert_eq!(returned_quote3.id, quote3.id); + assert!(returned_quote3.interactions.is_empty()); } } diff --git a/crates/orderbook/src/database/quotes.rs b/crates/orderbook/src/database/quotes.rs index ba28cdea59..7ef8810269 100644 --- a/crates/orderbook/src/database/quotes.rs +++ b/crates/orderbook/src/database/quotes.rs @@ -66,7 +66,7 @@ impl QuoteStoring for Postgres { .context("failed finding quote by parameters")?; query_result - .map(|query_result| Ok((query_result.0.id, query_result.try_into()?))) + .map(|query_result| Ok((query_result.id, query_result.try_into()?))) .transpose() } } diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index f98a369bb1..79c98488fa 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -207,26 +207,26 @@ impl TryFrom for QuoteData { fn try_from(input: database::quotes::QuoteWithInteractions) -> Result { Ok(QuoteData { - sell_token: H160(input.0.sell_token.0), - buy_token: H160(input.0.buy_token.0), - quoted_sell_amount: big_decimal_to_u256(&input.0.sell_amount) + sell_token: H160(input.sell_token.0), + buy_token: H160(input.buy_token.0), + quoted_sell_amount: big_decimal_to_u256(&input.sell_amount) .context("quoted sell amount is not a valid U256")?, - quoted_buy_amount: big_decimal_to_u256(&input.0.buy_amount) + quoted_buy_amount: big_decimal_to_u256(&input.buy_amount) .context("quoted buy amount is not a valid U256")?, fee_parameters: FeeParameters { - gas_amount: input.0.gas_amount, - gas_price: input.0.gas_price, - sell_token_price: input.0.sell_token_price, + gas_amount: input.gas_amount, + gas_price: input.gas_price, + sell_token_price: input.sell_token_price, }, - kind: order_kind_from(input.0.order_kind), - expiration: input.0.expiration_timestamp, - quote_kind: input.0.quote_kind, - solver: H160(input.0.solver.0), + kind: order_kind_from(input.order_kind), + expiration: input.expiration_timestamp, + quote_kind: input.quote_kind, + solver: H160(input.solver.0), // Even if the quote was verified at the time of creation // it might no longer be accurate. verified: false, interactions: input - .1 + .interactions .iter() .map(|data| { Ok(InteractionData { From f5cbc3ee8902fb044e68f5d6493f1f0af54c30b2 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Mon, 2 Dec 2024 23:48:54 +0100 Subject: [PATCH 47/70] Removed new error variant, fixed comment in sql script --- crates/orderbook/src/api/post_order.rs | 4 ---- crates/orderbook/src/database/orders.rs | 12 ++++-------- crates/orderbook/src/orderbook.rs | 3 --- ...075__add_call_data_to_quotes_and_order_quotes.sql | 2 +- 4 files changed, 5 insertions(+), 16 deletions(-) diff --git a/crates/orderbook/src/api/post_order.rs b/crates/orderbook/src/api/post_order.rs index cf0c768e68..2208bff6fd 100644 --- a/crates/orderbook/src/api/post_order.rs +++ b/crates/orderbook/src/api/post_order.rs @@ -262,10 +262,6 @@ impl IntoWarpReply for AddOrderError { super::error("InvalidReplacement", err.to_string()), StatusCode::UNAUTHORIZED, ), - AddOrderError::QuoteInteractionWrongIndex => { - tracing::error!("QuoteInteractionWrongIndex"); - crate::api::internal_error_reply() - } } } } diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index 2d9e522aca..349530c72c 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -113,9 +113,6 @@ pub enum InsertionError { DbError(sqlx::Error), /// Full app data to be inserted doesn't match existing. AppDataMismatch(Vec), - /// Type conversion (usize to i32) of sequence index of order quote - /// interaction failed. - IndexConversionFailed, } impl From for InsertionError { @@ -245,16 +242,15 @@ async fn insert_quote( .iter() .enumerate() .map(|(idx, interaction)| { - Ok(OrderQuoteInteraction { + OrderQuoteInteraction { order_uid: dbquote.order_uid, - index: idx.try_into()?, + index: idx.try_into().unwrap(), // safe to unwrap target: ByteArray(interaction.target.0), value: u256_to_big_decimal(&interaction.value), call_data: interaction.call_data.clone(), - }) + } }) - .collect::>>() - .map_err(|_| InsertionError::IndexConversionFailed)?; + .collect::>(); let mut transaction = ex.begin().await?; database::orders::insert_quote(&mut transaction, &dbquote) diff --git a/crates/orderbook/src/orderbook.rs b/crates/orderbook/src/orderbook.rs index 74163d17cb..fc637a1c2f 100644 --- a/crates/orderbook/src/orderbook.rs +++ b/crates/orderbook/src/orderbook.rs @@ -141,8 +141,6 @@ pub enum AddOrderError { provided: String, existing: String, }, - #[error("unable to convert type usize to i32 for one of the quote interactions")] - QuoteInteractionWrongIndex, } impl AddOrderError { @@ -163,7 +161,6 @@ impl AddOrderError { s.into_owned() }, }, - InsertionError::IndexConversionFailed => AddOrderError::QuoteInteractionWrongIndex, } } } diff --git a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql index bbb19aed3b..6c58bafc0b 100644 --- a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql +++ b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql @@ -4,7 +4,7 @@ ALTER TABLE quotes ADD COLUMN verified boolean; --- Step 2: Add two new columns to the order_quotes table +-- Step 2: Add a new column to the order_quotes table ALTER TABLE order_quotes ADD COLUMN verified boolean; From ddaf4b2aa868624415033833880e746bda7382f0 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Tue, 3 Dec 2024 10:01:21 +0100 Subject: [PATCH 48/70] Updated table descriptions --- crates/database/src/quotes.rs | 126 +++++++++++++--------------------- database/README.md | 8 +-- 2 files changed, 52 insertions(+), 82 deletions(-) diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index b708ff2d3f..f15f41e821 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -98,62 +98,6 @@ RETURNING id Ok(id) } -pub async fn get(ex: &mut PgConnection, id: QuoteId) -> Result, sqlx::Error> { - const QUERY: &str = r#" -SELECT * -FROM quotes -WHERE id = $1 - "#; - sqlx::query_as(QUERY).bind(id).fetch_optional(ex).await -} - -/// Fields for searching stored quotes. -#[derive(Clone)] -pub struct QuoteSearchParameters { - pub sell_token: Address, - pub buy_token: Address, - pub sell_amount_0: BigDecimal, - pub sell_amount_1: BigDecimal, - pub buy_amount: BigDecimal, - pub kind: OrderKind, - pub expiration: DateTime, - pub quote_kind: QuoteKind, -} - -pub async fn find( - ex: &mut PgConnection, - params: &QuoteSearchParameters, -) -> Result, sqlx::Error> { - const QUERY: &str = r#" -SELECT * -FROM quotes -WHERE - sell_token = $1 AND - buy_token = $2 AND - ( - (order_kind = 'sell' AND sell_amount = $3) OR - (order_kind = 'sell' AND sell_amount = $4) OR - (order_kind = 'buy' AND buy_amount = $5) - ) AND - order_kind = $6 AND - expiration_timestamp >= $7 AND - quote_kind = $8 -ORDER BY gas_amount * gas_price * sell_token_price ASC -LIMIT 1 - "#; - sqlx::query_as(QUERY) - .bind(params.sell_token) - .bind(params.buy_token) - .bind(¶ms.sell_amount_0) - .bind(¶ms.sell_amount_1) - .bind(¶ms.buy_amount) - .bind(params.kind) - .bind(params.expiration) - .bind(¶ms.quote_kind) - .fetch_optional(ex) - .await -} - #[derive(Clone, Debug, PartialEq)] pub struct QuoteWithInteractions { pub id: QuoteId, @@ -230,6 +174,19 @@ pub async fn get_quote_with_interactions( .await } +/// Fields for searching stored quotes. +#[derive(Clone)] +pub struct QuoteSearchParameters { + pub sell_token: Address, + pub buy_token: Address, + pub sell_amount_0: BigDecimal, + pub sell_amount_1: BigDecimal, + pub buy_amount: BigDecimal, + pub kind: OrderKind, + pub expiration: DateTime, + pub quote_kind: QuoteKind, +} + pub async fn find_quote_with_interactions( ex: &mut PgConnection, params: &QuoteSearchParameters, @@ -328,18 +285,6 @@ pub async fn insert_quote_interactions( Ok(()) } -pub async fn get_quote_interactions( - ex: &mut PgConnection, - quote_id: QuoteId, -) -> Result, sqlx::Error> { - const QUERY: &str = r#" -SELECT * -FROM quote_interactions -WHERE quote_id = $1 - "#; - sqlx::query_as(QUERY).bind(quote_id).fetch_all(ex).await -} - #[cfg(test)] mod tests { use { @@ -349,6 +294,30 @@ mod tests { sqlx::{types::chrono::TimeZone, Connection}, }; + pub async fn get_quote( + ex: &mut PgConnection, + id: QuoteId, + ) -> Result, sqlx::Error> { + const QUERY: &str = r#" + SELECT * + FROM quotes + WHERE id = $1 + "#; + sqlx::query_as(QUERY).bind(id).fetch_optional(ex).await + } + + pub async fn get_quote_interactions( + ex: &mut PgConnection, + quote_id: QuoteId, + ) -> Result, sqlx::Error> { + const QUERY: &str = r#" + SELECT * + FROM quote_interactions + WHERE quote_id = $1 + "#; + sqlx::query_as(QUERY).bind(quote_id).fetch_all(ex).await + } + /// The postgres database in our CI has different datetime precision than /// the `DateTime` uses. This leads to issues comparing round-tripped data. /// Work around the issue by created `DateTime`s with lower precision. @@ -381,12 +350,12 @@ mod tests { }; let id = save(&mut db, "e).await.unwrap(); quote.id = id; - assert_eq!(get(&mut db, id).await.unwrap().unwrap(), quote); + assert_eq!(get_quote(&mut db, id).await.unwrap().unwrap(), quote); remove_expired_quotes(&mut db, now + Duration::seconds(30)) .await .unwrap(); - assert_eq!(get(&mut db, id).await.unwrap(), None); + assert_eq!(get_quote(&mut db, id).await.unwrap(), None); } #[tokio::test] @@ -479,11 +448,14 @@ mod tests { quote_kind: QuoteKind::Standard, }; assert_eq!( - find(&mut db, &search_a).await.unwrap().unwrap(), - quotes_a[0], + find_quote_with_interactions(&mut db, &search_a) + .await + .unwrap() + .unwrap(), + quotes_a[0].to_quote_with_interactions(), ); assert_eq!( - find( + find_quote_with_interactions( &mut db, &QuoteSearchParameters { expiration: now + Duration::seconds(30), @@ -493,12 +465,12 @@ mod tests { .await .unwrap() .unwrap(), - quotes_a[1] + quotes_a[1].to_quote_with_interactions() ); // Token A has readings for sell + fee amount equal to quoted amount. assert_eq!( - find( + find_quote_with_interactions( &mut db, &QuoteSearchParameters { sell_amount_0: quote_a.sell_amount.clone() - BigDecimal::from(1), @@ -509,12 +481,12 @@ mod tests { .await .unwrap() .unwrap(), - quotes_a[0], + quotes_a[0].to_quote_with_interactions(), ); // Token A has no reading for wrong filter assert_eq!( - find( + find_quote_with_interactions( &mut db, &QuoteSearchParameters { sell_amount_0: quote_a.sell_amount.clone() - BigDecimal::from(1), diff --git a/database/README.md b/database/README.md index e86fa5b34b..fa570e95b2 100644 --- a/database/README.md +++ b/database/README.md @@ -277,8 +277,7 @@ Indexes: ### order\_quote\_interactions -Solver responding to quote request provides a list of interactions which need to be executed to fulfill the quote. When order is created -these interactions are copied from quote. These interactions are stored persistently and can be used to audit auction winning order. +This table contains all interactions provided by the Solver in response to the /quote API request. Interactions are saved persistently when creating an order from a quote. This data can be used to audit auction winning order. Column | Type | Nullable | Details --------------------|---------|----------|-------- @@ -365,9 +364,8 @@ Indexes: ### quote\_interactions -Solver responding to quote request provides a list of interactions which need to be executed to fulfill the quote. This table stores -these interactions. Later, when order is created basing on this quote, interactions are copied to `order_quote_interactions`. -Data in this table is removed together with the referenced quote, usually at quote expiration. +This table contains all interactions provided by the Solver in response to the /quote API request. When an order is created based on a quote, interactions of that particular quote is stored in `order_quote_interactions` table. +Data in this table is removed at referenced quote expiration. Column | Type | Nullable | Details --------------------|---------|----------|-------- From b08e7f9fb5fca80ef9caf5fccf6323dd2e909259 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Wed, 4 Dec 2024 23:43:54 +0100 Subject: [PATCH 49/70] New implementation --- crates/autopilot/src/database/auction.rs | 32 +- .../src/database/onchain_order_events/mod.rs | 19 +- crates/database/Cargo.toml | 1 + crates/database/src/lib.rs | 5 +- crates/database/src/orders.rs | 169 +++---- crates/database/src/quotes.rs | 426 ++++-------------- crates/orderbook/src/api/post_order.rs | 7 + crates/orderbook/src/database/orders.rs | 83 ++-- crates/orderbook/src/database/quotes.rs | 32 +- crates/orderbook/src/orderbook.rs | 9 +- crates/shared/src/event_storing_helpers.rs | 47 +- crates/shared/src/order_quoting.rs | 52 +-- ...d_call_data_to_quotes_and_order_quotes.sql | 38 -- ...dd_metadata_to_quotes_and_order_quotes.sql | 11 + 14 files changed, 298 insertions(+), 633 deletions(-) delete mode 100644 database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql create mode 100644 database/sql/V075__add_metadata_to_quotes_and_order_quotes.sql diff --git a/crates/autopilot/src/database/auction.rs b/crates/autopilot/src/database/auction.rs index fff4e65afc..ebc92fd7a5 100644 --- a/crates/autopilot/src/database/auction.rs +++ b/crates/autopilot/src/database/auction.rs @@ -8,14 +8,9 @@ use { num::ToPrimitive, shared::{ db_order_conversions::full_order_into_model_order, - event_storing_helpers::{ - create_db_search_parameters, - create_quote_interactions_insert_data, - create_quote_row, - }, + event_storing_helpers::{create_db_search_parameters, create_quote_row}, order_quoting::{QuoteData, QuoteSearchParameters, QuoteStoring}, }, - sqlx::Acquire, std::{collections::HashMap, ops::DerefMut}, }; @@ -28,15 +23,8 @@ impl QuoteStoring for Postgres { .start_timer(); let mut ex = self.pool.acquire().await?; - let row = create_quote_row(&data); - - let mut transaction = ex.begin().await?; - let id = database::quotes::save(&mut transaction, &row).await?; - if !data.interactions.is_empty() { - let interactions = create_quote_interactions_insert_data(id, &data)?; - database::quotes::insert_quote_interactions(&mut transaction, &interactions).await?; - } - transaction.commit().await.context("commit")?; + let row = create_quote_row(&data)?; + let id = database::quotes::save(&mut ex, &row).await?; Ok(id) } @@ -47,10 +35,8 @@ impl QuoteStoring for Postgres { .start_timer(); let mut ex = self.pool.acquire().await?; - - let query_result = database::quotes::get_quote_with_interactions(&mut ex, id).await?; - - Ok(query_result.map(QuoteData::try_from).transpose()?) + let quote = database::quotes::get(&mut ex, id).await?; + quote.map(TryFrom::try_from).transpose() } async fn find( @@ -65,13 +51,11 @@ impl QuoteStoring for Postgres { let mut ex = self.pool.acquire().await?; let params = create_db_search_parameters(params, expiration); - - let query_result = database::quotes::find_quote_with_interactions(&mut ex, ¶ms) + let quote = database::quotes::find(&mut ex, ¶ms) .await .context("failed finding quote by parameters")?; - - query_result - .map(|query_result| Ok((query_result.id, query_result.try_into()?))) + quote + .map(|quote| Ok((quote.id, quote.try_into()?))) .transpose() } } diff --git a/crates/autopilot/src/database/onchain_order_events/mod.rs b/crates/autopilot/src/database/onchain_order_events/mod.rs index ec7cb4ee59..79ac4dd102 100644 --- a/crates/autopilot/src/database/onchain_order_events/mod.rs +++ b/crates/autopilot/src/database/onchain_order_events/mod.rs @@ -49,7 +49,7 @@ use { signing_scheme_into, }, event_handling::EventStoring, - order_quoting::{OrderQuoting, Quote, QuoteSearchParameters}, + order_quoting::{OrderQuoting, Quote, QuoteMetadata, QuoteSearchParameters}, order_validation::{ convert_signing_scheme_into_quote_signing_scheme, get_quote_and_check_fee, @@ -487,7 +487,13 @@ async fn parse_general_onchain_order_placement_data<'a>( sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - verified: quote.data.verified, + verified: Some(quote.data.verified), + metadata: Some( + QuoteMetadata { + interactions: quote.data.interactions.clone(), + } + .try_into()?, + ), }), Err(err) => { let err_label = err.to_metrics_label(); @@ -1188,7 +1194,14 @@ mod test { sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - verified: quote.data.verified, + verified: Some(quote.data.verified), + metadata: Some( + QuoteMetadata { + interactions: quote.data.interactions, + } + .try_into() + .unwrap(), + ), }; assert_eq!(result.1, vec![Some(expected_quote)]); assert_eq!( diff --git a/crates/database/Cargo.toml b/crates/database/Cargo.toml index 572530a842..95a288e8df 100644 --- a/crates/database/Cargo.toml +++ b/crates/database/Cargo.toml @@ -13,6 +13,7 @@ futures = { workspace = true } hex = { workspace = true } sqlx = { workspace = true } strum = { workspace = true } +serde_json = { workspace = true } [dev-dependencies] maplit = { workspace = true } diff --git a/crates/database/src/lib.rs b/crates/database/src/lib.rs index 48e6496ef6..ec30cd557d 100644 --- a/crates/database/src/lib.rs +++ b/crates/database/src/lib.rs @@ -72,8 +72,6 @@ pub const TABLES: &[&str] = &[ "auction_participants", "app_data", "jit_orders", - "quote_interactions", - "order_quote_interactions", ]; /// The names of potentially big volume tables we use in the db. @@ -87,8 +85,7 @@ pub fn all_tables() -> impl Iterator { #[allow(non_snake_case)] pub async fn clear_DANGER_(ex: &mut PgTransaction<'_>) -> sqlx::Result<()> { for table in all_tables() { - ex.execute(format!("TRUNCATE {table} CASCADE;").as_str()) - .await?; + ex.execute(format!("TRUNCATE {table};").as_str()).await?; } Ok(()) } diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index d4c67b9cad..db2a2ad264 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -5,7 +5,6 @@ use { Address, AppId, OrderUid, - PgTransaction, TransactionHash, }, futures::stream::BoxStream, @@ -330,7 +329,8 @@ pub struct Quote { pub sell_amount: BigDecimal, pub buy_amount: BigDecimal, pub solver: Address, - pub verified: bool, + pub verified: Option, // Null value support + pub metadata: Option, // Null value support } pub async fn insert_quotes(ex: &mut PgConnection, quotes: &[Quote]) -> Result<(), sqlx::Error> { @@ -349,9 +349,10 @@ INSERT INTO order_quotes ( sell_amount, buy_amount, solver, - verified + verified, + metadata ) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8)"#; +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"#; pub async fn insert_quote_and_update_on_conflict( ex: &mut PgConnection, @@ -365,7 +366,7 @@ pub async fn insert_quote_and_update_on_conflict( " ON CONFLICT (order_uid) DO UPDATE SET gas_amount = $2, gas_price = $3, sell_token_price = $4, sell_amount = $5, -buy_amount = $6, verified = $8 +buy_amount = $6, verified = $8, metadata = $9 " ); sqlx::query(QUERY) @@ -377,6 +378,7 @@ buy_amount = $6, verified = $8 .bind("e.buy_amount) .bind(quote.solver) .bind(quote.verified) + .bind("e.metadata) .execute(ex) .await?; Ok(()) @@ -392,6 +394,7 @@ pub async fn insert_quote(ex: &mut PgConnection, quote: &Quote) -> Result<(), sq .bind("e.buy_amount) .bind(quote.solver) .bind(quote.verified) + .bind("e.metadata) .execute(ex) .await?; Ok(()) @@ -450,51 +453,6 @@ AND cancellation_timestamp IS NULL .map(|_| ()) } -/// One row in the `order_quote_interactions` table. -#[derive(Clone, Default, Debug, PartialEq, sqlx::FromRow)] -pub struct OrderQuoteInteraction { - pub order_uid: OrderUid, - pub index: i32, - pub target: Address, - pub value: BigDecimal, - pub call_data: Vec, -} - -pub async fn insert_order_quote_interaction( - ex: &mut PgConnection, - quote_interaction: &OrderQuoteInteraction, -) -> Result<(), sqlx::Error> { - const INSERT_ORDER_QUOTE_INTERACTION_QUERY: &str = r#" - INSERT INTO order_quote_interactions ( - order_uid, - index, - target, - value, - call_data - ) - VALUES ($1, $2, $3, $4, $5) - "#; - sqlx::query(INSERT_ORDER_QUOTE_INTERACTION_QUERY) - .bind(quote_interaction.order_uid) - .bind(quote_interaction.index) - .bind(quote_interaction.target) - .bind("e_interaction.value) - .bind("e_interaction.call_data) - .execute(ex) - .await?; - Ok(()) -} - -pub async fn insert_order_quote_interactions( - ex: &mut PgTransaction<'_>, - quote_interactions: &[OrderQuoteInteraction], -) -> Result<(), sqlx::Error> { - for interactions in quote_interactions { - insert_order_quote_interaction(ex, interactions).await?; - } - Ok(()) -} - /// Interactions are read as arrays of their fields: target, value, data. /// This is done as sqlx does not support reading arrays of more complicated /// types than just one field. The pre_ and post_interaction's data of @@ -549,6 +507,7 @@ pub struct FullOrderWithQuote { pub quote_gas_price: Option, pub quote_sell_token_price: Option, pub quote_verified: Option, + pub quote_metadata: Option, pub solver: Option
, } @@ -643,6 +602,7 @@ pub async fn single_full_order_with_quote( ", o_quotes.gas_price as quote_gas_price", ", o_quotes.sell_token_price as quote_sell_token_price", ", o_quotes.verified as quote_verified", + ", o_quotes.metadata as quote_metadata", ", o_quotes.solver as solver", " FROM ", FROM, " LEFT JOIN order_quotes o_quotes ON o.uid = o_quotes.order_uid", @@ -1253,7 +1213,8 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), - verified: false, + verified: None, + metadata: None, }; insert_quote(&mut db, "e).await.unwrap(); insert_quote_and_update_on_conflict(&mut db, "e) @@ -1306,6 +1267,20 @@ mod tests { let mut db = db.begin().await.unwrap(); crate::clear_DANGER_(&mut db).await.unwrap(); + let metadata: serde_json::Value = serde_json::from_str( + r#"{ "interactions": [ { + "target": "0102030405060708091011121314151617181920", + "value": 2.1, + "call_data": "0A0B0C102030" + },{ + "target": "FF02030405060708091011121314151617181920", + "value": 1.2, + "call_data": "FF0B0C102030" + }] + }"#, + ) + .unwrap(); + let quote = Quote { order_uid: Default::default(), gas_amount: 1., @@ -1314,7 +1289,8 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), - verified: false, + verified: Some(true), + metadata: Some(metadata), }; insert_quote(&mut db, "e).await.unwrap(); let quote_ = read_quote(&mut db, "e.order_uid) @@ -1341,7 +1317,8 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), - verified: false, + verified: None, + metadata: None, }; insert_quote(&mut db, "e).await.unwrap(); let order_with_quote = single_full_order_with_quote(&mut db, "e.order_uid) @@ -2200,63 +2177,53 @@ mod tests { #[tokio::test] #[ignore] - async fn postgres_insert_order_quote_interaction() { + async fn postgres_get_quote_with_no_metadata_and_validity() { + // This test checks backward compatibility let mut db = PgConnection::connect("postgresql://").await.unwrap(); let mut db = db.begin().await.unwrap(); crate::clear_DANGER_(&mut db).await.unwrap(); - let interaction = OrderQuoteInteraction { + let quote = Quote { order_uid: Default::default(), - index: Default::default(), - target: ByteArray([1; 20]), - value: 2.into(), - call_data: vec![3; 20], + gas_amount: 1., + gas_price: 2., + sell_token_price: 3., + sell_amount: 4.into(), + buy_amount: 5.into(), + solver: ByteArray([1; 20]), + verified: None, + metadata: None, }; - insert_order_quote_interaction(&mut db, &interaction) - .await - .unwrap(); - const QUERY: &str = r#" - SELECT * FROM order_quote_interactions - WHERE order_uid = $1 - "#; - - let interactions: Vec = sqlx::query_as(QUERY) - .bind(interaction.order_uid) - .fetch_all(&mut db as &mut PgConnection) - .await - .unwrap(); - assert_eq!(*interactions.first().unwrap(), interaction); - } - - #[tokio::test] - #[ignore] - async fn postgres_insert_order_quote_interaction_on_conflict() { - let mut db = PgConnection::connect("postgresql://").await.unwrap(); - let mut db = db.begin().await.unwrap(); - crate::clear_DANGER_(&mut db).await.unwrap(); - - let order_uid = Default::default(); - let interaction1 = OrderQuoteInteraction { + // insert quote with verified and metadata fields set to NULL + sqlx::query( + r#" + INSERT INTO order_quotes ( order_uid, - index: Default::default(), - target: ByteArray([1; 20]), - value: 2.into(), - call_data: vec![3; 20], - }; - insert_order_quote_interaction(&mut db, &interaction1) - .await - .unwrap(); + gas_amount, + gas_price, + sell_token_price, + sell_amount, + buy_amount, + solver + ) + VALUES ($1, $2, $3, $4, $5, $6, $7)"#, + ) + .bind(quote.order_uid) + .bind(quote.gas_amount) + .bind(quote.gas_price) + .bind(quote.sell_token_price) + .bind("e.sell_amount) + .bind("e.buy_amount) + .bind(quote.solver) + .execute(&mut db as &mut PgConnection) + .await + .unwrap(); - let interaction2 = OrderQuoteInteraction { - order_uid, - index: Default::default(), - target: ByteArray([4; 20]), - value: 4.into(), - call_data: vec![5; 20], - }; - insert_order_quote_interaction(&mut db, &interaction2) + let quote_ = read_quote(&mut db, "e.order_uid) .await - .expect_err("Inserting interaction for the same key should fail."); + .unwrap() + .unwrap(); + assert_eq!(quote, quote_); } } diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index f15f41e821..505dad79b9 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -4,7 +4,6 @@ use { sqlx::{ types::chrono::{DateTime, Utc}, PgConnection, - Row, }, }; @@ -35,28 +34,8 @@ pub struct Quote { pub expiration_timestamp: DateTime, pub quote_kind: QuoteKind, pub solver: Address, - pub verified: bool, -} - -impl Quote { - pub fn to_quote_with_interactions(&self) -> QuoteWithInteractions { - QuoteWithInteractions { - id: self.id, - sell_token: self.sell_token, - buy_token: self.buy_token, - sell_amount: self.sell_amount.clone(), - buy_amount: self.buy_amount.clone(), - gas_amount: self.gas_amount, - gas_price: self.gas_price, - sell_token_price: self.sell_token_price, - order_kind: self.order_kind, - expiration_timestamp: self.expiration_timestamp, - quote_kind: self.quote_kind.clone(), - solver: self.solver, - verified: self.verified, - interactions: vec![], - } - } + pub verified: Option, // Null value support + pub metadata: Option, // Null value support } /// Stores the quote and returns the id. The id of the quote parameter is not @@ -75,9 +54,10 @@ INSERT INTO quotes ( expiration_timestamp, quote_kind, solver, - verified + verified, + metadata ) -VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) +VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) RETURNING id "#; let (id,) = sqlx::query_as(QUERY) @@ -93,85 +73,19 @@ RETURNING id .bind("e.quote_kind) .bind(quote.solver) .bind(quote.verified) + .bind(sqlx::types::Json("e.metadata)) .fetch_one(ex) .await?; Ok(id) } -#[derive(Clone, Debug, PartialEq)] -pub struct QuoteWithInteractions { - pub id: QuoteId, - pub sell_token: Address, - pub buy_token: Address, - pub sell_amount: BigDecimal, - pub buy_amount: BigDecimal, - pub gas_amount: f64, - pub gas_price: f64, - pub sell_token_price: f64, - pub order_kind: OrderKind, - pub expiration_timestamp: DateTime, - pub quote_kind: QuoteKind, - pub solver: Address, - pub verified: bool, - pub interactions: Vec, -} - -impl sqlx::FromRow<'_, sqlx::postgres::PgRow> for QuoteWithInteractions { - fn from_row(row: &sqlx::postgres::PgRow) -> sqlx::Result { - let id = row.get("id"); - - let interactions = match row - .get::, BigDecimal, Vec)>>, &str>( - "interactions", - ) { - None => vec![], - Some(col) => col - .into_iter() - .map(|(index, target, value, call_data)| QuoteInteraction { - quote_id: id, - index, - target, - value, - call_data, - }) - .collect(), - }; - - Ok(QuoteWithInteractions { - id, - sell_token: row.get("sell_token"), - buy_token: row.get("buy_token"), - sell_amount: row.get("sell_amount"), - buy_amount: row.get("buy_amount"), - gas_amount: row.get("gas_amount"), - gas_price: row.get("gas_price"), - sell_token_price: row.get("sell_token_price"), - order_kind: row.get("order_kind"), - expiration_timestamp: row.get("expiration_timestamp"), - quote_kind: row.get("quote_kind"), - solver: row.get("solver"), - verified: row.get("verified"), - interactions, - }) - } -} - -pub async fn get_quote_with_interactions( - ex: &mut PgConnection, - id: QuoteId, -) -> Result, sqlx::Error> { - sqlx::query_as( - r#" - SELECT q.*, array_agg((i.index, i.target, i.value, i.call_data)) FILTER (WHERE i.quote_id IS NOT NULL) as "interactions" - FROM quotes q - LEFT JOIN quote_interactions i ON i.quote_id = q.id - WHERE q.id = $1 - GROUP BY q.id - "#, - ) - .bind(id) - .fetch_optional(ex) - .await +pub async fn get(ex: &mut PgConnection, id: QuoteId) -> Result, sqlx::Error> { + const QUERY: &str = r#" +SELECT * +FROM quotes +WHERE id = $1 + "#; + sqlx::query_as(QUERY).bind(id).fetch_optional(ex).await } /// Fields for searching stored quotes. @@ -187,14 +101,13 @@ pub struct QuoteSearchParameters { pub quote_kind: QuoteKind, } -pub async fn find_quote_with_interactions( +pub async fn find( ex: &mut PgConnection, params: &QuoteSearchParameters, -) -> Result, sqlx::Error> { +) -> Result, sqlx::Error> { const QUERY: &str = r#" -SELECT quotes.*, array_agg((i.index, i.target, i.value, i.call_data)) FILTER (WHERE i.quote_id IS NOT NULL) as "interactions" +SELECT * FROM quotes -LEFT JOIN quote_interactions i ON i.quote_id = id WHERE sell_token = $1 AND buy_token = $2 AND @@ -206,11 +119,9 @@ WHERE order_kind = $6 AND expiration_timestamp >= $7 AND quote_kind = $8 -GROUP BY id ORDER BY gas_amount * gas_price * sell_token_price ASC LIMIT 1 "#; - sqlx::query_as(QUERY) .bind(params.sell_token) .bind(params.buy_token) @@ -294,30 +205,6 @@ mod tests { sqlx::{types::chrono::TimeZone, Connection}, }; - pub async fn get_quote( - ex: &mut PgConnection, - id: QuoteId, - ) -> Result, sqlx::Error> { - const QUERY: &str = r#" - SELECT * - FROM quotes - WHERE id = $1 - "#; - sqlx::query_as(QUERY).bind(id).fetch_optional(ex).await - } - - pub async fn get_quote_interactions( - ex: &mut PgConnection, - quote_id: QuoteId, - ) -> Result, sqlx::Error> { - const QUERY: &str = r#" - SELECT * - FROM quote_interactions - WHERE quote_id = $1 - "#; - sqlx::query_as(QUERY).bind(quote_id).fetch_all(ex).await - } - /// The postgres database in our CI has different datetime precision than /// the `DateTime` uses. This leads to issues comparing round-tripped data. /// Work around the issue by created `DateTime`s with lower precision. @@ -346,16 +233,17 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Standard, solver: ByteArray([1; 20]), - verified: false, + verified: None, + metadata: None, }; let id = save(&mut db, "e).await.unwrap(); quote.id = id; - assert_eq!(get_quote(&mut db, id).await.unwrap().unwrap(), quote); + assert_eq!(get(&mut db, id).await.unwrap().unwrap(), quote); remove_expired_quotes(&mut db, now + Duration::seconds(30)) .await .unwrap(); - assert_eq!(get_quote(&mut db, id).await.unwrap(), None); + assert_eq!(get(&mut db, id).await.unwrap(), None); } #[tokio::test] @@ -380,7 +268,8 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Standard, solver: ByteArray([1; 20]), - verified: false, + verified: None, + metadata: None, }; let token_b = ByteArray([2; 20]); @@ -397,7 +286,8 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Standard, solver: ByteArray([2; 20]), - verified: false, + verified: None, + metadata: None, }; // Save two measurements for token_a @@ -448,14 +338,11 @@ mod tests { quote_kind: QuoteKind::Standard, }; assert_eq!( - find_quote_with_interactions(&mut db, &search_a) - .await - .unwrap() - .unwrap(), - quotes_a[0].to_quote_with_interactions(), + find(&mut db, &search_a).await.unwrap().unwrap(), + quotes_a[0] ); assert_eq!( - find_quote_with_interactions( + find( &mut db, &QuoteSearchParameters { expiration: now + Duration::seconds(30), @@ -465,12 +352,12 @@ mod tests { .await .unwrap() .unwrap(), - quotes_a[1].to_quote_with_interactions() + quotes_a[1] ); // Token A has readings for sell + fee amount equal to quoted amount. assert_eq!( - find_quote_with_interactions( + find( &mut db, &QuoteSearchParameters { sell_amount_0: quote_a.sell_amount.clone() - BigDecimal::from(1), @@ -481,12 +368,12 @@ mod tests { .await .unwrap() .unwrap(), - quotes_a[0].to_quote_with_interactions(), + quotes_a[0] ); // Token A has no reading for wrong filter assert_eq!( - find_quote_with_interactions( + find( &mut db, &QuoteSearchParameters { sell_amount_0: quote_a.sell_amount.clone() - BigDecimal::from(1), @@ -511,14 +398,11 @@ mod tests { quote_kind: QuoteKind::Standard, }; assert_eq!( - find_quote_with_interactions(&mut db, &search_b) - .await - .unwrap() - .unwrap(), - quotes_b[0].to_quote_with_interactions(), + find(&mut db, &search_b).await.unwrap().unwrap(), + quotes_b[0] ); assert_eq!( - find_quote_with_interactions( + find( &mut db, &QuoteSearchParameters { expiration: now + Duration::seconds(30), @@ -532,7 +416,7 @@ mod tests { // Token B has no reading for wrong filter assert_eq!( - find_quote_with_interactions( + find( &mut db, &QuoteSearchParameters { buy_amount: 99.into(), @@ -548,18 +432,8 @@ mod tests { remove_expired_quotes(&mut db, now + Duration::seconds(120)) .await .unwrap(); - assert_eq!( - find_quote_with_interactions(&mut db, &search_a) - .await - .unwrap(), - None - ); - assert_eq!( - find_quote_with_interactions(&mut db, &search_b) - .await - .unwrap(), - None - ); + assert_eq!(find(&mut db, &search_a).await.unwrap(), None); + assert_eq!(find(&mut db, &search_b).await.unwrap(), None); } #[tokio::test] @@ -585,7 +459,8 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Eip1271OnchainOrder, solver: ByteArray([1; 20]), - verified: false, + verified: None, + metadata: None, }; let id = save(&mut db, "e).await.unwrap(); quote.id = id; @@ -603,29 +478,32 @@ mod tests { quote_kind: quote.quote_kind.clone(), }; - assert_eq!( - find_quote_with_interactions(&mut db, &search_a) - .await - .unwrap() - .unwrap(), - quote.to_quote_with_interactions(), - ); + assert_eq!(find(&mut db, &search_a).await.unwrap().unwrap(), quote); search_a.quote_kind = QuoteKind::Standard; - assert_eq!( - find_quote_with_interactions(&mut db, &search_a) - .await - .unwrap(), - None, - ); + assert_eq!(find(&mut db, &search_a).await.unwrap(), None,); } #[tokio::test] #[ignore] - async fn postgres_insert_quote_interaction() { + async fn postgres_insert_quote_metadata() { let mut db = PgConnection::connect("postgresql://").await.unwrap(); let mut db = db.begin().await.unwrap(); crate::clear_DANGER_(&mut db).await.unwrap(); + let metadata: serde_json::Value = serde_json::from_str( + r#"{ "interactions": [ { + "target": "0102030405060708091011121314151617181920", + "value": 2.1, + "call_data": "0A0B0C102030" + },{ + "target": "FF02030405060708091011121314151617181920", + "value": 1.2, + "call_data": "FF0B0C102030" + }] + }"#, + ) + .unwrap(); + let quote = Quote { id: Default::default(), sell_token: ByteArray([1; 20]), @@ -639,36 +517,24 @@ mod tests { expiration_timestamp: low_precision_now(), quote_kind: QuoteKind::Standard, solver: ByteArray([1; 20]), - verified: false, + verified: None, + metadata: Some(metadata.clone()), }; // store quote in database let id = save(&mut db, "e).await.unwrap(); - let quote_interaction = QuoteInteraction { - quote_id: id, - index: Default::default(), - target: ByteArray([1; 20]), - value: 2.into(), - call_data: vec![3; 20], - }; - insert_quote_interaction(&mut db, "e_interaction) - .await - .unwrap(); - - let interactions = get_quote_interactions(&mut db, quote_interaction.quote_id) - .await - .unwrap(); - assert_eq!(*interactions.first().unwrap(), quote_interaction); + let stored_quote = get(&mut db, id).await.unwrap().unwrap(); + assert_eq!(stored_quote.metadata.unwrap(), metadata); } #[tokio::test] #[ignore] - async fn postgres_removed_quote_interactions_by_id() { + async fn postgres_get_quote_with_no_metadata_and_validity() { + // This test checks backward compatibility let mut db = PgConnection::connect("postgresql://").await.unwrap(); let mut db = db.begin().await.unwrap(); crate::clear_DANGER_(&mut db).await.unwrap(); - let now = low_precision_now(); let quote = Quote { id: Default::default(), sell_token: ByteArray([1; 20]), @@ -679,142 +545,50 @@ mod tests { gas_price: 6., sell_token_price: 7., order_kind: OrderKind::Sell, - expiration_timestamp: now, - quote_kind: QuoteKind::Standard, - solver: ByteArray([1; 20]), - verified: false, - }; - // store quote in database - let id = save(&mut db, "e).await.unwrap(); - - let quote_interactions = [ - QuoteInteraction { - quote_id: id, - index: 0, - target: ByteArray([1; 20]), - value: 2.into(), - call_data: vec![3; 20], - }, - QuoteInteraction { - quote_id: id, - index: 1, - target: ByteArray([1; 20]), - value: 2.into(), - call_data: vec![3; 20], - }, - ]; - // store interactions for the quote in database - insert_quote_interactions(&mut db, "e_interactions) - .await - .unwrap(); - - let interactions = get_quote_interactions(&mut db, id).await.unwrap(); - assert_eq!(interactions.len(), 2); - - // remove quote using expired functino call, should also remove interactions - remove_expired_quotes(&mut db, now + Duration::seconds(30)) - .await - .unwrap(); - - let interactions = get_quote_interactions(&mut db, id).await.unwrap(); - assert!(interactions.is_empty()); - } - - #[tokio::test] - #[ignore] - async fn postgres_get_quote_with_interactions_by_id() { - let mut db = PgConnection::connect("postgresql://").await.unwrap(); - let mut db = db.begin().await.unwrap(); - crate::clear_DANGER_(&mut db).await.unwrap(); - - let now = low_precision_now(); - let mut quote = Quote { - id: Default::default(), - sell_token: ByteArray([1; 20]), - buy_token: ByteArray([2; 20]), - sell_amount: 3.into(), - buy_amount: 4.into(), - gas_amount: 5., - gas_price: 6., - sell_token_price: 7., - order_kind: OrderKind::Sell, - expiration_timestamp: now, + expiration_timestamp: low_precision_now(), quote_kind: QuoteKind::Standard, solver: ByteArray([1; 20]), - verified: false, + verified: None, + metadata: None, }; - let mut quote2 = quote.clone(); - let mut quote3 = quote.clone(); - // store quote in database - let id = save(&mut db, "e).await.unwrap(); - let id2 = save(&mut db, "e2).await.unwrap(); - let id3 = save(&mut db, "e3).await.unwrap(); - quote.id = id; - quote2.id = id2; - quote3.id = id3; - - let quote_interactions = [ - QuoteInteraction { - quote_id: id, - index: 0, - target: ByteArray([1; 20]), - value: 2.into(), - call_data: vec![3; 20], - }, - QuoteInteraction { - quote_id: id, - index: 1, - target: ByteArray([4; 20]), - value: 5.into(), - call_data: vec![6; 20], - }, - QuoteInteraction { - quote_id: id2, - index: 0, - target: ByteArray([7; 20]), - value: 8.into(), - call_data: vec![9; 20], - }, - ]; - // store interactions for the quote in database - insert_quote_interactions(&mut db, "e_interactions) - .await - .unwrap(); - let returned_quote: QuoteWithInteractions = get_quote_with_interactions(&mut db, id) - .await - .unwrap() - .unwrap(); - - assert_eq!(returned_quote.id, quote.id); - assert_eq!(returned_quote.interactions.len(), 2); - for i in [0, 1] { - assert_eq!( - returned_quote - .interactions - .iter() - .find(|val| val.index == i) - .unwrap(), - "e_interactions[i as usize] - ); - } - - let returned_quote2: QuoteWithInteractions = get_quote_with_interactions(&mut db, id2) + // store quote with verified and metadata fields set to NULL + const QUERY: &str = r#" + INSERT INTO quotes ( + sell_token, + buy_token, + sell_amount, + buy_amount, + gas_amount, + gas_price, + sell_token_price, + order_kind, + expiration_timestamp, + quote_kind, + solver + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + RETURNING id + "#; + let (id,) = sqlx::query_as(QUERY) + .bind(quote.sell_token) + .bind(quote.buy_token) + .bind("e.sell_amount) + .bind("e.buy_amount) + .bind(quote.gas_amount) + .bind(quote.gas_price) + .bind(quote.sell_token_price) + .bind(quote.order_kind) + .bind(quote.expiration_timestamp) + .bind("e.quote_kind) + .bind(quote.solver) + .fetch_one(&mut db as &mut PgConnection) .await - .unwrap() .unwrap(); - assert_eq!(returned_quote2.id, quote2.id); - assert_eq!(returned_quote2.interactions.len(), 1); - assert_eq!( - returned_quote2.interactions.first().unwrap(), - "e_interactions[2] - ); - let returned_quote3: QuoteWithInteractions = get_quote_with_interactions(&mut db, id3) - .await - .unwrap() - .unwrap(); - assert_eq!(returned_quote3.id, quote3.id); - assert!(returned_quote3.interactions.is_empty()); + // read back stored quote + let stored_quote = get(&mut db, id).await.unwrap().unwrap(); + assert!(stored_quote.verified.is_none()); + assert!(stored_quote.metadata.is_none()); } } diff --git a/crates/orderbook/src/api/post_order.rs b/crates/orderbook/src/api/post_order.rs index 2208bff6fd..787f6e32a6 100644 --- a/crates/orderbook/src/api/post_order.rs +++ b/crates/orderbook/src/api/post_order.rs @@ -262,6 +262,13 @@ impl IntoWarpReply for AddOrderError { super::error("InvalidReplacement", err.to_string()), StatusCode::UNAUTHORIZED, ), + AddOrderError::MetadataSerializationFailed => reply::with_status( + super::error( + "MetadataSerializationFailed", + "quote metadata failed to serialize as json", + ), + StatusCode::BAD_REQUEST, + ), } } } diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index 349530c72c..9c5f6b2c44 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -1,5 +1,6 @@ use { super::Postgres, + crate::orderbook::AddOrderError, anyhow::{Context as _, Result}, app_data::AppDataHash, async_trait::async_trait, @@ -7,7 +8,7 @@ use { database::{ byte_array::ByteArray, order_events::{insert_order_event, OrderEvent, OrderEventLabel}, - orders::{self, FullOrder, OrderKind as DbOrderKind, OrderQuoteInteraction}, + orders::{self, FullOrder, OrderKind as DbOrderKind}, }, ethcontract::H256, futures::{stream::TryStreamExt, FutureExt, StreamExt}, @@ -45,7 +46,7 @@ use { signing_scheme_into, }, fee::FeeParameters, - order_quoting::Quote, + order_quoting::{Quote, QuoteMetadata}, order_validation::{is_order_outside_market_price, Amounts, LimitOrderCounting}, }, sqlx::{types::BigDecimal, Connection, PgConnection}, @@ -90,20 +91,31 @@ pub struct OrderWithQuote { } impl OrderWithQuote { - pub fn new(order: Order, quote: Option) -> Self { - Self { - quote: quote.map(|quote| orders::Quote { - order_uid: ByteArray(order.metadata.uid.0), - gas_amount: quote.data.fee_parameters.gas_amount, - gas_price: quote.data.fee_parameters.gas_price, - sell_token_price: quote.data.fee_parameters.sell_token_price, - sell_amount: u256_to_big_decimal("e.sell_amount), - buy_amount: u256_to_big_decimal("e.buy_amount), - solver: ByteArray(quote.data.solver.0), - verified: quote.data.verified, - }), + pub fn try_new(order: Order, quote: Option) -> Result { + Ok(Self { + quote: quote + .map(|quote| { + Ok::(orders::Quote { + order_uid: ByteArray(order.metadata.uid.0), + gas_amount: quote.data.fee_parameters.gas_amount, + gas_price: quote.data.fee_parameters.gas_price, + sell_token_price: quote.data.fee_parameters.sell_token_price, + sell_amount: u256_to_big_decimal("e.sell_amount), + buy_amount: u256_to_big_decimal("e.buy_amount), + solver: ByteArray(quote.data.solver.0), + verified: Some(quote.data.verified), + metadata: Some( + QuoteMetadata { + interactions: quote.data.interactions.clone(), + } + .try_into() + .map_err(|_| AddOrderError::MetadataSerializationFailed)?, + ), + }) + }) + .transpose()?, order, - } + }) } } @@ -113,6 +125,7 @@ pub enum InsertionError { DbError(sqlx::Error), /// Full app data to be inserted doesn't match existing. AppDataMismatch(Vec), + MetadataSerializationFailed, } impl From for InsertionError { @@ -234,32 +247,19 @@ async fn insert_quote( sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - verified: quote.data.verified, - }; - let dbinteractions = quote - .data - .interactions - .iter() - .enumerate() - .map(|(idx, interaction)| { - OrderQuoteInteraction { - order_uid: dbquote.order_uid, - index: idx.try_into().unwrap(), // safe to unwrap - target: ByteArray(interaction.target.0), - value: u256_to_big_decimal(&interaction.value), - call_data: interaction.call_data.clone(), + verified: Some(quote.data.verified), + metadata: Some( + QuoteMetadata { + interactions: quote.data.interactions.clone(), } - }) - .collect::>(); + .try_into() + .map_err(|_| InsertionError::MetadataSerializationFailed)?, + ), + }; - let mut transaction = ex.begin().await?; - database::orders::insert_quote(&mut transaction, &dbquote) + database::orders::insert_quote(ex, &dbquote) .await - .map_err(InsertionError::DbError)?; - database::orders::insert_order_quote_interactions(&mut transaction, dbinteractions.as_slice()) - .await - .map_err(InsertionError::DbError)?; - transaction.commit().await.map_err(InsertionError::DbError) + .map_err(InsertionError::DbError) } #[async_trait::async_trait] @@ -384,7 +384,6 @@ impl OrderStoring for Postgres { order_with_quote.quote_gas_amount, order_with_quote.quote_gas_price, order_with_quote.quote_sell_token_price, - order_with_quote.quote_verified, order_with_quote.solver, ) { ( @@ -393,7 +392,6 @@ impl OrderStoring for Postgres { Some(gas_amount), Some(gas_price), Some(sell_token_price), - Some(verified), Some(solver), ) => Some(orders::Quote { order_uid: order_with_quote.full_order.uid, @@ -403,7 +401,8 @@ impl OrderStoring for Postgres { sell_amount, buy_amount, solver, - verified, + verified: order_with_quote.quote_verified, + metadata: order_with_quote.quote_metadata, }), _ => None, }; @@ -1242,6 +1241,6 @@ mod tests { let single_order_with_quote = db.single_order_with_quote(&uid).await.unwrap().unwrap(); assert_eq!(single_order_with_quote.order, order); - assert!(single_order_with_quote.quote.unwrap().verified,); + assert!(single_order_with_quote.quote.unwrap().verified.unwrap()); } } diff --git a/crates/orderbook/src/database/quotes.rs b/crates/orderbook/src/database/quotes.rs index 7ef8810269..9223ba4b7e 100644 --- a/crates/orderbook/src/database/quotes.rs +++ b/crates/orderbook/src/database/quotes.rs @@ -4,14 +4,9 @@ use { chrono::{DateTime, Utc}, model::quote::QuoteId, shared::{ - event_storing_helpers::{ - create_db_search_parameters, - create_quote_interactions_insert_data, - create_quote_row, - }, + event_storing_helpers::{create_db_search_parameters, create_quote_row}, order_quoting::{QuoteData, QuoteSearchParameters, QuoteStoring}, }, - sqlx::Acquire, }; #[async_trait::async_trait] @@ -23,15 +18,8 @@ impl QuoteStoring for Postgres { .start_timer(); let mut ex = self.pool.acquire().await?; - let row = create_quote_row(&data); - - let mut transaction = ex.begin().await?; - let id = database::quotes::save(&mut transaction, &row).await?; - if !data.interactions.is_empty() { - let interactions = create_quote_interactions_insert_data(id, &data)?; - database::quotes::insert_quote_interactions(&mut transaction, &interactions).await?; - } - transaction.commit().await.context("commit")?; + let row = create_quote_row(&data)?; + let id = database::quotes::save(&mut ex, &row).await?; Ok(id) } @@ -42,10 +30,8 @@ impl QuoteStoring for Postgres { .start_timer(); let mut ex = self.pool.acquire().await?; - - let query_result = database::quotes::get_quote_with_interactions(&mut ex, id).await?; - - Ok(query_result.map(QuoteData::try_from).transpose()?) + let quote = database::quotes::get(&mut ex, id).await?; + quote.map(TryFrom::try_from).transpose() } async fn find( @@ -60,13 +46,11 @@ impl QuoteStoring for Postgres { let mut ex = self.pool.acquire().await?; let params = create_db_search_parameters(params, expiration); - - let query_result = database::quotes::find_quote_with_interactions(&mut ex, ¶ms) + let quote = database::quotes::find(&mut ex, ¶ms) .await .context("failed finding quote by parameters")?; - - query_result - .map(|query_result| Ok((query_result.id, query_result.try_into()?))) + quote + .map(|quote| Ok((quote.id, quote.try_into()?))) .transpose() } } diff --git a/crates/orderbook/src/orderbook.rs b/crates/orderbook/src/orderbook.rs index fc637a1c2f..063fa0aff5 100644 --- a/crates/orderbook/src/orderbook.rs +++ b/crates/orderbook/src/orderbook.rs @@ -141,6 +141,8 @@ pub enum AddOrderError { provided: String, existing: String, }, + #[error("quote metadata failed to serialize as json")] + MetadataSerializationFailed, } impl AddOrderError { @@ -161,6 +163,9 @@ impl AddOrderError { s.into_owned() }, }, + InsertionError::MetadataSerializationFailed => { + AddOrderError::MetadataSerializationFailed + } } } } @@ -255,7 +260,7 @@ impl Orderbook { .await .map_err(|err| AddOrderError::from_insertion(err, &order))?; Metrics::on_order_operation( - &OrderWithQuote::new(order.clone(), quote), + &OrderWithQuote::try_new(order.clone(), quote)?, OrderOperation::Created, ); @@ -413,7 +418,7 @@ impl Orderbook { .map_err(|err| AddOrderError::from_insertion(err, &validated_new_order))?; Metrics::on_order_operation(&old_order, OrderOperation::Cancelled); Metrics::on_order_operation( - &OrderWithQuote::new(validated_new_order.clone(), quote), + &OrderWithQuote::try_new(validated_new_order.clone(), quote)?, OrderOperation::Created, ); diff --git a/crates/shared/src/event_storing_helpers.rs b/crates/shared/src/event_storing_helpers.rs index 3f2f160ca6..dbcaeb327a 100644 --- a/crates/shared/src/event_storing_helpers.rs +++ b/crates/shared/src/event_storing_helpers.rs @@ -1,24 +1,24 @@ use { crate::{ db_order_conversions::order_kind_into, - order_quoting::{quote_kind_from_signing_scheme, QuoteData, QuoteSearchParameters}, + order_quoting::{ + quote_kind_from_signing_scheme, + QuoteData, + QuoteMetadata, + QuoteSearchParameters, + }, }, anyhow::Result, chrono::{DateTime, Utc}, database::{ byte_array::ByteArray, - quotes::{ - Quote as DbQuote, - QuoteId, - QuoteInteraction as DbQuoteInteraction, - QuoteSearchParameters as DbQuoteSearchParameters, - }, + quotes::{Quote as DbQuote, QuoteSearchParameters as DbQuoteSearchParameters}, }, number::conversions::u256_to_big_decimal, }; -pub fn create_quote_row(data: &QuoteData) -> DbQuote { - DbQuote { +pub fn create_quote_row(data: &QuoteData) -> Result { + Ok(DbQuote { id: Default::default(), sell_token: ByteArray(data.sell_token.0), buy_token: ByteArray(data.buy_token.0), @@ -31,27 +31,14 @@ pub fn create_quote_row(data: &QuoteData) -> DbQuote { expiration_timestamp: data.expiration, quote_kind: data.quote_kind.clone(), solver: ByteArray(data.solver.0), - verified: data.verified, - } -} - -pub fn create_quote_interactions_insert_data( - id: QuoteId, - data: &QuoteData, -) -> Result> { - data.interactions - .iter() - .enumerate() - .map(|(index, interaction)| { - Ok(DbQuoteInteraction { - quote_id: id, - index: index.try_into()?, - target: ByteArray(interaction.target.0), - value: u256_to_big_decimal(&interaction.value), - call_data: interaction.call_data.clone(), - }) - }) - .collect() + verified: Some(data.verified), + metadata: Some( + QuoteMetadata { + interactions: data.interactions.clone(), + } + .try_into()?, + ), + }) } pub fn create_db_search_parameters( diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index 79c98488fa..f30654710a 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -202,45 +202,6 @@ impl TryFrom for QuoteData { } } -impl TryFrom for QuoteData { - type Error = anyhow::Error; - - fn try_from(input: database::quotes::QuoteWithInteractions) -> Result { - Ok(QuoteData { - sell_token: H160(input.sell_token.0), - buy_token: H160(input.buy_token.0), - quoted_sell_amount: big_decimal_to_u256(&input.sell_amount) - .context("quoted sell amount is not a valid U256")?, - quoted_buy_amount: big_decimal_to_u256(&input.buy_amount) - .context("quoted buy amount is not a valid U256")?, - fee_parameters: FeeParameters { - gas_amount: input.gas_amount, - gas_price: input.gas_price, - sell_token_price: input.sell_token_price, - }, - kind: order_kind_from(input.order_kind), - expiration: input.expiration_timestamp, - quote_kind: input.quote_kind, - solver: H160(input.solver.0), - // Even if the quote was verified at the time of creation - // it might no longer be accurate. - verified: false, - interactions: input - .interactions - .iter() - .map(|data| { - Ok(InteractionData { - target: H160(data.target.0), - value: big_decimal_to_u256(&data.value) - .context("quote interaction value is not a valid U256")?, - call_data: data.call_data.clone(), - }) - }) - .collect::>>()?, - }) - } -} - #[mockall::automock] #[async_trait::async_trait] pub trait OrderQuoting: Send + Sync { @@ -672,6 +633,19 @@ pub fn quote_kind_from_signing_scheme(scheme: &QuoteSigningScheme) -> QuoteKind } } +#[derive(serde::Serialize)] +pub struct QuoteMetadata { + pub interactions: Vec, +} + +impl TryInto for QuoteMetadata { + type Error = serde_json::Error; + + fn try_into(self) -> std::result::Result { + serde_json::to_value(self) + } +} + #[cfg(test)] mod tests { use { diff --git a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql b/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql deleted file mode 100644 index 6c58bafc0b..0000000000 --- a/database/sql/V075__add_call_data_to_quotes_and_order_quotes.sql +++ /dev/null @@ -1,38 +0,0 @@ --- This migration script is reversible. - --- Step 1: Add a new column to the quotes table -ALTER TABLE quotes - ADD COLUMN verified boolean; - --- Step 2: Add a new column to the order_quotes table -ALTER TABLE order_quotes - ADD COLUMN verified boolean; - --- Step 3: Create table with quote interactions -CREATE TABLE quote_interactions ( - quote_id bigint NOT NULL, - index int NOT NULL, - target bytea NOT NULL, - value numeric(78,0) NOT NULL, - call_data bytea NOT NULL, - - PRIMARY KEY (quote_id, index), - FOREIGN KEY (quote_id) REFERENCES quotes(id) ON DELETE CASCADE -); - --- Get a specific quote's interactions. -CREATE INDEX quote_id_interactions ON quote_interactions USING HASH (quote_id); - --- Step 4: Create table with quote interactions for order -CREATE TABLE order_quote_interactions ( - order_uid bytea NOT NULL, - index int NOT NULL, - target bytea NOT NULL, - value numeric(78,0) NOT NULL, - call_data bytea NOT NULL, - - PRIMARY KEY (order_uid, index) -); - --- Get a specific order's interactions. -CREATE INDEX order_uid_interactions ON order_quote_interactions USING HASH (order_uid); diff --git a/database/sql/V075__add_metadata_to_quotes_and_order_quotes.sql b/database/sql/V075__add_metadata_to_quotes_and_order_quotes.sql new file mode 100644 index 0000000000..426e540c9f --- /dev/null +++ b/database/sql/V075__add_metadata_to_quotes_and_order_quotes.sql @@ -0,0 +1,11 @@ +-- This migration script is reversible. + +-- Step 1: Add two new columns to the quotes table +ALTER TABLE quotes + ADD COLUMN verified boolean, + ADD COLUMN metadata json; + +-- Step 2: Add two new columns to the order_quotes table +ALTER TABLE order_quotes + ADD COLUMN verified boolean, + ADD COLUMN metadata json; From 48728435d8f2b72be11c7df2419f2cd22842ebd0 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Thu, 5 Dec 2024 00:00:52 +0100 Subject: [PATCH 50/70] Fixed formatting and implementation --- Cargo.lock | 1 + crates/database/src/orders.rs | 4 ++-- crates/database/src/quotes.rs | 4 ++-- crates/shared/src/order_quoting.rs | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17ca562be0..90dc68d1b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1683,6 +1683,7 @@ dependencies = [ "futures", "hex", "maplit", + "serde_json", "sqlx", "strum", "tokio", diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index db2a2ad264..f380042e1f 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -329,8 +329,8 @@ pub struct Quote { pub sell_amount: BigDecimal, pub buy_amount: BigDecimal, pub solver: Address, - pub verified: Option, // Null value support - pub metadata: Option, // Null value support + pub verified: Option, // Null value support + pub metadata: Option, // Null value support } pub async fn insert_quotes(ex: &mut PgConnection, quotes: &[Quote]) -> Result<(), sqlx::Error> { diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 505dad79b9..acf2d5306b 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -34,7 +34,7 @@ pub struct Quote { pub expiration_timestamp: DateTime, pub quote_kind: QuoteKind, pub solver: Address, - pub verified: Option, // Null value support + pub verified: Option, // Null value support pub metadata: Option, // Null value support } @@ -73,7 +73,7 @@ RETURNING id .bind("e.quote_kind) .bind(quote.solver) .bind(quote.verified) - .bind(sqlx::types::Json("e.metadata)) + .bind("e.metadata) .fetch_one(ex) .await?; Ok(id) diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index f30654710a..8b4a16b9f7 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -633,7 +633,8 @@ pub fn quote_kind_from_signing_scheme(scheme: &QuoteSigningScheme) -> QuoteKind } } -#[derive(serde::Serialize)] +/// Used to store in database any quote metadata. +#[derive(Debug, serde::Serialize)] pub struct QuoteMetadata { pub interactions: Vec, } From 3ee496d00711c7e8764e02397a91331b0dde16ff Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Thu, 5 Dec 2024 00:18:58 +0100 Subject: [PATCH 51/70] Updated database readme --- database/README.md | 36 ++---------------------------------- 1 file changed, 2 insertions(+), 34 deletions(-) diff --git a/database/README.md b/database/README.md index fa570e95b2..dac697e8b7 100644 --- a/database/README.md +++ b/database/README.md @@ -239,6 +239,7 @@ Quotes that an order was created with. These quotes get stored persistently and buy\_amount | numeric | not null | buy\_amount of the quote used to create the order with solver | bytea | not null | public address of the solver that provided this quote verified | boolean | | information if quote was verified + metadata | json | | additional data associated with the quote in json format: interactions Indexes: - PRIMARY KEY: btree(`order_uid`) @@ -275,22 +276,6 @@ Column | Type | Nullable | Details Indexes: - PRIMARY KEY: btree(`uid`) -### order\_quote\_interactions - -This table contains all interactions provided by the Solver in response to the /quote API request. Interactions are saved persistently when creating an order from a quote. This data can be used to audit auction winning order. - - Column | Type | Nullable | Details ---------------------|---------|----------|-------- - order\_uid | bytea | not null | order that this interaction belongs to - index | integer | not null | sequence number of the interaction for that order - target | bytea | not null | address of the smart contract this interaction should call - value | numeric | not null | amount of ETH this interaction should send to the smart contract - call\_data | bytea | not null | call data that contains the function selector and the bytes passed to it - -Indexes: -- PRIMARY KEY: composite key(`order_uid`, `index`) -- order\_uid\_interactions: hash(`order_uid`) - ### fee_policies Contains all relevant data of fee policies applied to orders during auctions. @@ -357,29 +342,12 @@ Stores quotes in order to determine whether it makes sense to allow a user to cr quote\_kind | [enum](#quotekind) | not null | quotekind for which this quote is considered valid solver | bytea | not null | public address of the solver that provided this quote verified | boolean | | information if quote was verified + metadata | json | | additional data associated with the quote in json format: interactions Indexes: - PRIMARY KEY: btree(`id`) - quotes\_token\_expiration: btree (`sell_token`, `buy_token`, `expiration_timestamp` DESC) -### quote\_interactions - -This table contains all interactions provided by the Solver in response to the /quote API request. When an order is created based on a quote, interactions of that particular quote is stored in `order_quote_interactions` table. -Data in this table is removed at referenced quote expiration. - - Column | Type | Nullable | Details ---------------------|---------|----------|-------- - quote_id | bigint | not null | quote that this interaction belongs to - index | integer | not null | sequence number of the interaction for that quote - target | bytea | not null | address of the smart contract this interaction should call - value | numeric | not null | amount of ETH this interaction should send to the smart contract - call\_data | bytea | not null | call data that contains the function selector and the bytes passed to it - -Indexes: -- PRIMARY KEY: composite key(`quote_id`, `index`) -- FOREIGN KEY: `quote_id` into `quotes(id)` table -- quote\_id\_interactions: hash(`quote_id`) - ### proposed\_solutions All solutions reported by solvers, that were part of a solver competition. A solver competition can have more than one winner. From 36ed84bf946beadfe3816a18e28d2c783ef0d314 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 6 Dec 2024 10:34:08 +0100 Subject: [PATCH 52/70] Updated tests --- crates/database/src/orders.rs | 32 +++++--------------------- crates/database/src/quotes.rs | 43 +++++------------------------------ 2 files changed, 12 insertions(+), 63 deletions(-) diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index f380042e1f..9cc45bab09 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -1270,11 +1270,11 @@ mod tests { let metadata: serde_json::Value = serde_json::from_str( r#"{ "interactions": [ { "target": "0102030405060708091011121314151617181920", - "value": 2.1, + "value": 1, "call_data": "0A0B0C102030" },{ "target": "FF02030405060708091011121314151617181920", - "value": 1.2, + "value": 2, "call_data": "FF0B0C102030" }] }"#, @@ -2195,30 +2195,10 @@ mod tests { metadata: None, }; - // insert quote with verified and metadata fields set to NULL - sqlx::query( - r#" - INSERT INTO order_quotes ( - order_uid, - gas_amount, - gas_price, - sell_token_price, - sell_amount, - buy_amount, - solver - ) - VALUES ($1, $2, $3, $4, $5, $6, $7)"#, - ) - .bind(quote.order_uid) - .bind(quote.gas_amount) - .bind(quote.gas_price) - .bind(quote.sell_token_price) - .bind("e.sell_amount) - .bind("e.buy_amount) - .bind(quote.solver) - .execute(&mut db as &mut PgConnection) - .await - .unwrap(); + // insert quote with verified and metadata fields stored as NULL + insert_quote_and_update_on_conflict(&mut db, "e) + .await + .unwrap(); let quote_ = read_quote(&mut db, "e.order_uid) .await diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index acf2d5306b..2de88a7229 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -339,7 +339,7 @@ mod tests { }; assert_eq!( find(&mut db, &search_a).await.unwrap().unwrap(), - quotes_a[0] + quotes_a[0], ); assert_eq!( find( @@ -368,7 +368,7 @@ mod tests { .await .unwrap() .unwrap(), - quotes_a[0] + quotes_a[0], ); // Token A has no reading for wrong filter @@ -399,7 +399,7 @@ mod tests { }; assert_eq!( find(&mut db, &search_b).await.unwrap().unwrap(), - quotes_b[0] + quotes_b[0], ); assert_eq!( find( @@ -478,7 +478,7 @@ mod tests { quote_kind: quote.quote_kind.clone(), }; - assert_eq!(find(&mut db, &search_a).await.unwrap().unwrap(), quote); + assert_eq!(find(&mut db, &search_a).await.unwrap().unwrap(), quote,); search_a.quote_kind = QuoteKind::Standard; assert_eq!(find(&mut db, &search_a).await.unwrap(), None,); } @@ -552,39 +552,8 @@ mod tests { metadata: None, }; - // store quote with verified and metadata fields set to NULL - const QUERY: &str = r#" - INSERT INTO quotes ( - sell_token, - buy_token, - sell_amount, - buy_amount, - gas_amount, - gas_price, - sell_token_price, - order_kind, - expiration_timestamp, - quote_kind, - solver - ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) - RETURNING id - "#; - let (id,) = sqlx::query_as(QUERY) - .bind(quote.sell_token) - .bind(quote.buy_token) - .bind("e.sell_amount) - .bind("e.buy_amount) - .bind(quote.gas_amount) - .bind(quote.gas_price) - .bind(quote.sell_token_price) - .bind(quote.order_kind) - .bind(quote.expiration_timestamp) - .bind("e.quote_kind) - .bind(quote.solver) - .fetch_one(&mut db as &mut PgConnection) - .await - .unwrap(); + // save quote with verified and metadata fields stored as NULL + let id = save(&mut db, "e).await.unwrap(); // read back stored quote let stored_quote = get(&mut db, id).await.unwrap().unwrap(); From c25d4f91a579c5a831f45d1fdddc9b57d2e3e4c6 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 6 Dec 2024 11:11:09 +0100 Subject: [PATCH 53/70] Small updates --- crates/autopilot/src/database/auction.rs | 2 +- crates/database/src/quotes.rs | 52 ++-------------------- crates/orderbook/src/database/orders.rs | 13 +++--- crates/orderbook/src/database/quotes.rs | 2 +- crates/shared/src/event_storing_helpers.rs | 6 +-- 5 files changed, 14 insertions(+), 61 deletions(-) diff --git a/crates/autopilot/src/database/auction.rs b/crates/autopilot/src/database/auction.rs index ebc92fd7a5..606f0ce765 100644 --- a/crates/autopilot/src/database/auction.rs +++ b/crates/autopilot/src/database/auction.rs @@ -23,7 +23,7 @@ impl QuoteStoring for Postgres { .start_timer(); let mut ex = self.pool.acquire().await?; - let row = create_quote_row(&data)?; + let row = create_quote_row(data)?; let id = database::quotes::save(&mut ex, &row).await?; Ok(id) } diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 2de88a7229..8fc347da92 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -1,5 +1,5 @@ use { - crate::{orders::OrderKind, Address, PgTransaction}, + crate::{orders::OrderKind, Address}, bigdecimal::BigDecimal, sqlx::{ types::chrono::{DateTime, Utc}, @@ -150,52 +150,6 @@ WHERE expiration_timestamp < $1 .map(|_| ()) } -/// One row in the `quote_interactions` table. -#[derive(Clone, Debug, PartialEq, sqlx::FromRow)] -pub struct QuoteInteraction { - pub quote_id: QuoteId, - pub index: i32, - pub target: Address, - pub value: BigDecimal, - pub call_data: Vec, -} - -/// Stores interactions provided by the solver for quote. -pub async fn insert_quote_interaction( - ex: &mut PgConnection, - quote_interaction: &QuoteInteraction, -) -> Result<(), sqlx::Error> { - const QUERY: &str = r#" -INSERT INTO quote_interactions ( - quote_id, - index, - target, - value, - call_data -) -VALUES ($1, $2, $3, $4, $5) - "#; - sqlx::query(QUERY) - .bind(quote_interaction.quote_id) - .bind(quote_interaction.index) - .bind(quote_interaction.target) - .bind("e_interaction.value) - .bind("e_interaction.call_data) - .execute(ex) - .await?; - Ok(()) -} - -pub async fn insert_quote_interactions( - ex: &mut PgTransaction<'_>, - quote_interactions: &[QuoteInteraction], -) -> Result<(), sqlx::Error> { - for interaction in quote_interactions { - insert_quote_interaction(ex, interaction).await?; - } - Ok(()) -} - #[cfg(test)] mod tests { use { @@ -493,11 +447,11 @@ mod tests { let metadata: serde_json::Value = serde_json::from_str( r#"{ "interactions": [ { "target": "0102030405060708091011121314151617181920", - "value": 2.1, + "value": 1, "call_data": "0A0B0C102030" },{ "target": "FF02030405060708091011121314151617181920", - "value": 1.2, + "value": 2, "call_data": "FF0B0C102030" }] }"#, diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index 9c5f6b2c44..5a725ff368 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -235,12 +235,12 @@ async fn insert_order(order: &Order, ex: &mut PgConnection) -> Result<(), Insert } async fn insert_quote( - order: &Order, + uid: &OrderUid, quote: &Quote, ex: &mut PgConnection, ) -> Result<(), InsertionError> { - let dbquote = database::orders::Quote { - order_uid: ByteArray(order.metadata.uid.0), + let quote = database::orders::Quote { + order_uid: ByteArray(uid.0), gas_amount: quote.data.fee_parameters.gas_amount, gas_price: quote.data.fee_parameters.gas_price, sell_token_price: quote.data.fee_parameters.sell_token_price, @@ -256,8 +256,7 @@ async fn insert_quote( .map_err(|_| InsertionError::MetadataSerializationFailed)?, ), }; - - database::orders::insert_quote(ex, &dbquote) + database::orders::insert_quote(ex, "e) .await .map_err(InsertionError::DbError) } @@ -280,7 +279,7 @@ impl OrderStoring for Postgres { insert_order(&order, &mut ex).await?; if let Some(quote) = quote { - insert_quote(&order, "e, &mut ex).await?; + insert_quote(&order.metadata.uid, "e, &mut ex).await?; } Self::insert_order_app_data(&order, &mut ex).await?; @@ -340,7 +339,7 @@ impl OrderStoring for Postgres { .await?; insert_order(&new_order, ex).await?; if let Some(quote) = new_quote { - insert_quote(&new_order, "e, ex).await?; + insert_quote(&new_order.metadata.uid, "e, ex).await?; } Self::insert_order_app_data(&new_order, ex).await?; diff --git a/crates/orderbook/src/database/quotes.rs b/crates/orderbook/src/database/quotes.rs index 9223ba4b7e..8e2608e0c1 100644 --- a/crates/orderbook/src/database/quotes.rs +++ b/crates/orderbook/src/database/quotes.rs @@ -18,7 +18,7 @@ impl QuoteStoring for Postgres { .start_timer(); let mut ex = self.pool.acquire().await?; - let row = create_quote_row(&data)?; + let row = create_quote_row(data)?; let id = database::quotes::save(&mut ex, &row).await?; Ok(id) } diff --git a/crates/shared/src/event_storing_helpers.rs b/crates/shared/src/event_storing_helpers.rs index dbcaeb327a..c9f91a025a 100644 --- a/crates/shared/src/event_storing_helpers.rs +++ b/crates/shared/src/event_storing_helpers.rs @@ -17,7 +17,7 @@ use { number::conversions::u256_to_big_decimal, }; -pub fn create_quote_row(data: &QuoteData) -> Result { +pub fn create_quote_row(data: QuoteData) -> Result { Ok(DbQuote { id: Default::default(), sell_token: ByteArray(data.sell_token.0), @@ -29,12 +29,12 @@ pub fn create_quote_row(data: &QuoteData) -> Result { sell_token_price: data.fee_parameters.sell_token_price, order_kind: order_kind_into(data.kind), expiration_timestamp: data.expiration, - quote_kind: data.quote_kind.clone(), + quote_kind: data.quote_kind, solver: ByteArray(data.solver.0), verified: Some(data.verified), metadata: Some( QuoteMetadata { - interactions: data.interactions.clone(), + interactions: data.interactions, } .try_into()?, ), From 7aafd11130150474b5f1625b4750cd9af480f08a Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 6 Dec 2024 11:11:25 +0100 Subject: [PATCH 54/70] Updated HTTP error code --- crates/orderbook/src/api/post_order.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/orderbook/src/api/post_order.rs b/crates/orderbook/src/api/post_order.rs index 787f6e32a6..37a1a72f3b 100644 --- a/crates/orderbook/src/api/post_order.rs +++ b/crates/orderbook/src/api/post_order.rs @@ -267,7 +267,7 @@ impl IntoWarpReply for AddOrderError { "MetadataSerializationFailed", "quote metadata failed to serialize as json", ), - StatusCode::BAD_REQUEST, + StatusCode::INTERNAL_SERVER_ERROR, ), } } From b059d2e6fba02e63a9554049ce1daf4ed1a463af Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 6 Dec 2024 12:42:46 +0100 Subject: [PATCH 55/70] Added metadata field to QuoteData struct --- .../src/database/onchain_order_events/mod.rs | 17 ++----- crates/orderbook/src/database/orders.rs | 51 ++++++++++--------- crates/shared/src/event_storing_helpers.rs | 7 +-- crates/shared/src/order_quoting.rs | 49 +++++++++++------- 4 files changed, 61 insertions(+), 63 deletions(-) diff --git a/crates/autopilot/src/database/onchain_order_events/mod.rs b/crates/autopilot/src/database/onchain_order_events/mod.rs index 79ac4dd102..d9d94f6824 100644 --- a/crates/autopilot/src/database/onchain_order_events/mod.rs +++ b/crates/autopilot/src/database/onchain_order_events/mod.rs @@ -49,7 +49,7 @@ use { signing_scheme_into, }, event_handling::EventStoring, - order_quoting::{OrderQuoting, Quote, QuoteMetadata, QuoteSearchParameters}, + order_quoting::{OrderQuoting, Quote, QuoteSearchParameters}, order_validation::{ convert_signing_scheme_into_quote_signing_scheme, get_quote_and_check_fee, @@ -488,12 +488,7 @@ async fn parse_general_onchain_order_placement_data<'a>( buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), verified: Some(quote.data.verified), - metadata: Some( - QuoteMetadata { - interactions: quote.data.interactions.clone(), - } - .try_into()?, - ), + metadata: Some(quote.data.metadata.try_into()?), }), Err(err) => { let err_label = err.to_metrics_label(); @@ -1195,13 +1190,7 @@ mod test { buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), verified: Some(quote.data.verified), - metadata: Some( - QuoteMetadata { - interactions: quote.data.interactions, - } - .try_into() - .unwrap(), - ), + metadata: Some(quote.data.metadata.try_into().unwrap()), }; assert_eq!(result.1, vec![Some(expected_quote)]); assert_eq!( diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index 5a725ff368..a874025c8f 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -46,7 +46,7 @@ use { signing_scheme_into, }, fee::FeeParameters, - order_quoting::{Quote, QuoteMetadata}, + order_quoting::Quote, order_validation::{is_order_outside_market_price, Amounts, LimitOrderCounting}, }, sqlx::{types::BigDecimal, Connection, PgConnection}, @@ -105,11 +105,11 @@ impl OrderWithQuote { solver: ByteArray(quote.data.solver.0), verified: Some(quote.data.verified), metadata: Some( - QuoteMetadata { - interactions: quote.data.interactions.clone(), - } - .try_into() - .map_err(|_| AddOrderError::MetadataSerializationFailed)?, + quote + .data + .metadata + .try_into() + .map_err(|_| AddOrderError::MetadataSerializationFailed)?, ), }) }) @@ -249,11 +249,12 @@ async fn insert_quote( solver: ByteArray(quote.data.solver.0), verified: Some(quote.data.verified), metadata: Some( - QuoteMetadata { - interactions: quote.data.interactions.clone(), - } - .try_into() - .map_err(|_| InsertionError::MetadataSerializationFailed)?, + quote + .data + .metadata + .clone() + .try_into() + .map_err(|_| InsertionError::MetadataSerializationFailed)?, ), }; database::orders::insert_quote(ex, "e) @@ -691,7 +692,7 @@ mod tests { signature::{Signature, SigningScheme}, }, primitive_types::U256, - shared::order_quoting::QuoteData, + shared::order_quoting::{QuoteData, QuoteMetadata}, std::sync::atomic::{AtomicI64, Ordering}, }; @@ -1220,18 +1221,20 @@ mod tests { buy_amount: U256::from(2), data: QuoteData { verified: true, - interactions: vec![ - InteractionData { - target: H160([1; 20]), - value: U256::from(100), - call_data: vec![1, 20], - }, - InteractionData { - target: H160([2; 20]), - value: U256::from(10), - call_data: vec![2, 20], - }, - ], + metadata: QuoteMetadata { + interactions: vec![ + InteractionData { + target: H160([1; 20]), + value: U256::from(100), + call_data: vec![1, 20], + }, + InteractionData { + target: H160([2; 20]), + value: U256::from(10), + call_data: vec![2, 20], + }, + ], + }, ..Default::default() }, ..Default::default() diff --git a/crates/shared/src/event_storing_helpers.rs b/crates/shared/src/event_storing_helpers.rs index c9f91a025a..89c99da1a5 100644 --- a/crates/shared/src/event_storing_helpers.rs +++ b/crates/shared/src/event_storing_helpers.rs @@ -1,12 +1,7 @@ use { crate::{ db_order_conversions::order_kind_into, - order_quoting::{ - quote_kind_from_signing_scheme, - QuoteData, - QuoteMetadata, - QuoteSearchParameters, - }, + order_quoting::{quote_kind_from_signing_scheme, QuoteData, QuoteSearchParameters}, }, anyhow::Result, chrono::{DateTime, Utc}, diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index 8b4a16b9f7..719ee6a828 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -170,8 +170,8 @@ pub struct QuoteData { pub solver: H160, /// Were we able to verify that this quote is accurate? pub verified: bool, - /// Data provided by the solver in response to /quote request. - pub interactions: Vec, + /// Additional data associated with the quote. + pub metadata: QuoteMetadata, } impl TryFrom for QuoteData { @@ -197,7 +197,7 @@ impl TryFrom for QuoteData { // Even if the quote was verified at the time of creation // it might no longer be accurate. verified: false, - interactions: vec![], + metadata: row.metadata.try_into()?, }) } } @@ -445,7 +445,9 @@ impl OrderQuoter { quote_kind, solver: trade_estimate.solver, verified: trade_estimate.verified, - interactions: trade_estimate.interactions, + metadata: QuoteMetadata { + interactions: trade_estimate.interactions, + }, }; Ok(quote) @@ -634,8 +636,9 @@ pub fn quote_kind_from_signing_scheme(scheme: &QuoteSigningScheme) -> QuoteKind } /// Used to store in database any quote metadata. -#[derive(Debug, serde::Serialize)] +#[derive(Clone, Debug, Default, PartialEq, serde::Deserialize, serde::Serialize)] pub struct QuoteMetadata { + /// Data provided by the solver in response to /quote request. pub interactions: Vec, } @@ -647,6 +650,14 @@ impl TryInto for QuoteMetadata { } } +impl TryFrom for QuoteMetadata { + type Error = serde_json::Error; + + fn try_from(value: serde_json::Value) -> std::result::Result { + serde_json::from_value(value) + } +} + #[cfg(test)] mod tests { use { @@ -787,7 +798,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - interactions: vec![], + metadata: Default::default(), })) .returning(|_| Ok(1337)); @@ -824,7 +835,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - interactions: vec![], + metadata: Default::default(), }, sell_amount: 70.into(), buy_amount: 29.into(), @@ -925,7 +936,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - interactions: vec![], + metadata: Default::default(), })) .returning(|_| Ok(1337)); @@ -962,7 +973,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - interactions: vec![], + metadata: Default::default(), }, sell_amount: 100.into(), buy_amount: 42.into(), @@ -1058,7 +1069,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - interactions: vec![], + metadata: Default::default(), })) .returning(|_| Ok(1337)); @@ -1095,7 +1106,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - interactions: vec![], + metadata: Default::default(), }, sell_amount: 100.into(), buy_amount: 42.into(), @@ -1284,7 +1295,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - interactions: vec![], + metadata: Default::default(), })) }); @@ -1318,7 +1329,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - interactions: vec![], + metadata: Default::default(), }, sell_amount: 85.into(), // Allows for "out-of-price" buy amounts. This means that order @@ -1366,7 +1377,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - interactions: vec![], + metadata: Default::default(), })) }); @@ -1400,7 +1411,7 @@ mod tests { quote_kind: QuoteKind::Standard, solver: H160([1; 20]), verified: false, - interactions: vec![], + metadata: Default::default(), }, sell_amount: 100.into(), buy_amount: 42.into(), @@ -1448,8 +1459,8 @@ mod tests { expiration: now + chrono::Duration::seconds(10), quote_kind: QuoteKind::Standard, solver: H160([1; 20]), - verified: true, - interactions: vec![], + verified: false, + metadata: Default::default(), }, ))) }); @@ -1483,8 +1494,8 @@ mod tests { expiration: now + chrono::Duration::seconds(10), quote_kind: QuoteKind::Standard, solver: H160([1; 20]), - verified: true, - interactions: vec![], + verified: false, + metadata: Default::default(), }, sell_amount: 100.into(), buy_amount: 42.into(), From 171dadbd4e4a63e87c506f48fa13aaa81570a657 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 6 Dec 2024 12:49:24 +0100 Subject: [PATCH 56/70] Added NOT NULL constraint for new columns to quotes table --- crates/database/src/quotes.rs | 60 ++++--------------- crates/shared/src/event_storing_helpers.rs | 9 +-- ...dd_metadata_to_quotes_and_order_quotes.sql | 15 ++++- 3 files changed, 27 insertions(+), 57 deletions(-) diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index 8fc347da92..d5ab8a2614 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -34,8 +34,8 @@ pub struct Quote { pub expiration_timestamp: DateTime, pub quote_kind: QuoteKind, pub solver: Address, - pub verified: Option, // Null value support - pub metadata: Option, // Null value support + pub verified: bool, + pub metadata: serde_json::Value, } /// Stores the quote and returns the id. The id of the quote parameter is not @@ -187,8 +187,8 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Standard, solver: ByteArray([1; 20]), - verified: None, - metadata: None, + verified: false, + metadata: Default::default(), }; let id = save(&mut db, "e).await.unwrap(); quote.id = id; @@ -222,8 +222,8 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Standard, solver: ByteArray([1; 20]), - verified: None, - metadata: None, + verified: false, + metadata: Default::default(), }; let token_b = ByteArray([2; 20]); @@ -240,8 +240,8 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Standard, solver: ByteArray([2; 20]), - verified: None, - metadata: None, + verified: false, + metadata: Default::default(), }; // Save two measurements for token_a @@ -413,8 +413,8 @@ mod tests { expiration_timestamp: now, quote_kind: QuoteKind::Eip1271OnchainOrder, solver: ByteArray([1; 20]), - verified: None, - metadata: None, + verified: false, + metadata: Default::default(), }; let id = save(&mut db, "e).await.unwrap(); quote.id = id; @@ -471,47 +471,13 @@ mod tests { expiration_timestamp: low_precision_now(), quote_kind: QuoteKind::Standard, solver: ByteArray([1; 20]), - verified: None, - metadata: Some(metadata.clone()), + verified: false, + metadata: metadata.clone(), }; // store quote in database let id = save(&mut db, "e).await.unwrap(); let stored_quote = get(&mut db, id).await.unwrap().unwrap(); - assert_eq!(stored_quote.metadata.unwrap(), metadata); - } - - #[tokio::test] - #[ignore] - async fn postgres_get_quote_with_no_metadata_and_validity() { - // This test checks backward compatibility - let mut db = PgConnection::connect("postgresql://").await.unwrap(); - let mut db = db.begin().await.unwrap(); - crate::clear_DANGER_(&mut db).await.unwrap(); - - let quote = Quote { - id: Default::default(), - sell_token: ByteArray([1; 20]), - buy_token: ByteArray([2; 20]), - sell_amount: 3.into(), - buy_amount: 4.into(), - gas_amount: 5., - gas_price: 6., - sell_token_price: 7., - order_kind: OrderKind::Sell, - expiration_timestamp: low_precision_now(), - quote_kind: QuoteKind::Standard, - solver: ByteArray([1; 20]), - verified: None, - metadata: None, - }; - - // save quote with verified and metadata fields stored as NULL - let id = save(&mut db, "e).await.unwrap(); - - // read back stored quote - let stored_quote = get(&mut db, id).await.unwrap().unwrap(); - assert!(stored_quote.verified.is_none()); - assert!(stored_quote.metadata.is_none()); + assert_eq!(stored_quote.metadata, metadata); } } diff --git a/crates/shared/src/event_storing_helpers.rs b/crates/shared/src/event_storing_helpers.rs index 89c99da1a5..e3684df76f 100644 --- a/crates/shared/src/event_storing_helpers.rs +++ b/crates/shared/src/event_storing_helpers.rs @@ -26,13 +26,8 @@ pub fn create_quote_row(data: QuoteData) -> Result { expiration_timestamp: data.expiration, quote_kind: data.quote_kind, solver: ByteArray(data.solver.0), - verified: Some(data.verified), - metadata: Some( - QuoteMetadata { - interactions: data.interactions, - } - .try_into()?, - ), + verified: data.verified, + metadata: data.metadata.try_into()?, }) } diff --git a/database/sql/V075__add_metadata_to_quotes_and_order_quotes.sql b/database/sql/V075__add_metadata_to_quotes_and_order_quotes.sql index 426e540c9f..02a192b1f1 100644 --- a/database/sql/V075__add_metadata_to_quotes_and_order_quotes.sql +++ b/database/sql/V075__add_metadata_to_quotes_and_order_quotes.sql @@ -1,11 +1,20 @@ --- This migration script is reversible. +-- This migration script is not reversible. --- Step 1: Add two new columns to the quotes table +-- Step 1: Add two new columns to the quotes table ALTER TABLE quotes ADD COLUMN verified boolean, ADD COLUMN metadata json; --- Step 2: Add two new columns to the order_quotes table +-- Step 2: Update existing data with non-null values +UPDATE quotes SET verified = false, metadata = '{}'::json; + +-- Step 3: Add NOT NULL constraint for newly added columns +ALTER TABLE quotes + ALTER COLUMN verified SET NOT NULL, + ALTER COLUMN metadata SET NOT NULL; + + +-- Step 4: Add two new columns to the order_quotes table ALTER TABLE order_quotes ADD COLUMN verified boolean, ADD COLUMN metadata json; From 4761f8a1d5ddcfc67eaca96729485a010b5c634e Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Fri, 6 Dec 2024 12:50:02 +0100 Subject: [PATCH 57/70] Added sql migration V075 revert script --- ...revert_add_metadata_to_quotes_and_order_quotes.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 database/sql_revert/V075__revert_add_metadata_to_quotes_and_order_quotes.sql diff --git a/database/sql_revert/V075__revert_add_metadata_to_quotes_and_order_quotes.sql b/database/sql_revert/V075__revert_add_metadata_to_quotes_and_order_quotes.sql new file mode 100644 index 0000000000..5be236e8dc --- /dev/null +++ b/database/sql_revert/V075__revert_add_metadata_to_quotes_and_order_quotes.sql @@ -0,0 +1,11 @@ +-- This script reverts changes applied in V075__add_metadata_to_quotes_and_order_quotes.sql migration script. + +-- Step 1: Drop two columns from the quotes table +ALTER TABLE quotes + DROP COLUMN verified, + DROP COLUMN metadata; + +-- Step 2: Drop two columns from the order_quotes table +ALTER TABLE order_quotes + DROP COLUMN verified, + DROP COLUMN metadata; From e252f0e50705d85fd8e368c71d92d943142c442f Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Mon, 9 Dec 2024 22:32:43 +0100 Subject: [PATCH 58/70] Added support for versioning into QuoteMetadata type --- crates/orderbook/src/database/orders.rs | 7 +++--- crates/shared/src/order_quoting.rs | 30 ++++++++++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index a874025c8f..dbc0c89221 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -692,7 +692,7 @@ mod tests { signature::{Signature, SigningScheme}, }, primitive_types::U256, - shared::order_quoting::{QuoteData, QuoteMetadata}, + shared::order_quoting::{QuoteData, QuoteMetadataV1}, std::sync::atomic::{AtomicI64, Ordering}, }; @@ -1221,7 +1221,7 @@ mod tests { buy_amount: U256::from(2), data: QuoteData { verified: true, - metadata: QuoteMetadata { + metadata: QuoteMetadataV1 { interactions: vec![ InteractionData { target: H160([1; 20]), @@ -1234,7 +1234,8 @@ mod tests { call_data: vec![2, 20], }, ], - }, + } + .into(), ..Default::default() }, ..Default::default() diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index 719ee6a828..952d5bf94f 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -445,9 +445,10 @@ impl OrderQuoter { quote_kind, solver: trade_estimate.solver, verified: trade_estimate.verified, - metadata: QuoteMetadata { + metadata: QuoteMetadataV1 { interactions: trade_estimate.interactions, - }, + } + .into(), }; Ok(quote) @@ -636,10 +637,9 @@ pub fn quote_kind_from_signing_scheme(scheme: &QuoteSigningScheme) -> QuoteKind } /// Used to store in database any quote metadata. -#[derive(Clone, Debug, Default, PartialEq, serde::Deserialize, serde::Serialize)] -pub struct QuoteMetadata { - /// Data provided by the solver in response to /quote request. - pub interactions: Vec, +#[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)] +pub enum QuoteMetadata { + V1(QuoteMetadataV1), } impl TryInto for QuoteMetadata { @@ -658,6 +658,24 @@ impl TryFrom for QuoteMetadata { } } +impl Default for QuoteMetadata { + fn default() -> Self { + Self::V1(Default::default()) + } +} + +impl From for QuoteMetadata { + fn from(val: QuoteMetadataV1) -> Self { + QuoteMetadata::V1(val) + } +} + +#[derive(Clone, Debug, Default, PartialEq, serde::Deserialize, serde::Serialize)] +pub struct QuoteMetadataV1 { + /// Data provided by the solver in response to /quote request. + pub interactions: Vec, +} + #[cfg(test)] mod tests { use { From eaa203c4ed1629fc826e66050bae177373789fcb Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Mon, 9 Dec 2024 23:06:45 +0100 Subject: [PATCH 59/70] Small optimization --- crates/orderbook/src/orderbook.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/orderbook/src/orderbook.rs b/crates/orderbook/src/orderbook.rs index 063fa0aff5..3a55d2c423 100644 --- a/crates/orderbook/src/orderbook.rs +++ b/crates/orderbook/src/orderbook.rs @@ -254,17 +254,18 @@ impl Orderbook { self.replace_order(order, old_order, quote).await } else { let quote_id = quote.as_ref().and_then(|quote| quote.id); + let order_uid = order.metadata.uid; self.database .insert_order(&order, quote.clone()) .await .map_err(|err| AddOrderError::from_insertion(err, &order))?; Metrics::on_order_operation( - &OrderWithQuote::try_new(order.clone(), quote)?, + &OrderWithQuote::try_new(order, quote)?, OrderOperation::Created, ); - Ok((order.metadata.uid, quote_id)) + Ok((order_uid, quote_id)) } } @@ -407,6 +408,7 @@ impl Orderbook { } let quote_id = quote.as_ref().and_then(|quote| quote.id); + let order_uid = validated_new_order.metadata.uid; self.database .replace_order( @@ -418,11 +420,11 @@ impl Orderbook { .map_err(|err| AddOrderError::from_insertion(err, &validated_new_order))?; Metrics::on_order_operation(&old_order, OrderOperation::Cancelled); Metrics::on_order_operation( - &OrderWithQuote::try_new(validated_new_order.clone(), quote)?, + &OrderWithQuote::try_new(validated_new_order, quote)?, OrderOperation::Created, ); - Ok((validated_new_order.metadata.uid, quote_id)) + Ok((order_uid, quote_id)) } pub async fn get_order(&self, uid: &OrderUid) -> Result> { From d5c5b2173da52edd241ee3855262daf9f4d19cf4 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Mon, 9 Dec 2024 23:20:32 +0100 Subject: [PATCH 60/70] Improved error handling --- crates/orderbook/src/api/post_order.rs | 7 ++----- crates/orderbook/src/database/orders.rs | 16 +++++++--------- crates/orderbook/src/orderbook.rs | 8 ++++---- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/crates/orderbook/src/api/post_order.rs b/crates/orderbook/src/api/post_order.rs index 37a1a72f3b..e4d5bff45a 100644 --- a/crates/orderbook/src/api/post_order.rs +++ b/crates/orderbook/src/api/post_order.rs @@ -262,11 +262,8 @@ impl IntoWarpReply for AddOrderError { super::error("InvalidReplacement", err.to_string()), StatusCode::UNAUTHORIZED, ), - AddOrderError::MetadataSerializationFailed => reply::with_status( - super::error( - "MetadataSerializationFailed", - "quote metadata failed to serialize as json", - ), + AddOrderError::MetadataSerializationFailed(err) => reply::with_status( + super::error("MetadataSerializationFailed", err.to_string()), StatusCode::INTERNAL_SERVER_ERROR, ), } diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index dbc0c89221..47af351ba3 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -104,13 +104,11 @@ impl OrderWithQuote { buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), verified: Some(quote.data.verified), - metadata: Some( - quote - .data - .metadata - .try_into() - .map_err(|_| AddOrderError::MetadataSerializationFailed)?, - ), + metadata: Some(quote.data.metadata.try_into().map_err( + |e: serde_json::Error| { + AddOrderError::MetadataSerializationFailed(e.into()) + }, + )?), }) }) .transpose()?, @@ -125,7 +123,7 @@ pub enum InsertionError { DbError(sqlx::Error), /// Full app data to be inserted doesn't match existing. AppDataMismatch(Vec), - MetadataSerializationFailed, + MetadataSerializationFailed(serde_json::Error), } impl From for InsertionError { @@ -254,7 +252,7 @@ async fn insert_quote( .metadata .clone() .try_into() - .map_err(|_| InsertionError::MetadataSerializationFailed)?, + .map_err(InsertionError::MetadataSerializationFailed)?, ), }; database::orders::insert_quote(ex, "e) diff --git a/crates/orderbook/src/orderbook.rs b/crates/orderbook/src/orderbook.rs index 3a55d2c423..a28afb6f89 100644 --- a/crates/orderbook/src/orderbook.rs +++ b/crates/orderbook/src/orderbook.rs @@ -141,8 +141,8 @@ pub enum AddOrderError { provided: String, existing: String, }, - #[error("quote metadata failed to serialize as json")] - MetadataSerializationFailed, + #[error("quote metadata failed to serialize as json, error: {0}")] + MetadataSerializationFailed(#[source] anyhow::Error), } impl AddOrderError { @@ -163,8 +163,8 @@ impl AddOrderError { s.into_owned() }, }, - InsertionError::MetadataSerializationFailed => { - AddOrderError::MetadataSerializationFailed + InsertionError::MetadataSerializationFailed(err) => { + AddOrderError::MetadataSerializationFailed(err.into()) } } } From 978e436e0e1d7b065c012afc58d1d641ea964655 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Tue, 10 Dec 2024 01:11:17 +0100 Subject: [PATCH 61/70] Added e2e test --- crates/e2e/tests/e2e/database.rs | 11 ++ crates/e2e/tests/e2e/main.rs | 1 + .../e2e/tests/e2e/place_order_with_quote.rs | 106 ++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 crates/e2e/tests/e2e/place_order_with_quote.rs diff --git a/crates/e2e/tests/e2e/database.rs b/crates/e2e/tests/e2e/database.rs index 382b74d685..5807361f05 100644 --- a/crates/e2e/tests/e2e/database.rs +++ b/crates/e2e/tests/e2e/database.rs @@ -19,6 +19,17 @@ pub async fn events_of_order(db: &Db, uid: &OrderUid) -> Vec Option<(serde_json::Value,)> { + const QUERY: &str = "SELECT metadata FROM quotes WHERE id = $1"; + let mut db = db.acquire().await.unwrap(); + sqlx::query_as(QUERY) + .bind(quote_id) + .fetch_optional(db.deref_mut()) + .await + .unwrap() +} + #[allow(dead_code)] #[derive(Clone, Debug, sqlx::FromRow)] pub struct AuctionTransaction { diff --git a/crates/e2e/tests/e2e/main.rs b/crates/e2e/tests/e2e/main.rs index 37534b4b70..e525829381 100644 --- a/crates/e2e/tests/e2e/main.rs +++ b/crates/e2e/tests/e2e/main.rs @@ -21,6 +21,7 @@ mod order_cancellation; mod partial_fill; mod partially_fillable_balance; mod partially_fillable_pool; +mod place_order_with_quote; mod protocol_fee; mod quote_verification; mod quoting; diff --git a/crates/e2e/tests/e2e/place_order_with_quote.rs b/crates/e2e/tests/e2e/place_order_with_quote.rs new file mode 100644 index 0000000000..57bfa542a9 --- /dev/null +++ b/crates/e2e/tests/e2e/place_order_with_quote.rs @@ -0,0 +1,106 @@ +use { + driver::domain::eth::NonZeroU256, + e2e::{nodes::local_node::TestNodeApi, setup::*, tx, tx_value}, + ethcontract::U256, + model::{ + order::{OrderCreation, OrderKind}, + quote::{OrderQuoteRequest, OrderQuoteSide, SellAmount}, + signature::EcdsaSigningScheme, + }, + secp256k1::SecretKey, + shared::ethrpc::Web3, + std::ops::DerefMut, + web3::signing::SecretKeyRef, +}; + +#[tokio::test] +#[ignore] +async fn local_node_test() { + run_test(place_order_from_quote).await; +} + +async fn place_order_from_quote(web3: Web3) { + let mut onchain = OnchainComponents::deploy(web3.clone()).await; + + let [solver] = onchain.make_solvers(to_wei(10)).await; + let [trader] = onchain.make_accounts(to_wei(10)).await; + let [token] = onchain + .deploy_tokens_with_weth_uni_v2_pools(to_wei(1_000), to_wei(1_000)) + .await; + + tx!( + trader.account(), + onchain + .contracts() + .weth + .approve(onchain.contracts().allowance, to_wei(3)) + ); + tx_value!( + trader.account(), + to_wei(3), + onchain.contracts().weth.deposit() + ); + + tracing::info!("Starting services."); + let services = Services::new(&onchain).await; + services.start_protocol(solver.clone()).await; + + // Disable auto-mine so we don't accidentally mine a settlement + web3.api::>() + .disable_automine() + .await + .expect("Must be able to disable automine"); + + tracing::info!("Quoting"); + let quote_request = OrderQuoteRequest { + from: trader.address(), + sell_token: onchain.contracts().weth.address(), + buy_token: token.address(), + side: OrderQuoteSide::Sell { + sell_amount: SellAmount::BeforeFee { + value: NonZeroU256::try_from(to_wei(1)).unwrap(), + }, + }, + ..Default::default() + }; + let quote_response = services.submit_quote("e_request).await.unwrap(); + tracing::debug!(?quote_response); + assert!(quote_response.id.is_some()); + + let quote_metadata = + crate::database::quote_metadata(services.db(), quote_response.id.unwrap()).await; + assert!(quote_metadata.is_some()); + tracing::debug!(?quote_metadata); + + tracing::info!("Placing order"); + let balance = token.balance_of(trader.address()).call().await.unwrap(); + assert_eq!(balance, 0.into()); + let order = OrderCreation { + quote_id: quote_response.id, + sell_token: onchain.contracts().weth.address(), + sell_amount: quote_response.quote.sell_amount, + buy_token: token.address(), + buy_amount: quote_response.quote.buy_amount, + valid_to: model::time::now_in_epoch_seconds() + 300, + kind: OrderKind::Sell, + ..Default::default() + } + .sign( + EcdsaSigningScheme::Eip712, + &onchain.contracts().domain_separator, + SecretKeyRef::from(&SecretKey::from_slice(trader.private_key()).unwrap()), + ); + let order_uid = services.create_order(&order).await.unwrap(); + + tracing::info!("Order quote verification"); + let order_quote = database::orders::read_quote( + services.db().acquire().await.unwrap().deref_mut(), + &database::byte_array::ByteArray(order_uid.0), + ) + .await + .unwrap(); + assert!(order_quote.is_some()); + let order_quote_metadata = order_quote.unwrap().metadata; + assert!(order_quote_metadata.is_some()); + assert_eq!(quote_metadata.unwrap().0, order_quote_metadata.unwrap()); +} From 331e70b3ef6659a9b67e846a42c5e8b77ff340d3 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Tue, 10 Dec 2024 12:02:31 +0100 Subject: [PATCH 62/70] Changed constraints of columns in order_quotes table --- .../src/database/onchain_order_events/mod.rs | 8 ++--- crates/database/src/orders.rs | 20 ++++++------ .../e2e/tests/e2e/place_order_with_quote.rs | 7 ++-- crates/orderbook/src/database/orders.rs | 32 ++++++++++--------- ...dd_metadata_to_quotes_and_order_quotes.sql | 11 ++++++- 5 files changed, 44 insertions(+), 34 deletions(-) diff --git a/crates/autopilot/src/database/onchain_order_events/mod.rs b/crates/autopilot/src/database/onchain_order_events/mod.rs index d9d94f6824..9ae6051493 100644 --- a/crates/autopilot/src/database/onchain_order_events/mod.rs +++ b/crates/autopilot/src/database/onchain_order_events/mod.rs @@ -487,8 +487,8 @@ async fn parse_general_onchain_order_placement_data<'a>( sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - verified: Some(quote.data.verified), - metadata: Some(quote.data.metadata.try_into()?), + verified: quote.data.verified, + metadata: quote.data.metadata.try_into()?, }), Err(err) => { let err_label = err.to_metrics_label(); @@ -1189,8 +1189,8 @@ mod test { sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - verified: Some(quote.data.verified), - metadata: Some(quote.data.metadata.try_into().unwrap()), + verified: quote.data.verified, + metadata: quote.data.metadata.try_into().unwrap(), }; assert_eq!(result.1, vec![Some(expected_quote)]); assert_eq!( diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 9cc45bab09..06a2e4be0d 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -329,8 +329,8 @@ pub struct Quote { pub sell_amount: BigDecimal, pub buy_amount: BigDecimal, pub solver: Address, - pub verified: Option, // Null value support - pub metadata: Option, // Null value support + pub verified: bool, + pub metadata: serde_json::Value, } pub async fn insert_quotes(ex: &mut PgConnection, quotes: &[Quote]) -> Result<(), sqlx::Error> { @@ -1213,8 +1213,8 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), - verified: None, - metadata: None, + verified: false, + metadata: Default::default(), }; insert_quote(&mut db, "e).await.unwrap(); insert_quote_and_update_on_conflict(&mut db, "e) @@ -1289,8 +1289,8 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), - verified: Some(true), - metadata: Some(metadata), + verified: true, + metadata, }; insert_quote(&mut db, "e).await.unwrap(); let quote_ = read_quote(&mut db, "e.order_uid) @@ -1317,8 +1317,8 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), - verified: None, - metadata: None, + verified: false, + metadata: Default::default(), }; insert_quote(&mut db, "e).await.unwrap(); let order_with_quote = single_full_order_with_quote(&mut db, "e.order_uid) @@ -2191,8 +2191,8 @@ mod tests { sell_amount: 4.into(), buy_amount: 5.into(), solver: ByteArray([1; 20]), - verified: None, - metadata: None, + verified: false, + metadata: Default::default(), }; // insert quote with verified and metadata fields stored as NULL diff --git a/crates/e2e/tests/e2e/place_order_with_quote.rs b/crates/e2e/tests/e2e/place_order_with_quote.rs index 57bfa542a9..35aaf0a948 100644 --- a/crates/e2e/tests/e2e/place_order_with_quote.rs +++ b/crates/e2e/tests/e2e/place_order_with_quote.rs @@ -16,10 +16,10 @@ use { #[tokio::test] #[ignore] async fn local_node_test() { - run_test(place_order_from_quote).await; + run_test(place_order_with_quote).await; } -async fn place_order_from_quote(web3: Web3) { +async fn place_order_with_quote(web3: Web3) { let mut onchain = OnchainComponents::deploy(web3.clone()).await; let [solver] = onchain.make_solvers(to_wei(10)).await; @@ -101,6 +101,5 @@ async fn place_order_from_quote(web3: Web3) { .unwrap(); assert!(order_quote.is_some()); let order_quote_metadata = order_quote.unwrap().metadata; - assert!(order_quote_metadata.is_some()); - assert_eq!(quote_metadata.unwrap().0, order_quote_metadata.unwrap()); + assert_eq!(quote_metadata.unwrap().0, order_quote_metadata); } diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index 47af351ba3..dff26428f1 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -103,12 +103,12 @@ impl OrderWithQuote { sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - verified: Some(quote.data.verified), - metadata: Some(quote.data.metadata.try_into().map_err( + verified: quote.data.verified, + metadata: quote.data.metadata.try_into().map_err( |e: serde_json::Error| { AddOrderError::MetadataSerializationFailed(e.into()) }, - )?), + )?, }) }) .transpose()?, @@ -245,15 +245,13 @@ async fn insert_quote( sell_amount: u256_to_big_decimal("e.sell_amount), buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), - verified: Some(quote.data.verified), - metadata: Some( - quote - .data - .metadata - .clone() - .try_into() - .map_err(InsertionError::MetadataSerializationFailed)?, - ), + verified: quote.data.verified, + metadata: quote + .data + .metadata + .clone() + .try_into() + .map_err(InsertionError::MetadataSerializationFailed)?, }; database::orders::insert_quote(ex, "e) .await @@ -382,6 +380,8 @@ impl OrderStoring for Postgres { order_with_quote.quote_gas_amount, order_with_quote.quote_gas_price, order_with_quote.quote_sell_token_price, + order_with_quote.quote_verified, + order_with_quote.quote_metadata, order_with_quote.solver, ) { ( @@ -390,6 +390,8 @@ impl OrderStoring for Postgres { Some(gas_amount), Some(gas_price), Some(sell_token_price), + Some(verified), + Some(metadata), Some(solver), ) => Some(orders::Quote { order_uid: order_with_quote.full_order.uid, @@ -399,8 +401,8 @@ impl OrderStoring for Postgres { sell_amount, buy_amount, solver, - verified: order_with_quote.quote_verified, - metadata: order_with_quote.quote_metadata, + verified, + metadata, }), _ => None, }; @@ -1242,6 +1244,6 @@ mod tests { let single_order_with_quote = db.single_order_with_quote(&uid).await.unwrap().unwrap(); assert_eq!(single_order_with_quote.order, order); - assert!(single_order_with_quote.quote.unwrap().verified.unwrap()); + assert!(single_order_with_quote.quote.unwrap().verified); } } diff --git a/database/sql/V075__add_metadata_to_quotes_and_order_quotes.sql b/database/sql/V075__add_metadata_to_quotes_and_order_quotes.sql index 02a192b1f1..363c098f90 100644 --- a/database/sql/V075__add_metadata_to_quotes_and_order_quotes.sql +++ b/database/sql/V075__add_metadata_to_quotes_and_order_quotes.sql @@ -8,7 +8,7 @@ ALTER TABLE quotes -- Step 2: Update existing data with non-null values UPDATE quotes SET verified = false, metadata = '{}'::json; --- Step 3: Add NOT NULL constraint for newly added columns +-- Step 3: Add NOT NULL constraint to newly added columns ALTER TABLE quotes ALTER COLUMN verified SET NOT NULL, ALTER COLUMN metadata SET NOT NULL; @@ -18,3 +18,12 @@ ALTER TABLE quotes ALTER TABLE order_quotes ADD COLUMN verified boolean, ADD COLUMN metadata json; + +-- Step 5: Update existing data with non-null values +UPDATE order_quotes SET verified = false, metadata = '{}'::json; + +-- Step 6: Add NOT NULL constraint to newly added columns +ALTER TABLE order_quotes + ALTER COLUMN verified SET NOT NULL, + ALTER COLUMN metadata SET NOT NULL; + From 11b74d3539a70a11dd4b3c89ec203f117e28c6e1 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Tue, 10 Dec 2024 12:04:00 +0100 Subject: [PATCH 63/70] Updated revert sql script --- database/README.md | 8 ++++---- ... => U075__add_metadata_to_quotes_and_order_quotes.sql} | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) rename database/sql_revert/{V075__revert_add_metadata_to_quotes_and_order_quotes.sql => U075__add_metadata_to_quotes_and_order_quotes.sql} (73%) diff --git a/database/README.md b/database/README.md index dac697e8b7..abb2c53b29 100644 --- a/database/README.md +++ b/database/README.md @@ -238,8 +238,8 @@ Quotes that an order was created with. These quotes get stored persistently and sell\_amount | numeric | not null | sell\_amount of the quote used to create the order with buy\_amount | numeric | not null | buy\_amount of the quote used to create the order with solver | bytea | not null | public address of the solver that provided this quote - verified | boolean | | information if quote was verified - metadata | json | | additional data associated with the quote in json format: interactions + verified | boolean | not null | information if quote was verified + metadata | json | not null | additional data associated with the quote in json format Indexes: - PRIMARY KEY: btree(`order_uid`) @@ -341,8 +341,8 @@ Stores quotes in order to determine whether it makes sense to allow a user to cr id | bigint | not null | unique identifier of this quote quote\_kind | [enum](#quotekind) | not null | quotekind for which this quote is considered valid solver | bytea | not null | public address of the solver that provided this quote - verified | boolean | | information if quote was verified - metadata | json | | additional data associated with the quote in json format: interactions + verified | boolean | not null | information if quote was verified + metadata | json | not null | additional data associated with the quote in json format Indexes: - PRIMARY KEY: btree(`id`) diff --git a/database/sql_revert/V075__revert_add_metadata_to_quotes_and_order_quotes.sql b/database/sql_revert/U075__add_metadata_to_quotes_and_order_quotes.sql similarity index 73% rename from database/sql_revert/V075__revert_add_metadata_to_quotes_and_order_quotes.sql rename to database/sql_revert/U075__add_metadata_to_quotes_and_order_quotes.sql index 5be236e8dc..e0f15152ff 100644 --- a/database/sql_revert/V075__revert_add_metadata_to_quotes_and_order_quotes.sql +++ b/database/sql_revert/U075__add_metadata_to_quotes_and_order_quotes.sql @@ -1,11 +1,11 @@ -- This script reverts changes applied in V075__add_metadata_to_quotes_and_order_quotes.sql migration script. --- Step 1: Drop two columns from the quotes table -ALTER TABLE quotes +-- Step 1: Drop two columns from the quotes table +ALTER TABLE quotes DROP COLUMN verified, DROP COLUMN metadata; -- Step 2: Drop two columns from the order_quotes table -ALTER TABLE order_quotes +ALTER TABLE order_quotes DROP COLUMN verified, DROP COLUMN metadata; From 35f15422c8fe5a4acab87b195f2ea3ef63f59360 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Tue, 10 Dec 2024 12:26:44 +0100 Subject: [PATCH 64/70] Moved undo sql script to sql folder --- .../U075__add_metadata_to_quotes_and_order_quotes.sql | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename database/{sql_revert => sql}/U075__add_metadata_to_quotes_and_order_quotes.sql (100%) diff --git a/database/sql_revert/U075__add_metadata_to_quotes_and_order_quotes.sql b/database/sql/U075__add_metadata_to_quotes_and_order_quotes.sql similarity index 100% rename from database/sql_revert/U075__add_metadata_to_quotes_and_order_quotes.sql rename to database/sql/U075__add_metadata_to_quotes_and_order_quotes.sql From c990aabbcd9e6feb0c53ca88ea90c9cbbc62c79a Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Tue, 10 Dec 2024 13:25:15 +0100 Subject: [PATCH 65/70] Moved interactions to separate struct in trade_finding and price_estimation --- crates/e2e/tests/e2e/quote_verification.rs | 14 ++++++---- crates/shared/src/order_quoting.rs | 12 ++++---- crates/shared/src/price_estimation/mod.rs | 11 +++----- .../shared/src/price_estimation/native/mod.rs | 2 +- .../shared/src/price_estimation/sanitized.rs | 28 +++++++++---------- .../src/price_estimation/trade_finder.rs | 2 +- .../src/price_estimation/trade_verifier.rs | 15 +++++++--- crates/shared/src/trade_finding/external.rs | 5 +++- crates/shared/src/trade_finding/mod.rs | 8 +++++- 9 files changed, 56 insertions(+), 41 deletions(-) diff --git a/crates/e2e/tests/e2e/quote_verification.rs b/crates/e2e/tests/e2e/quote_verification.rs index 0daaaaff6c..3b50d7ecc3 100644 --- a/crates/e2e/tests/e2e/quote_verification.rs +++ b/crates/e2e/tests/e2e/quote_verification.rs @@ -20,7 +20,7 @@ use { Estimate, Verification, }, - trade_finding::{Interaction, LegacyTrade, TradeKind}, + trade_finding::{Interaction, LegacyTrade, QuoteExecution, TradeKind}, }, std::{str::FromStr, sync::Arc}, }; @@ -176,11 +176,13 @@ async fn test_bypass_verification_for_rfq_quotes(web3: Web3) { gas: 225000, solver: H160::from_str("0xe3067c7c27c1038de4e8ad95a83b927d23dfbd99").unwrap(), verified: true, - interactions: vec![InteractionData { - target: H160::from_str("0xdef1c0ded9bec7f1a1670819833240f027b25eff").unwrap(), - value: 0.into(), - call_data: hex::decode("aa77476c000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000002260fac5e5542a773aa44fbcfedf7c193bc2c599000000000000000000000000000000000000000000000000e357b42c3a9d8ccf0000000000000000000000000000000000000000000000000000000004d0e79e000000000000000000000000a69babef1ca67a37ffaf7a485dfff3382056e78c0000000000000000000000009008d19f58aabd9ed0d60971565aa8510560ab41000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000066360af101ffffffffffffffffffffffffffffffffffffff0f3f47f166360a8d0000003f0000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000001c66b3383f287dd9c85ad90e7c5a576ea4ba1bdf5a001d794a9afa379e6b2517b47e487a1aef32e75af432cbdbd301ada42754eaeac21ec4ca744afd92732f47540000000000000000000000000000000000000000000000000000000004d0c80f").unwrap() - }], + execution: QuoteExecution { + interactions: vec![InteractionData { + target: H160::from_str("0xdef1c0ded9bec7f1a1670819833240f027b25eff").unwrap(), + value: 0.into(), + call_data: hex::decode("aa77476c000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc20000000000000000000000002260fac5e5542a773aa44fbcfedf7c193bc2c599000000000000000000000000000000000000000000000000e357b42c3a9d8ccf0000000000000000000000000000000000000000000000000000000004d0e79e000000000000000000000000a69babef1ca67a37ffaf7a485dfff3382056e78c0000000000000000000000009008d19f58aabd9ed0d60971565aa8510560ab41000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000066360af101ffffffffffffffffffffffffffffffffffffff0f3f47f166360a8d0000003f0000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000000000000000000000000000000000001c66b3383f287dd9c85ad90e7c5a576ea4ba1bdf5a001d794a9afa379e6b2517b47e487a1aef32e75af432cbdbd301ada42754eaeac21ec4ca744afd92732f47540000000000000000000000000000000000000000000000000000000004d0c80f").unwrap() + }], + }, }; // `tx_origin: 0x0000` is currently used to bypass quote verification due to an diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index 952d5bf94f..e94735c9d1 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -446,7 +446,7 @@ impl OrderQuoter { solver: trade_estimate.solver, verified: trade_estimate.verified, metadata: QuoteMetadataV1 { - interactions: trade_estimate.interactions, + interactions: trade_estimate.execution.interactions, } .into(), }; @@ -774,7 +774,7 @@ mod tests { gas: 3, solver: H160([1; 20]), verified: false, - interactions: vec![], + execution: Default::default(), }) } .boxed() @@ -912,7 +912,7 @@ mod tests { gas: 3, solver: H160([1; 20]), verified: false, - interactions: vec![], + execution: Default::default(), }) } .boxed() @@ -1045,7 +1045,7 @@ mod tests { gas: 3, solver: H160([1; 20]), verified: false, - interactions: vec![], + execution: Default::default(), }) } .boxed() @@ -1164,7 +1164,7 @@ mod tests { gas: 200, solver: H160([1; 20]), verified: false, - interactions: vec![], + execution: Default::default(), }) } .boxed() @@ -1236,7 +1236,7 @@ mod tests { gas: 200, solver: H160([1; 20]), verified: false, - interactions: vec![], + execution: Default::default(), }) } .boxed() diff --git a/crates/shared/src/price_estimation/mod.rs b/crates/shared/src/price_estimation/mod.rs index 499bf79eda..c077f83906 100644 --- a/crates/shared/src/price_estimation/mod.rs +++ b/crates/shared/src/price_estimation/mod.rs @@ -2,17 +2,14 @@ use { self::trade_verifier::balance_overrides, crate::{ arguments::{display_option, display_secret_option, ExternalSolver}, - trade_finding::Interaction, + trade_finding::{Interaction, QuoteExecution}, }, anyhow::Result, bigdecimal::BigDecimal, ethcontract::{H160, U256}, futures::future::BoxFuture, itertools::Itertools, - model::{ - interaction::InteractionData, - order::{BuyTokenDestination, OrderKind, SellTokenSource}, - }, + model::order::{BuyTokenDestination, OrderKind, SellTokenSource}, number::nonzero::U256 as NonZeroU256, rate_limit::{RateLimiter, Strategy}, reqwest::Url, @@ -473,8 +470,8 @@ pub struct Estimate { pub solver: H160, /// Did we verify the correctness of this estimate's properties? pub verified: bool, - /// Interactions provided by the solver in response to /quote request. - pub interactions: Vec, + /// Data associated with this estimation. + pub execution: QuoteExecution, } impl Estimate { diff --git a/crates/shared/src/price_estimation/native/mod.rs b/crates/shared/src/price_estimation/native/mod.rs index 6901ce3072..31ca840575 100644 --- a/crates/shared/src/price_estimation/native/mod.rs +++ b/crates/shared/src/price_estimation/native/mod.rs @@ -130,7 +130,7 @@ mod tests { gas: 0, solver: H160([1; 20]), verified: false, - interactions: vec![], + execution: Default::default(), }) } .boxed() diff --git a/crates/shared/src/price_estimation/sanitized.rs b/crates/shared/src/price_estimation/sanitized.rs index 7181b28baf..926703a4b8 100644 --- a/crates/shared/src/price_estimation/sanitized.rs +++ b/crates/shared/src/price_estimation/sanitized.rs @@ -67,7 +67,7 @@ impl PriceEstimating for SanitizedPriceEstimator { gas: 0, solver: Default::default(), verified: true, - interactions: vec![], + execution: Default::default(), }; tracing::debug!(?query, ?estimation, "generate trivial price estimation"); return Ok(estimation); @@ -80,7 +80,7 @@ impl PriceEstimating for SanitizedPriceEstimator { gas: GAS_PER_WETH_UNWRAP, solver: Default::default(), verified: true, - interactions: vec![], + execution: Default::default(), }; tracing::debug!(?query, ?estimation, "generate trivial unwrap estimation"); return Ok(estimation); @@ -93,7 +93,7 @@ impl PriceEstimating for SanitizedPriceEstimator { gas: GAS_PER_WETH_WRAP, solver: Default::default(), verified: true, - interactions: vec![], + execution: Default::default(), }; tracing::debug!(?query, ?estimation, "generate trivial wrap estimation"); return Ok(estimation); @@ -188,7 +188,7 @@ mod tests { gas: 100, solver: Default::default(), verified: false, - interactions: vec![], + execution: Default::default(), }), ), // `sanitized_estimator` will replace `buy_token` with `native_token` before querying @@ -210,7 +210,7 @@ mod tests { gas: GAS_PER_WETH_UNWRAP + 100, solver: Default::default(), verified: false, - interactions: vec![], + execution: Default::default(), }), ), // Will cause buffer overflow of gas price in `sanitized_estimator`. @@ -246,7 +246,7 @@ mod tests { gas: GAS_PER_WETH_WRAP + 100, solver: Default::default(), verified: false, - interactions: vec![], + execution: Default::default(), }), ), // Can be estimated by `sanitized_estimator` because `buy_token` and `sell_token` are @@ -265,7 +265,7 @@ mod tests { gas: 0, solver: Default::default(), verified: true, - interactions: vec![], + execution: Default::default(), }), ), // Can be estimated by `sanitized_estimator` because both tokens are the native token. @@ -283,7 +283,7 @@ mod tests { gas: 0, solver: Default::default(), verified: true, - interactions: vec![], + execution: Default::default(), }), ), // Can be estimated by `sanitized_estimator` because it is a native token unwrap. @@ -302,7 +302,7 @@ mod tests { gas: GAS_PER_WETH_UNWRAP, solver: Default::default(), verified: true, - interactions: vec![], + execution: Default::default(), }), ), // Can be estimated by `sanitized_estimator` because it is a native token wrap. @@ -321,7 +321,7 @@ mod tests { gas: GAS_PER_WETH_WRAP, solver: Default::default(), verified: true, - interactions: vec![], + execution: Default::default(), }), ), // Will throw `UnsupportedToken` error in `sanitized_estimator`. @@ -386,7 +386,7 @@ mod tests { gas: 100, solver: Default::default(), verified: false, - interactions: vec![], + execution: Default::default(), }) } .boxed() @@ -402,7 +402,7 @@ mod tests { gas: 100, solver: Default::default(), verified: false, - interactions: vec![], + execution: Default::default(), }) } .boxed() @@ -418,7 +418,7 @@ mod tests { gas: u64::MAX, solver: Default::default(), verified: false, - interactions: vec![], + execution: Default::default(), }) } .boxed() @@ -434,7 +434,7 @@ mod tests { gas: 100, solver: Default::default(), verified: false, - interactions: vec![], + execution: Default::default(), }) } .boxed() diff --git a/crates/shared/src/price_estimation/trade_finder.rs b/crates/shared/src/price_estimation/trade_finder.rs index d35d6751d3..dcfd7a8d30 100644 --- a/crates/shared/src/price_estimation/trade_finder.rs +++ b/crates/shared/src/price_estimation/trade_finder.rs @@ -89,7 +89,7 @@ impl Inner { gas: quote.gas_estimate, solver: quote.solver, verified: false, - interactions: quote.interactions, + execution: quote.execution, }) } } diff --git a/crates/shared/src/price_estimation/trade_verifier.rs b/crates/shared/src/price_estimation/trade_verifier.rs index 930d135752..7098695c91 100644 --- a/crates/shared/src/price_estimation/trade_verifier.rs +++ b/crates/shared/src/price_estimation/trade_verifier.rs @@ -9,9 +9,10 @@ use { encoded_settlement::{encode_trade, EncodedSettlement, EncodedTrade}, interaction::EncodedInteraction, trade_finding::{ - external::{dto, dto::JitOrder}, + external::dto::{self, JitOrder}, map_interactions_data, Interaction, + QuoteExecution, TradeKind, }, }, @@ -217,7 +218,9 @@ impl TradeVerifier { gas: trade.gas_estimate().context("no gas estimate")?, solver: trade.solver(), verified: true, - interactions: map_interactions_data(&trade.interactions()), + execution: QuoteExecution { + interactions: map_interactions_data(&trade.interactions()), + }, }; tracing::warn!( ?estimate, @@ -426,7 +429,9 @@ impl TradeVerifying for TradeVerifier { gas, solver: trade.solver(), verified: false, - interactions: map_interactions_data(&trade.interactions()), + execution: QuoteExecution { + interactions: map_interactions_data(&trade.interactions()), + }, }; tracing::warn!( ?err, @@ -773,7 +778,9 @@ fn ensure_quote_accuracy( gas: summary.gas_used.as_u64(), solver: trade.solver(), verified: true, - interactions: map_interactions_data(&trade.interactions()), + execution: QuoteExecution { + interactions: map_interactions_data(&trade.interactions()), + }, }) } diff --git a/crates/shared/src/trade_finding/external.rs b/crates/shared/src/trade_finding/external.rs index 7ad10a07d4..95e820eb70 100644 --- a/crates/shared/src/trade_finding/external.rs +++ b/crates/shared/src/trade_finding/external.rs @@ -9,6 +9,7 @@ use { Interaction, LegacyTrade, Quote, + QuoteExecution, Trade, TradeError, TradeFinding, @@ -221,7 +222,9 @@ impl TradeFinding for ExternalTradeFinder { .map_err(TradeError::Other)?, gas_estimate, solver: trade.solver(), - interactions: map_interactions_data(&trade.interactions()), + execution: QuoteExecution { + interactions: map_interactions_data(&trade.interactions()), + }, }) } diff --git a/crates/shared/src/trade_finding/mod.rs b/crates/shared/src/trade_finding/mod.rs index 698f457ed3..0c0cc9a5ce 100644 --- a/crates/shared/src/trade_finding/mod.rs +++ b/crates/shared/src/trade_finding/mod.rs @@ -15,7 +15,7 @@ use { model::{interaction::InteractionData, order::OrderKind}, num::CheckedDiv, number::conversions::big_rational_to_u256, - serde::Serialize, + serde::{Deserialize, Serialize}, std::{collections::HashMap, ops::Mul}, thiserror::Error, }; @@ -36,6 +36,12 @@ pub struct Quote { pub out_amount: U256, pub gas_estimate: u64, pub solver: H160, + pub execution: QuoteExecution, +} + +/// Quote execution metadata. +#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize)] +pub struct QuoteExecution { pub interactions: Vec, } From 4a9f686816d1eeabf11bfff287ed27298a90ab5e Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Tue, 10 Dec 2024 14:06:11 +0100 Subject: [PATCH 66/70] Updated json format for QuoteMetadata --- crates/database/src/orders.rs | 14 ++++++------ crates/database/src/quotes.rs | 14 ++++++------ crates/shared/src/order_quoting.rs | 35 ++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 14 deletions(-) diff --git a/crates/database/src/orders.rs b/crates/database/src/orders.rs index 06a2e4be0d..ebcc249abe 100644 --- a/crates/database/src/orders.rs +++ b/crates/database/src/orders.rs @@ -1268,14 +1268,14 @@ mod tests { crate::clear_DANGER_(&mut db).await.unwrap(); let metadata: serde_json::Value = serde_json::from_str( - r#"{ "interactions": [ { - "target": "0102030405060708091011121314151617181920", - "value": 1, - "call_data": "0A0B0C102030" + r#"{ "version":"1.0", "interactions": [ { + "target": "0x0102030405060708091011121314151617181920", + "value": "1", + "callData": "0x0A0B0C102030" },{ - "target": "FF02030405060708091011121314151617181920", - "value": 2, - "call_data": "FF0B0C102030" + "target": "0xFF02030405060708091011121314151617181920", + "value": "2", + "callData": "0xFF0B0C102030" }] }"#, ) diff --git a/crates/database/src/quotes.rs b/crates/database/src/quotes.rs index d5ab8a2614..a932566c55 100644 --- a/crates/database/src/quotes.rs +++ b/crates/database/src/quotes.rs @@ -445,14 +445,14 @@ mod tests { crate::clear_DANGER_(&mut db).await.unwrap(); let metadata: serde_json::Value = serde_json::from_str( - r#"{ "interactions": [ { - "target": "0102030405060708091011121314151617181920", - "value": 1, - "call_data": "0A0B0C102030" + r#"{ "version":"1.0", "interactions": [ { + "target": "0x0102030405060708091011121314151617181920", + "value": "1", + "callData": "0x0A0B0C102030" },{ - "target": "FF02030405060708091011121314151617181920", - "value": 2, - "call_data": "FF0B0C102030" + "target": "0xFF02030405060708091011121314151617181920", + "value": "2", + "callData": "0xFF0B0C102030" }] }"#, ) diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index e94735c9d1..dc1829815b 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -638,7 +638,10 @@ pub fn quote_kind_from_signing_scheme(scheme: &QuoteSigningScheme) -> QuoteKind /// Used to store in database any quote metadata. #[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "camelCase")] +#[serde(tag = "version")] pub enum QuoteMetadata { + #[serde(rename = "1.0")] V1(QuoteMetadataV1), } @@ -1611,4 +1614,36 @@ mod tests { FindQuoteError::NotFound(None), )); } + + #[test] + fn check_quote_metadata_format() { + let q: QuoteMetadata = QuoteMetadataV1 { + interactions: vec![ + InteractionData { + target: H160::from([1; 20]), + value: U256::one(), + call_data: vec![1], + }, + InteractionData { + target: H160::from([2; 20]), + value: U256::from(2), + call_data: vec![2], + }, + ], + } + .into(); + let v = serde_json::to_value(q).unwrap(); + + let req: serde_json::Value = serde_json::from_str( + r#" + {"version":"1.0", + "interactions":[ + {"target":"0x0101010101010101010101010101010101010101","value":"1","callData":"0x01"} + {"target":"0x0202020202020202020202020202020202020202","value":"2","callData":"0x02"} + ]}"#, + ) + .unwrap(); + + assert_eq!(req, v); + } } From 71bfe2bb7a3b2d958c5fbb2d4c40be22824beb10 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Tue, 10 Dec 2024 14:20:06 +0100 Subject: [PATCH 67/70] Fixed unit test --- crates/shared/src/order_quoting.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index dc1829815b..a42fb31953 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -1638,7 +1638,7 @@ mod tests { r#" {"version":"1.0", "interactions":[ - {"target":"0x0101010101010101010101010101010101010101","value":"1","callData":"0x01"} + {"target":"0x0101010101010101010101010101010101010101","value":"1","callData":"0x01"}, {"target":"0x0202020202020202020202020202020202020202","value":"2","callData":"0x02"} ]}"#, ) From c9ceaa4b4d0b8676be2f22bb51df0b8eaf265c3a Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Wed, 11 Dec 2024 00:38:33 +0100 Subject: [PATCH 68/70] Added support for empty json in quote metadata column --- crates/shared/src/order_quoting.rs | 47 +++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 4 deletions(-) diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index a42fb31953..706e3129cc 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -636,15 +636,25 @@ pub fn quote_kind_from_signing_scheme(scheme: &QuoteSigningScheme) -> QuoteKind } } -/// Used to store in database any quote metadata. +/// Used to store quote metadata in the database. +/// Versioning is used for the backward compatibility. +/// In case new metadata needs to be associated with a quote create a new +/// variant version and apply serde rename attribute with proper number. #[derive(Clone, Debug, PartialEq, serde::Deserialize, serde::Serialize)] -#[serde(rename_all = "camelCase")] -#[serde(tag = "version")] +#[serde(rename_all = "camelCase", tag = "version")] pub enum QuoteMetadata { #[serde(rename = "1.0")] V1(QuoteMetadataV1), } +// Handles deserialization of empty json value {} in metadata column. +#[derive(Clone, Debug, PartialEq, serde::Deserialize)] +#[serde(untagged)] +enum QuoteMetadataDeserializationHelper { + Data(QuoteMetadata), + Empty {}, +} + impl TryInto for QuoteMetadata { type Error = serde_json::Error; @@ -657,7 +667,10 @@ impl TryFrom for QuoteMetadata { type Error = serde_json::Error; fn try_from(value: serde_json::Value) -> std::result::Result { - serde_json::from_value(value) + Ok(match serde_json::from_value(value)? { + QuoteMetadataDeserializationHelper::Data(value) => value, + QuoteMetadataDeserializationHelper::Empty {} => Default::default(), + }) } } @@ -1646,4 +1659,30 @@ mod tests { assert_eq!(req, v); } + + #[test] + fn check_quote_metadata_deserialize_from_empty_json() { + let empty_json: serde_json::Value = serde_json::from_str("{}").unwrap(); + let metadata: QuoteMetadata = empty_json.try_into().unwrap(); + // Empty json is converted to QuoteMetadata default value + assert_eq!(metadata, QuoteMetadata::default()); + } + + #[test] + fn check_quote_metadata_deserialize_from_v1_json() { + let v1: serde_json::Value = serde_json::from_str( + r#" + {"version":"1.0", + "interactions":[ + {"target":"0x0101010101010101010101010101010101010101","value":"1","callData":"0x01"}, + {"target":"0x0202020202020202020202020202020202020202","value":"2","callData":"0x02"} + ]}"#, + ) + .unwrap(); + let metadata: QuoteMetadata = v1.try_into().unwrap(); + + match metadata { + QuoteMetadata::V1(v1) => assert_eq!(v1.interactions.len(), 2), + } + } } From 373f8918de62d7e88262d52b85bfc891983342b3 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Wed, 11 Dec 2024 01:45:16 +0100 Subject: [PATCH 69/70] Fixed new e2e test --- crates/e2e/tests/e2e/place_order_with_quote.rs | 6 ++++-- crates/shared/src/order_quoting.rs | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/e2e/tests/e2e/place_order_with_quote.rs b/crates/e2e/tests/e2e/place_order_with_quote.rs index 35aaf0a948..09d3bb55d3 100644 --- a/crates/e2e/tests/e2e/place_order_with_quote.rs +++ b/crates/e2e/tests/e2e/place_order_with_quote.rs @@ -52,13 +52,14 @@ async fn place_order_with_quote(web3: Web3) { .expect("Must be able to disable automine"); tracing::info!("Quoting"); + let quote_sell_amount = to_wei(1); let quote_request = OrderQuoteRequest { from: trader.address(), sell_token: onchain.contracts().weth.address(), buy_token: token.address(), side: OrderQuoteSide::Sell { sell_amount: SellAmount::BeforeFee { - value: NonZeroU256::try_from(to_wei(1)).unwrap(), + value: NonZeroU256::try_from(quote_sell_amount).unwrap(), }, }, ..Default::default() @@ -78,7 +79,7 @@ async fn place_order_with_quote(web3: Web3) { let order = OrderCreation { quote_id: quote_response.id, sell_token: onchain.contracts().weth.address(), - sell_amount: quote_response.quote.sell_amount, + sell_amount: quote_sell_amount, buy_token: token.address(), buy_amount: quote_response.quote.buy_amount, valid_to: model::time::now_in_epoch_seconds() + 300, @@ -100,6 +101,7 @@ async fn place_order_with_quote(web3: Web3) { .await .unwrap(); assert!(order_quote.is_some()); + // compare quote metadata and order quote metadata let order_quote_metadata = order_quote.unwrap().metadata; assert_eq!(quote_metadata.unwrap().0, order_quote_metadata); } diff --git a/crates/shared/src/order_quoting.rs b/crates/shared/src/order_quoting.rs index 706e3129cc..6942655449 100644 --- a/crates/shared/src/order_quoting.rs +++ b/crates/shared/src/order_quoting.rs @@ -1671,7 +1671,7 @@ mod tests { #[test] fn check_quote_metadata_deserialize_from_v1_json() { let v1: serde_json::Value = serde_json::from_str( - r#" + r#" {"version":"1.0", "interactions":[ {"target":"0x0101010101010101010101010101010101010101","value":"1","callData":"0x01"}, From ebd3e3be0423404af3151ba93362c7c8e76d1465 Mon Sep 17 00:00:00 2001 From: Michal Strug Date: Wed, 11 Dec 2024 11:11:47 +0100 Subject: [PATCH 70/70] Improved errors conversions --- crates/orderbook/src/database/orders.rs | 10 +++++----- crates/orderbook/src/orderbook.rs | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crates/orderbook/src/database/orders.rs b/crates/orderbook/src/database/orders.rs index dff26428f1..4a1d78fd30 100644 --- a/crates/orderbook/src/database/orders.rs +++ b/crates/orderbook/src/database/orders.rs @@ -104,11 +104,11 @@ impl OrderWithQuote { buy_amount: u256_to_big_decimal("e.buy_amount), solver: ByteArray(quote.data.solver.0), verified: quote.data.verified, - metadata: quote.data.metadata.try_into().map_err( - |e: serde_json::Error| { - AddOrderError::MetadataSerializationFailed(e.into()) - }, - )?, + metadata: quote + .data + .metadata + .try_into() + .map_err(AddOrderError::MetadataSerializationFailed)?, }) }) .transpose()?, diff --git a/crates/orderbook/src/orderbook.rs b/crates/orderbook/src/orderbook.rs index a28afb6f89..2b4a6251c4 100644 --- a/crates/orderbook/src/orderbook.rs +++ b/crates/orderbook/src/orderbook.rs @@ -142,7 +142,7 @@ pub enum AddOrderError { existing: String, }, #[error("quote metadata failed to serialize as json, error: {0}")] - MetadataSerializationFailed(#[source] anyhow::Error), + MetadataSerializationFailed(serde_json::Error), } impl AddOrderError { @@ -164,7 +164,7 @@ impl AddOrderError { }, }, InsertionError::MetadataSerializationFailed(err) => { - AddOrderError::MetadataSerializationFailed(err.into()) + AddOrderError::MetadataSerializationFailed(err) } } }