From 409f9cce65392a23e6f4a5ff04ad19886eb9d9c6 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 5 Mar 2024 12:13:25 +0000 Subject: [PATCH 01/13] Store price improvement fee --- crates/autopilot/src/database/fee_policies.rs | 54 +++++++++++-- .../src/infra/persistence/dto/fee_policy.rs | 81 ++++++++++++++++--- database/README.md | 24 +++--- .../V063__price_improvement_policy_fee.sql | 14 ++++ 4 files changed, 148 insertions(+), 25 deletions(-) create mode 100644 database/sql/V063__price_improvement_policy_fee.sql diff --git a/crates/autopilot/src/database/fee_policies.rs b/crates/autopilot/src/database/fee_policies.rs index 0796d6ba6c..71cdf4cf83 100644 --- a/crates/autopilot/src/database/fee_policies.rs +++ b/crates/autopilot/src/database/fee_policies.rs @@ -9,7 +9,9 @@ pub async fn insert_batch( ) -> Result<(), sqlx::Error> { let mut query_builder = QueryBuilder::new( "INSERT INTO fee_policies (auction_id, order_uid, kind, surplus_factor, \ - max_volume_factor, volume_factor) ", + surplus_max_volume_factor, volume_factor, price_improvement_factor, \ + price_improvement_volume_factor, price_improvement_quote_sell_amount, \ + price_improvement_quote_buy_amount, price_improvement_quote_fee) ", ); query_builder.push_values(fee_policies, |mut b, fee_policy| { @@ -17,8 +19,13 @@ pub async fn insert_batch( .push_bind(fee_policy.order_uid) .push_bind(fee_policy.kind) .push_bind(fee_policy.surplus_factor) - .push_bind(fee_policy.max_volume_factor) - .push_bind(fee_policy.volume_factor); + .push_bind(fee_policy.surplus_max_volume_factor) + .push_bind(fee_policy.volume_factor) + .push_bind(fee_policy.price_improvement_factor) + .push_bind(fee_policy.price_improvement_volume_factor) + .push_bind(fee_policy.price_improvement_quote_sell_amount) + .push_bind(fee_policy.price_improvement_quote_buy_amount) + .push_bind(fee_policy.price_improvement_quote_fee); }); query_builder.build().execute(ex).await.map(|_| ()) @@ -64,8 +71,13 @@ mod tests { order_uid, kind: dto::fee_policy::FeePolicyKind::Surplus, surplus_factor: Some(0.1), - max_volume_factor: Some(1.0), + surplus_max_volume_factor: Some(1.0), volume_factor: None, + price_improvement_factor: None, + price_improvement_volume_factor: None, + price_improvement_quote_sell_amount: None, + price_improvement_quote_buy_amount: None, + price_improvement_quote_fee: None, }; // surplus fee policy with caps let fee_policy_2 = dto::FeePolicy { @@ -73,8 +85,13 @@ mod tests { order_uid, kind: dto::fee_policy::FeePolicyKind::Surplus, surplus_factor: Some(0.2), - max_volume_factor: Some(0.05), + surplus_max_volume_factor: Some(0.05), volume_factor: None, + price_improvement_factor: None, + price_improvement_volume_factor: None, + price_improvement_quote_sell_amount: None, + price_improvement_quote_buy_amount: None, + price_improvement_quote_fee: None, }; // volume based fee policy let fee_policy_3 = dto::FeePolicy { @@ -82,8 +99,27 @@ mod tests { order_uid, kind: dto::fee_policy::FeePolicyKind::Volume, surplus_factor: None, - max_volume_factor: None, + surplus_max_volume_factor: None, volume_factor: Some(0.06), + price_improvement_factor: None, + price_improvement_volume_factor: None, + price_improvement_quote_sell_amount: None, + price_improvement_quote_buy_amount: None, + price_improvement_quote_fee: None, + }; + // price improvement fee policy + let fee_policy_4 = dto::FeePolicy { + auction_id, + order_uid, + kind: dto::fee_policy::FeePolicyKind::Surplus, + surplus_factor: None, + surplus_max_volume_factor: None, + volume_factor: None, + price_improvement_factor: Some(0.1), + price_improvement_volume_factor: Some(1.0), + price_improvement_quote_sell_amount: Some(10.into()), + price_improvement_quote_buy_amount: Some(20.into()), + price_improvement_quote_fee: Some(1.into()), }; insert_batch( &mut db, @@ -91,12 +127,16 @@ mod tests { fee_policy_1.clone(), fee_policy_2.clone(), fee_policy_3.clone(), + fee_policy_4.clone(), ], ) .await .unwrap(); let output = fetch(&mut db, 1, order_uid).await.unwrap(); - assert_eq!(output, vec![fee_policy_1, fee_policy_2, fee_policy_3]); + assert_eq!( + output, + vec![fee_policy_1, fee_policy_2, fee_policy_3, fee_policy_4] + ); } } diff --git a/crates/autopilot/src/infra/persistence/dto/fee_policy.rs b/crates/autopilot/src/infra/persistence/dto/fee_policy.rs index 4bae7ec7b2..29295398c7 100644 --- a/crates/autopilot/src/infra/persistence/dto/fee_policy.rs +++ b/crates/autopilot/src/infra/persistence/dto/fee_policy.rs @@ -1,4 +1,9 @@ -use crate::{boundary, domain}; +use { + crate::{boundary, domain}, + anyhow::anyhow, + bigdecimal::BigDecimal, + number::conversions::{big_decimal_to_u256, u256_to_big_decimal}, +}; #[derive(Debug, Clone, PartialEq, sqlx::FromRow)] pub struct FeePolicy { @@ -6,8 +11,13 @@ pub struct FeePolicy { pub order_uid: boundary::database::OrderUid, pub kind: FeePolicyKind, pub surplus_factor: Option, - pub max_volume_factor: Option, + pub surplus_max_volume_factor: Option, pub volume_factor: Option, + pub price_improvement_factor: Option, + pub price_improvement_volume_factor: Option, + pub price_improvement_quote_sell_amount: Option, + pub price_improvement_quote_buy_amount: Option, + pub price_improvement_quote_fee: Option, } impl FeePolicy { @@ -25,28 +35,43 @@ impl FeePolicy { order_uid: boundary::database::byte_array::ByteArray(order_uid.0), kind: FeePolicyKind::Surplus, surplus_factor: Some(factor), - max_volume_factor: Some(max_volume_factor), + surplus_max_volume_factor: Some(max_volume_factor), volume_factor: None, + price_improvement_factor: None, + price_improvement_volume_factor: None, + price_improvement_quote_sell_amount: None, + price_improvement_quote_buy_amount: None, + price_improvement_quote_fee: None, }, domain::fee::Policy::Volume { factor } => Self { auction_id, order_uid: boundary::database::byte_array::ByteArray(order_uid.0), kind: FeePolicyKind::Volume, surplus_factor: None, - max_volume_factor: None, + surplus_max_volume_factor: None, volume_factor: Some(factor), + price_improvement_factor: None, + price_improvement_volume_factor: None, + price_improvement_quote_sell_amount: None, + price_improvement_quote_buy_amount: None, + price_improvement_quote_fee: None, }, domain::fee::Policy::PriceImprovement { factor, max_volume_factor, - .. + quote, } => Self { auction_id, order_uid: boundary::database::byte_array::ByteArray(order_uid.0), kind: FeePolicyKind::PriceImprovement, - surplus_factor: Some(factor), - max_volume_factor: Some(max_volume_factor), + surplus_factor: None, + surplus_max_volume_factor: None, volume_factor: None, + price_improvement_factor: Some(factor), + price_improvement_volume_factor: Some(max_volume_factor), + price_improvement_quote_sell_amount: Some(u256_to_big_decimal("e.sell_amount)), + price_improvement_quote_buy_amount: Some(u256_to_big_decimal("e.buy_amount)), + price_improvement_quote_fee: Some(u256_to_big_decimal("e.fee)), }, } } @@ -57,12 +82,50 @@ impl From for domain::fee::Policy { match row.kind { FeePolicyKind::Surplus => domain::fee::Policy::Surplus { factor: row.surplus_factor.expect("missing surplus factor"), - max_volume_factor: row.max_volume_factor.expect("missing max volume factor"), + max_volume_factor: row + .surplus_max_volume_factor + .expect("missing max volume factor"), }, FeePolicyKind::Volume => domain::fee::Policy::Volume { factor: row.volume_factor.expect("missing volume factor"), }, - FeePolicyKind::PriceImprovement => todo!(), + FeePolicyKind::PriceImprovement => domain::fee::Policy::PriceImprovement { + factor: row + .price_improvement_factor + .expect("missing price improvement factor"), + max_volume_factor: row + .surplus_max_volume_factor + .expect("missing price improvement max volume factor"), + quote: domain::fee::Quote { + sell_amount: row + .price_improvement_quote_sell_amount + .ok_or(anyhow!("missing price improvement quote sell amount")) + .and_then(|sell_amount| { + big_decimal_to_u256(&sell_amount).ok_or(anyhow!( + "price improvement quote sell amount is not a valid BigDecimal" + )) + }) + .unwrap(), + buy_amount: row + .price_improvement_quote_buy_amount + .ok_or(anyhow!("missing price improvement quote buy amount")) + .and_then(|sell_amount| { + big_decimal_to_u256(&sell_amount).ok_or(anyhow!( + "price improvement quote buy amount is not a valid BigDecimal" + )) + }) + .unwrap(), + fee: row + .price_improvement_quote_fee + .ok_or(anyhow!("missing price improvement quote fee")) + .and_then(|sell_amount| { + big_decimal_to_u256(&sell_amount).ok_or(anyhow!( + "price improvement quote fee is not a valid BigDecimal" + )) + }) + .unwrap(), + }, + }, } } } diff --git a/database/README.md b/database/README.md index 5a9e2c336c..9f536e5cea 100644 --- a/database/README.md +++ b/database/README.md @@ -239,15 +239,20 @@ Indexes: Contains all relevant data of fee policies applied to orders during auctions. -Column | Type | Nullable | Details ---------------------------|------------------------------|----------|-------- - auction_id | bigint | not null | unique identifier for the auction - order_uid | bytea | not null | 56 bytes identifier linking to the order in the `orders` table - application_order | serial | not null | the order in which the fee policies are inserted and applied - kind | [PolicyKind](#policykind) | not null | type of the fee policy, defined in the PolicyKind enum - surplus_factor | double precision | | percentage of the surplus for fee calculation; value is between 0 and 1 - max_volume_factor | double precision | | cap for the fee as a percentage of the order volume; value is between 0 and 1 - volume_factor | double precision | | fee percentage of the order volume; value is between 0 and 1 +Column | Type | Nullable | Details +-------------------------------------|------------------------------|----------|-------- + auction_id | bigint | not null | unique identifier for the auction + order_uid | bytea | not null | 56 bytes identifier linking to the order in the `orders` table + application_order | serial | not null | the order in which the fee policies are inserted and applied + kind | [PolicyKind](#policykind) | not null | type of the fee policy, defined in the PolicyKind enum + surplus_factor | double precision | | percentage of the surplus for fee calculation; value is between 0 and 1 + surplus_max_volume_factor | double precision | | cap for the fee as a percentage of the order volume; value is between 0 and 1 + volume_factor | double precision | | fee percentage of the order volume; value is between 0 and 1 + price_improvement_factor | double precision | | percentage of the price improvement for fee calculation; value is between 0 and 1 + price_improvement_max_volume_factor | double precision | | cap for the fee as a percentage of the order volume; value is between 0 and 1 + price_improvement_quote_sell_amount | numeric | | sell token quote amount + price_improvement_quote_buy_amount | numeric | | buy token quote amount + price_improvement_quote_buy_amount | numeric | | quote fee amount in sell token Indexes: - PRIMARY KEY: composite key(`auction_id`, `order_uid`, `application_order`) @@ -259,6 +264,7 @@ Indexes: Values: - `surplus`: The fee is based on the surplus achieved in the trade. + - `price_improvement`: The fee is based on a better executed price than the top quote. - `volume`: The fee is based on the volume of the order. ### presignature\_events diff --git a/database/sql/V063__price_improvement_policy_fee.sql b/database/sql/V063__price_improvement_policy_fee.sql new file mode 100644 index 0000000000..366155fb9e --- /dev/null +++ b/database/sql/V063__price_improvement_policy_fee.sql @@ -0,0 +1,14 @@ +-- Rename max_volume_factor column +ALTER TABLE fee_policies + RENAME COLUMN max_volume_factor TO surplus_max_volume_factor; + +-- Add `price_improvement` policy fee kind +ALTER TYPE PolicyKind ADD VALUE 'price_improvement'; + +-- Add price improvement fee columns +ALTER TABLE fee_policies + ADD COLUMN price_improvement_factor double precision, + ADD COLUMN price_improvement_volume_factor double precision, + ADD COLUMN price_improvement_quote_sell_amount numeric(78), + ADD COLUMN price_improvement_quote_buy_amount numeric(78), + ADD COLUMN price_improvement_quote_fee numeric(78); From 12e3beb02a59de50b1b23d594d61db83c0aacb10 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 5 Mar 2024 12:18:00 +0000 Subject: [PATCH 02/13] Naming --- crates/autopilot/src/database/fee_policies.rs | 12 ++++++------ .../src/infra/persistence/dto/fee_policy.rs | 8 ++++---- database/sql/V063__price_improvement_policy_fee.sql | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/autopilot/src/database/fee_policies.rs b/crates/autopilot/src/database/fee_policies.rs index 71cdf4cf83..8fc36506ad 100644 --- a/crates/autopilot/src/database/fee_policies.rs +++ b/crates/autopilot/src/database/fee_policies.rs @@ -10,7 +10,7 @@ pub async fn insert_batch( let mut query_builder = QueryBuilder::new( "INSERT INTO fee_policies (auction_id, order_uid, kind, surplus_factor, \ surplus_max_volume_factor, volume_factor, price_improvement_factor, \ - price_improvement_volume_factor, price_improvement_quote_sell_amount, \ + price_improvement_max_volume_factor, price_improvement_quote_sell_amount, \ price_improvement_quote_buy_amount, price_improvement_quote_fee) ", ); @@ -22,7 +22,7 @@ pub async fn insert_batch( .push_bind(fee_policy.surplus_max_volume_factor) .push_bind(fee_policy.volume_factor) .push_bind(fee_policy.price_improvement_factor) - .push_bind(fee_policy.price_improvement_volume_factor) + .push_bind(fee_policy.price_improvement_max_volume_factor) .push_bind(fee_policy.price_improvement_quote_sell_amount) .push_bind(fee_policy.price_improvement_quote_buy_amount) .push_bind(fee_policy.price_improvement_quote_fee); @@ -74,7 +74,7 @@ mod tests { surplus_max_volume_factor: Some(1.0), volume_factor: None, price_improvement_factor: None, - price_improvement_volume_factor: None, + price_improvement_max_volume_factor: None, price_improvement_quote_sell_amount: None, price_improvement_quote_buy_amount: None, price_improvement_quote_fee: None, @@ -88,7 +88,7 @@ mod tests { surplus_max_volume_factor: Some(0.05), volume_factor: None, price_improvement_factor: None, - price_improvement_volume_factor: None, + price_improvement_max_volume_factor: None, price_improvement_quote_sell_amount: None, price_improvement_quote_buy_amount: None, price_improvement_quote_fee: None, @@ -102,7 +102,7 @@ mod tests { surplus_max_volume_factor: None, volume_factor: Some(0.06), price_improvement_factor: None, - price_improvement_volume_factor: None, + price_improvement_max_volume_factor: None, price_improvement_quote_sell_amount: None, price_improvement_quote_buy_amount: None, price_improvement_quote_fee: None, @@ -116,7 +116,7 @@ mod tests { surplus_max_volume_factor: None, volume_factor: None, price_improvement_factor: Some(0.1), - price_improvement_volume_factor: Some(1.0), + price_improvement_max_volume_factor: Some(1.0), price_improvement_quote_sell_amount: Some(10.into()), price_improvement_quote_buy_amount: Some(20.into()), price_improvement_quote_fee: Some(1.into()), diff --git a/crates/autopilot/src/infra/persistence/dto/fee_policy.rs b/crates/autopilot/src/infra/persistence/dto/fee_policy.rs index 29295398c7..207172f381 100644 --- a/crates/autopilot/src/infra/persistence/dto/fee_policy.rs +++ b/crates/autopilot/src/infra/persistence/dto/fee_policy.rs @@ -14,7 +14,7 @@ pub struct FeePolicy { pub surplus_max_volume_factor: Option, pub volume_factor: Option, pub price_improvement_factor: Option, - pub price_improvement_volume_factor: Option, + pub price_improvement_max_volume_factor: Option, pub price_improvement_quote_sell_amount: Option, pub price_improvement_quote_buy_amount: Option, pub price_improvement_quote_fee: Option, @@ -38,7 +38,7 @@ impl FeePolicy { surplus_max_volume_factor: Some(max_volume_factor), volume_factor: None, price_improvement_factor: None, - price_improvement_volume_factor: None, + price_improvement_max_volume_factor: None, price_improvement_quote_sell_amount: None, price_improvement_quote_buy_amount: None, price_improvement_quote_fee: None, @@ -51,7 +51,7 @@ impl FeePolicy { surplus_max_volume_factor: None, volume_factor: Some(factor), price_improvement_factor: None, - price_improvement_volume_factor: None, + price_improvement_max_volume_factor: None, price_improvement_quote_sell_amount: None, price_improvement_quote_buy_amount: None, price_improvement_quote_fee: None, @@ -68,7 +68,7 @@ impl FeePolicy { surplus_max_volume_factor: None, volume_factor: None, price_improvement_factor: Some(factor), - price_improvement_volume_factor: Some(max_volume_factor), + price_improvement_max_volume_factor: Some(max_volume_factor), price_improvement_quote_sell_amount: Some(u256_to_big_decimal("e.sell_amount)), price_improvement_quote_buy_amount: Some(u256_to_big_decimal("e.buy_amount)), price_improvement_quote_fee: Some(u256_to_big_decimal("e.fee)), diff --git a/database/sql/V063__price_improvement_policy_fee.sql b/database/sql/V063__price_improvement_policy_fee.sql index 366155fb9e..5d02efb3f8 100644 --- a/database/sql/V063__price_improvement_policy_fee.sql +++ b/database/sql/V063__price_improvement_policy_fee.sql @@ -8,7 +8,7 @@ ALTER TYPE PolicyKind ADD VALUE 'price_improvement'; -- Add price improvement fee columns ALTER TABLE fee_policies ADD COLUMN price_improvement_factor double precision, - ADD COLUMN price_improvement_volume_factor double precision, + ADD COLUMN price_improvement_max_volume_factor double precision, ADD COLUMN price_improvement_quote_sell_amount numeric(78), ADD COLUMN price_improvement_quote_buy_amount numeric(78), ADD COLUMN price_improvement_quote_fee numeric(78); From f38ede86e66abbd42273d2609b39fa947d2d7eb5 Mon Sep 17 00:00:00 2001 From: ilya Date: Tue, 5 Mar 2024 12:23:44 +0000 Subject: [PATCH 03/13] Readme fix --- database/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/README.md b/database/README.md index 9f536e5cea..7447d72b11 100644 --- a/database/README.md +++ b/database/README.md @@ -252,7 +252,7 @@ Column | Type | Nullable | price_improvement_max_volume_factor | double precision | | cap for the fee as a percentage of the order volume; value is between 0 and 1 price_improvement_quote_sell_amount | numeric | | sell token quote amount price_improvement_quote_buy_amount | numeric | | buy token quote amount - price_improvement_quote_buy_amount | numeric | | quote fee amount in sell token + price_improvement_quote_fee | numeric | | fee amount that needs to be paid, denominated in the sell token Indexes: - PRIMARY KEY: composite key(`auction_id`, `order_uid`, `application_order`) From aa2d0618e33c34823b396e323278e78c63fefc0f Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 7 Mar 2024 12:23:43 +0000 Subject: [PATCH 04/13] Work with domain policies --- crates/autopilot/src/database/fee_policies.rs | 60 +++++++++++++------ crates/autopilot/src/infra/persistence/mod.rs | 12 +--- 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/crates/autopilot/src/database/fee_policies.rs b/crates/autopilot/src/database/fee_policies.rs index 8fc36506ad..8cee2d6d58 100644 --- a/crates/autopilot/src/database/fee_policies.rs +++ b/crates/autopilot/src/database/fee_policies.rs @@ -1,11 +1,12 @@ use { - crate::infra::persistence::dto, + crate::{domain, infra::persistence::dto}, sqlx::{PgConnection, QueryBuilder}, }; pub async fn insert_batch( ex: &mut PgConnection, - fee_policies: impl IntoIterator, + auction_id: domain::AuctionId, + fee_policies: impl IntoIterator)>, ) -> Result<(), sqlx::Error> { let mut query_builder = QueryBuilder::new( "INSERT INTO fee_policies (auction_id, order_uid, kind, surplus_factor, \ @@ -14,6 +15,12 @@ pub async fn insert_batch( price_improvement_quote_buy_amount, price_improvement_quote_fee) ", ); + let fee_policies = fee_policies.into_iter().flat_map(|(order_uid, policies)| { + policies + .into_iter() + .map(move |policy| dto::FeePolicy::from_domain(auction_id, order_uid, policy)) + }); + query_builder.push_values(fee_policies, |mut b, fee_policy| { b.push_bind(fee_policy.auction_id) .push_bind(fee_policy.order_uid) @@ -65,6 +72,38 @@ mod tests { // same primary key for all fee policies let (auction_id, order_uid) = (1, ByteArray([1; 56])); + // surplus fee policy without caps + let fee_policy_1 = domain::fee::Policy::Surplus { + factor: 0.1, + max_volume_factor: 1.0, + }; + // surplus fee policy with caps + let fee_policy_2 = domain::fee::Policy::Surplus { + factor: 0.2, + max_volume_factor: 0.05, + }; + // volume based fee policy + let fee_policy_3 = domain::fee::Policy::Volume { factor: 0.06 }; + // price improvement fee policy + let fee_policy_4 = domain::fee::Policy::PriceImprovement { + factor: 0.1, + max_volume_factor: 1.0, + quote: domain::fee::Quote { + sell_amount: 10.into(), + buy_amount: 20.into(), + fee: 1.into(), + }, + }; + let input_policies = vec![fee_policy_1, fee_policy_2, fee_policy_3, fee_policy_4]; + + insert_batch( + &mut db, + auction_id, + vec![(domain::OrderUid(order_uid.0), input_policies)], + ) + .await + .unwrap(); + // surplus fee policy without caps let fee_policy_1 = dto::FeePolicy { auction_id, @@ -121,22 +160,9 @@ mod tests { price_improvement_quote_buy_amount: Some(20.into()), price_improvement_quote_fee: Some(1.into()), }; - insert_batch( - &mut db, - vec![ - fee_policy_1.clone(), - fee_policy_2.clone(), - fee_policy_3.clone(), - fee_policy_4.clone(), - ], - ) - .await - .unwrap(); + let expected = vec![fee_policy_1, fee_policy_2, fee_policy_3, fee_policy_4]; let output = fetch(&mut db, 1, order_uid).await.unwrap(); - assert_eq!( - output, - vec![fee_policy_1, fee_policy_2, fee_policy_3, fee_policy_4] - ); + assert_eq!(output, expected); } } diff --git a/crates/autopilot/src/infra/persistence/mod.rs b/crates/autopilot/src/infra/persistence/mod.rs index 1394d08f74..93f63ce92b 100644 --- a/crates/autopilot/src/infra/persistence/mod.rs +++ b/crates/autopilot/src/infra/persistence/mod.rs @@ -2,7 +2,6 @@ use { crate::{boundary, database::Postgres, domain}, anyhow::Context, chrono::Utc, - itertools::Itertools, std::sync::Arc, tokio::time::Instant, tracing::Instrument, @@ -122,18 +121,9 @@ impl Persistence { auction_id: domain::AuctionId, fee_policies: Vec<(domain::OrderUid, Vec)>, ) -> anyhow::Result<()> { - let fee_policies = fee_policies - .into_iter() - .flat_map(|(order_uid, policies)| { - policies - .into_iter() - .map(move |policy| dto::FeePolicy::from_domain(auction_id, order_uid, policy)) - }) - .collect_vec(); - let mut ex = self.postgres.pool.begin().await.context("begin")?; for chunk in fee_policies.chunks(self.postgres.config.insert_batch_size.get()) { - crate::database::fee_policies::insert_batch(&mut ex, chunk.iter().cloned()) + crate::database::fee_policies::insert_batch(&mut ex, auction_id, chunk.iter().cloned()) .await .context("fee_policies::insert_batch")?; } From 794dc8a84abe41dea316ea9b3d37f8529244b4f2 Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 7 Mar 2024 12:33:06 +0000 Subject: [PATCH 05/13] Review comments --- .../src/infra/persistence/dto/fee_policy.rs | 43 +++++++------------ database/README.md | 2 +- 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/crates/autopilot/src/infra/persistence/dto/fee_policy.rs b/crates/autopilot/src/infra/persistence/dto/fee_policy.rs index 207172f381..6e78d34d24 100644 --- a/crates/autopilot/src/infra/persistence/dto/fee_policy.rs +++ b/crates/autopilot/src/infra/persistence/dto/fee_policy.rs @@ -1,6 +1,5 @@ use { crate::{boundary, domain}, - anyhow::anyhow, bigdecimal::BigDecimal, number::conversions::{big_decimal_to_u256, u256_to_big_decimal}, }; @@ -97,33 +96,21 @@ impl From for domain::fee::Policy { .surplus_max_volume_factor .expect("missing price improvement max volume factor"), quote: domain::fee::Quote { - sell_amount: row - .price_improvement_quote_sell_amount - .ok_or(anyhow!("missing price improvement quote sell amount")) - .and_then(|sell_amount| { - big_decimal_to_u256(&sell_amount).ok_or(anyhow!( - "price improvement quote sell amount is not a valid BigDecimal" - )) - }) - .unwrap(), - buy_amount: row - .price_improvement_quote_buy_amount - .ok_or(anyhow!("missing price improvement quote buy amount")) - .and_then(|sell_amount| { - big_decimal_to_u256(&sell_amount).ok_or(anyhow!( - "price improvement quote buy amount is not a valid BigDecimal" - )) - }) - .unwrap(), - fee: row - .price_improvement_quote_fee - .ok_or(anyhow!("missing price improvement quote fee")) - .and_then(|sell_amount| { - big_decimal_to_u256(&sell_amount).ok_or(anyhow!( - "price improvement quote fee is not a valid BigDecimal" - )) - }) - .unwrap(), + sell_amount: big_decimal_to_u256( + &row.price_improvement_quote_sell_amount + .expect("missing price improvement quote sell amount"), + ) + .expect("price improvement quote sell amount is not a valid BigDecimal"), + buy_amount: big_decimal_to_u256( + &row.price_improvement_quote_buy_amount + .expect("missing price improvement quote buy amount"), + ) + .expect("price improvement quote buy amount is not a valid BigDecimal"), + fee: big_decimal_to_u256( + &row.price_improvement_quote_fee + .expect("missing price improvement quote fee"), + ) + .expect("price improvement quote fee is not a valid BigDecimal"), }, }, } diff --git a/database/README.md b/database/README.md index 7447d72b11..7238ecb7b6 100644 --- a/database/README.md +++ b/database/README.md @@ -248,7 +248,7 @@ Column | Type | Nullable | surplus_factor | double precision | | percentage of the surplus for fee calculation; value is between 0 and 1 surplus_max_volume_factor | double precision | | cap for the fee as a percentage of the order volume; value is between 0 and 1 volume_factor | double precision | | fee percentage of the order volume; value is between 0 and 1 - price_improvement_factor | double precision | | percentage of the price improvement for fee calculation; value is between 0 and 1 + price_improvement_factor | double precision | | percentage of the price improvement over the best quote received during order creation; value is between 0 and 1 price_improvement_max_volume_factor | double precision | | cap for the fee as a percentage of the order volume; value is between 0 and 1 price_improvement_quote_sell_amount | numeric | | sell token quote amount price_improvement_quote_buy_amount | numeric | | buy token quote amount From 8ed5ab2a2410730fd7f03a0b61884ed9037e943f Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 7 Mar 2024 12:53:05 +0000 Subject: [PATCH 06/13] Drop quote --- crates/autopilot/src/database/fee_policies.rs | 20 +------ .../src/infra/persistence/dto/fee_policy.rs | 55 ++++--------------- database/README.md | 3 - .../V063__price_improvement_policy_fee.sql | 5 +- 4 files changed, 14 insertions(+), 69 deletions(-) diff --git a/crates/autopilot/src/database/fee_policies.rs b/crates/autopilot/src/database/fee_policies.rs index 8cee2d6d58..1cd0db4fed 100644 --- a/crates/autopilot/src/database/fee_policies.rs +++ b/crates/autopilot/src/database/fee_policies.rs @@ -11,8 +11,7 @@ pub async fn insert_batch( let mut query_builder = QueryBuilder::new( "INSERT INTO fee_policies (auction_id, order_uid, kind, surplus_factor, \ surplus_max_volume_factor, volume_factor, price_improvement_factor, \ - price_improvement_max_volume_factor, price_improvement_quote_sell_amount, \ - price_improvement_quote_buy_amount, price_improvement_quote_fee) ", + price_improvement_max_volume_factor)", ); let fee_policies = fee_policies.into_iter().flat_map(|(order_uid, policies)| { @@ -29,10 +28,7 @@ pub async fn insert_batch( .push_bind(fee_policy.surplus_max_volume_factor) .push_bind(fee_policy.volume_factor) .push_bind(fee_policy.price_improvement_factor) - .push_bind(fee_policy.price_improvement_max_volume_factor) - .push_bind(fee_policy.price_improvement_quote_sell_amount) - .push_bind(fee_policy.price_improvement_quote_buy_amount) - .push_bind(fee_policy.price_improvement_quote_fee); + .push_bind(fee_policy.price_improvement_max_volume_factor); }); query_builder.build().execute(ex).await.map(|_| ()) @@ -114,9 +110,6 @@ mod tests { volume_factor: None, price_improvement_factor: None, price_improvement_max_volume_factor: None, - price_improvement_quote_sell_amount: None, - price_improvement_quote_buy_amount: None, - price_improvement_quote_fee: None, }; // surplus fee policy with caps let fee_policy_2 = dto::FeePolicy { @@ -128,9 +121,6 @@ mod tests { volume_factor: None, price_improvement_factor: None, price_improvement_max_volume_factor: None, - price_improvement_quote_sell_amount: None, - price_improvement_quote_buy_amount: None, - price_improvement_quote_fee: None, }; // volume based fee policy let fee_policy_3 = dto::FeePolicy { @@ -142,9 +132,6 @@ mod tests { volume_factor: Some(0.06), price_improvement_factor: None, price_improvement_max_volume_factor: None, - price_improvement_quote_sell_amount: None, - price_improvement_quote_buy_amount: None, - price_improvement_quote_fee: None, }; // price improvement fee policy let fee_policy_4 = dto::FeePolicy { @@ -156,9 +143,6 @@ mod tests { volume_factor: None, price_improvement_factor: Some(0.1), price_improvement_max_volume_factor: Some(1.0), - price_improvement_quote_sell_amount: Some(10.into()), - price_improvement_quote_buy_amount: Some(20.into()), - price_improvement_quote_fee: Some(1.into()), }; let expected = vec![fee_policy_1, fee_policy_2, fee_policy_3, fee_policy_4]; diff --git a/crates/autopilot/src/infra/persistence/dto/fee_policy.rs b/crates/autopilot/src/infra/persistence/dto/fee_policy.rs index 6e78d34d24..73d794cb38 100644 --- a/crates/autopilot/src/infra/persistence/dto/fee_policy.rs +++ b/crates/autopilot/src/infra/persistence/dto/fee_policy.rs @@ -1,8 +1,4 @@ -use { - crate::{boundary, domain}, - bigdecimal::BigDecimal, - number::conversions::{big_decimal_to_u256, u256_to_big_decimal}, -}; +use crate::{boundary, domain}; #[derive(Debug, Clone, PartialEq, sqlx::FromRow)] pub struct FeePolicy { @@ -14,9 +10,6 @@ pub struct FeePolicy { pub volume_factor: Option, pub price_improvement_factor: Option, pub price_improvement_max_volume_factor: Option, - pub price_improvement_quote_sell_amount: Option, - pub price_improvement_quote_buy_amount: Option, - pub price_improvement_quote_fee: Option, } impl FeePolicy { @@ -38,9 +31,6 @@ impl FeePolicy { volume_factor: None, price_improvement_factor: None, price_improvement_max_volume_factor: None, - price_improvement_quote_sell_amount: None, - price_improvement_quote_buy_amount: None, - price_improvement_quote_fee: None, }, domain::fee::Policy::Volume { factor } => Self { auction_id, @@ -51,14 +41,11 @@ impl FeePolicy { volume_factor: Some(factor), price_improvement_factor: None, price_improvement_max_volume_factor: None, - price_improvement_quote_sell_amount: None, - price_improvement_quote_buy_amount: None, - price_improvement_quote_fee: None, }, domain::fee::Policy::PriceImprovement { factor, max_volume_factor, - quote, + quote: _, } => Self { auction_id, order_uid: boundary::database::byte_array::ByteArray(order_uid.0), @@ -68,50 +55,30 @@ impl FeePolicy { volume_factor: None, price_improvement_factor: Some(factor), price_improvement_max_volume_factor: Some(max_volume_factor), - price_improvement_quote_sell_amount: Some(u256_to_big_decimal("e.sell_amount)), - price_improvement_quote_buy_amount: Some(u256_to_big_decimal("e.buy_amount)), - price_improvement_quote_fee: Some(u256_to_big_decimal("e.fee)), }, } } -} -impl From for domain::fee::Policy { - fn from(row: FeePolicy) -> domain::fee::Policy { - match row.kind { + #[allow(dead_code)] + pub fn into_domain(self, quote: Option) -> domain::fee::Policy { + match self.kind { FeePolicyKind::Surplus => domain::fee::Policy::Surplus { - factor: row.surplus_factor.expect("missing surplus factor"), - max_volume_factor: row + factor: self.surplus_factor.expect("missing surplus factor"), + max_volume_factor: self .surplus_max_volume_factor .expect("missing max volume factor"), }, FeePolicyKind::Volume => domain::fee::Policy::Volume { - factor: row.volume_factor.expect("missing volume factor"), + factor: self.volume_factor.expect("missing volume factor"), }, FeePolicyKind::PriceImprovement => domain::fee::Policy::PriceImprovement { - factor: row + factor: self .price_improvement_factor .expect("missing price improvement factor"), - max_volume_factor: row + max_volume_factor: self .surplus_max_volume_factor .expect("missing price improvement max volume factor"), - quote: domain::fee::Quote { - sell_amount: big_decimal_to_u256( - &row.price_improvement_quote_sell_amount - .expect("missing price improvement quote sell amount"), - ) - .expect("price improvement quote sell amount is not a valid BigDecimal"), - buy_amount: big_decimal_to_u256( - &row.price_improvement_quote_buy_amount - .expect("missing price improvement quote buy amount"), - ) - .expect("price improvement quote buy amount is not a valid BigDecimal"), - fee: big_decimal_to_u256( - &row.price_improvement_quote_fee - .expect("missing price improvement quote fee"), - ) - .expect("price improvement quote fee is not a valid BigDecimal"), - }, + quote: quote.expect("quote is required for the PriceImprovement policy fee"), }, } } diff --git a/database/README.md b/database/README.md index 7238ecb7b6..688e83d338 100644 --- a/database/README.md +++ b/database/README.md @@ -250,9 +250,6 @@ Column | Type | Nullable | volume_factor | double precision | | fee percentage of the order volume; value is between 0 and 1 price_improvement_factor | double precision | | percentage of the price improvement over the best quote received during order creation; value is between 0 and 1 price_improvement_max_volume_factor | double precision | | cap for the fee as a percentage of the order volume; value is between 0 and 1 - price_improvement_quote_sell_amount | numeric | | sell token quote amount - price_improvement_quote_buy_amount | numeric | | buy token quote amount - price_improvement_quote_fee | numeric | | fee amount that needs to be paid, denominated in the sell token Indexes: - PRIMARY KEY: composite key(`auction_id`, `order_uid`, `application_order`) diff --git a/database/sql/V063__price_improvement_policy_fee.sql b/database/sql/V063__price_improvement_policy_fee.sql index 5d02efb3f8..754891b0f2 100644 --- a/database/sql/V063__price_improvement_policy_fee.sql +++ b/database/sql/V063__price_improvement_policy_fee.sql @@ -8,7 +8,4 @@ ALTER TYPE PolicyKind ADD VALUE 'price_improvement'; -- Add price improvement fee columns ALTER TABLE fee_policies ADD COLUMN price_improvement_factor double precision, - ADD COLUMN price_improvement_max_volume_factor double precision, - ADD COLUMN price_improvement_quote_sell_amount numeric(78), - ADD COLUMN price_improvement_quote_buy_amount numeric(78), - ADD COLUMN price_improvement_quote_fee numeric(78); + ADD COLUMN price_improvement_max_volume_factor double precision; From 506e9be487a8a16a10c6e6d8ad8124f9ad83f7d1 Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 7 Mar 2024 17:50:16 +0000 Subject: [PATCH 07/13] Dead code --- .../src/infra/persistence/dto/fee_policy.rs | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/crates/autopilot/src/infra/persistence/dto/fee_policy.rs b/crates/autopilot/src/infra/persistence/dto/fee_policy.rs index 73d794cb38..8f60223dba 100644 --- a/crates/autopilot/src/infra/persistence/dto/fee_policy.rs +++ b/crates/autopilot/src/infra/persistence/dto/fee_policy.rs @@ -58,30 +58,6 @@ impl FeePolicy { }, } } - - #[allow(dead_code)] - pub fn into_domain(self, quote: Option) -> domain::fee::Policy { - match self.kind { - FeePolicyKind::Surplus => domain::fee::Policy::Surplus { - factor: self.surplus_factor.expect("missing surplus factor"), - max_volume_factor: self - .surplus_max_volume_factor - .expect("missing max volume factor"), - }, - FeePolicyKind::Volume => domain::fee::Policy::Volume { - factor: self.volume_factor.expect("missing volume factor"), - }, - FeePolicyKind::PriceImprovement => domain::fee::Policy::PriceImprovement { - factor: self - .price_improvement_factor - .expect("missing price improvement factor"), - max_volume_factor: self - .surplus_max_volume_factor - .expect("missing price improvement max volume factor"), - quote: quote.expect("quote is required for the PriceImprovement policy fee"), - }, - } - } } #[derive(Debug, Clone, PartialEq, sqlx::Type)] From 0306a58d5578613a94289d2a50debb3c9690e889 Mon Sep 17 00:00:00 2001 From: ilya Date: Thu, 7 Mar 2024 19:00:15 +0000 Subject: [PATCH 08/13] Remove swatinem --- .github/workflows/pull-request.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index e4acbd90f1..b52fca014f 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -12,7 +12,6 @@ jobs: steps: - uses: actions/checkout@v3 - run: rustup toolchain install stable --profile minimal - - uses: Swatinem/rust-cache@v2 - uses: actions-rs/toolchain@v1 with: profile: minimal @@ -47,7 +46,6 @@ jobs: steps: - uses: actions/checkout@v3 - run: rustup toolchain install stable --profile minimal - - uses: Swatinem/rust-cache@v2 - uses: taiki-e/install-action@nextest - run: cargo build --tests - run: cargo nextest run @@ -63,7 +61,6 @@ jobs: steps: - uses: actions/checkout@v3 - run: rustup toolchain install stable --profile minimal - - uses: Swatinem/rust-cache@v2 # Start the build process in the background. The following cargo test command will automatically # wait for the build process to be done before proceeding. - run: cargo build -p orderbook -p database -p autopilot --tests & @@ -87,7 +84,6 @@ jobs: - uses: actions/checkout@v3 - run: rustup toolchain install stable --profile minimal - uses: foundry-rs/foundry-toolchain@v1 - - uses: Swatinem/rust-cache@v2 # Start the build process in the background. The following cargo test command will automatically # wait for the build process to be done before proceeding. - run: cargo build -p e2e --tests & @@ -115,7 +111,6 @@ jobs: - uses: actions/checkout@v3 - run: rustup toolchain install stable --profile minimal - uses: foundry-rs/foundry-toolchain@v1 - - uses: Swatinem/rust-cache@v2 # Start the build process in the background. The following cargo test command will automatically # wait for the build process to be done before proceeding. - run: cargo build -p e2e --tests & @@ -139,7 +134,6 @@ jobs: - uses: actions/checkout@v3 - run: rustup toolchain install stable --profile minimal - uses: foundry-rs/foundry-toolchain@v1 - - uses: Swatinem/rust-cache@v2 - uses: taiki-e/install-action@nextest # Build the driver's tests. - run: cargo build -p driver --tests From 0784877b819fbba1b37e0f2182706344805a0067 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 8 Mar 2024 12:43:30 +0000 Subject: [PATCH 09/13] Revert "Remove swatinem" This reverts commit 0306a58d5578613a94289d2a50debb3c9690e889. --- .github/workflows/pull-request.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index b52fca014f..e4acbd90f1 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -12,6 +12,7 @@ jobs: steps: - uses: actions/checkout@v3 - run: rustup toolchain install stable --profile minimal + - uses: Swatinem/rust-cache@v2 - uses: actions-rs/toolchain@v1 with: profile: minimal @@ -46,6 +47,7 @@ jobs: steps: - uses: actions/checkout@v3 - run: rustup toolchain install stable --profile minimal + - uses: Swatinem/rust-cache@v2 - uses: taiki-e/install-action@nextest - run: cargo build --tests - run: cargo nextest run @@ -61,6 +63,7 @@ jobs: steps: - uses: actions/checkout@v3 - run: rustup toolchain install stable --profile minimal + - uses: Swatinem/rust-cache@v2 # Start the build process in the background. The following cargo test command will automatically # wait for the build process to be done before proceeding. - run: cargo build -p orderbook -p database -p autopilot --tests & @@ -84,6 +87,7 @@ jobs: - uses: actions/checkout@v3 - run: rustup toolchain install stable --profile minimal - uses: foundry-rs/foundry-toolchain@v1 + - uses: Swatinem/rust-cache@v2 # Start the build process in the background. The following cargo test command will automatically # wait for the build process to be done before proceeding. - run: cargo build -p e2e --tests & @@ -111,6 +115,7 @@ jobs: - uses: actions/checkout@v3 - run: rustup toolchain install stable --profile minimal - uses: foundry-rs/foundry-toolchain@v1 + - uses: Swatinem/rust-cache@v2 # Start the build process in the background. The following cargo test command will automatically # wait for the build process to be done before proceeding. - run: cargo build -p e2e --tests & @@ -134,6 +139,7 @@ jobs: - uses: actions/checkout@v3 - run: rustup toolchain install stable --profile minimal - uses: foundry-rs/foundry-toolchain@v1 + - uses: Swatinem/rust-cache@v2 - uses: taiki-e/install-action@nextest # Build the driver's tests. - run: cargo build -p driver --tests From a6f67123c693d3cb6e850b132b6d5c421955cba8 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 8 Mar 2024 12:46:17 +0000 Subject: [PATCH 10/13] Compilation fix --- crates/autopilot/src/database/fee_policies.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/autopilot/src/database/fee_policies.rs b/crates/autopilot/src/database/fee_policies.rs index 1cd0db4fed..752e1f81c0 100644 --- a/crates/autopilot/src/database/fee_policies.rs +++ b/crates/autopilot/src/database/fee_policies.rs @@ -5,7 +5,7 @@ use { pub async fn insert_batch( ex: &mut PgConnection, - auction_id: domain::AuctionId, + auction_id: domain::auction::Id, fee_policies: impl IntoIterator)>, ) -> Result<(), sqlx::Error> { let mut query_builder = QueryBuilder::new( From a340a51c4cc06829c20d44a4843c961eae4007c7 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 8 Mar 2024 12:58:09 +0000 Subject: [PATCH 11/13] Kind naming --- database/sql/V063__price_improvement_policy_fee.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/sql/V063__price_improvement_policy_fee.sql b/database/sql/V063__price_improvement_policy_fee.sql index 754891b0f2..92eb5c3ae2 100644 --- a/database/sql/V063__price_improvement_policy_fee.sql +++ b/database/sql/V063__price_improvement_policy_fee.sql @@ -3,7 +3,7 @@ ALTER TABLE fee_policies RENAME COLUMN max_volume_factor TO surplus_max_volume_factor; -- Add `price_improvement` policy fee kind -ALTER TYPE PolicyKind ADD VALUE 'price_improvement'; +ALTER TYPE PolicyKind ADD VALUE 'priceimprovement'; -- Add price improvement fee columns ALTER TABLE fee_policies From c10d72cc2b027ab6bc239830f0766042a8d6cb47 Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 8 Mar 2024 12:59:37 +0000 Subject: [PATCH 12/13] Doc --- database/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/README.md b/database/README.md index 688e83d338..c08a518cb4 100644 --- a/database/README.md +++ b/database/README.md @@ -261,7 +261,7 @@ Indexes: Values: - `surplus`: The fee is based on the surplus achieved in the trade. - - `price_improvement`: The fee is based on a better executed price than the top quote. + - `priceimprovement`: The fee is based on a better executed price than the top quote. - `volume`: The fee is based on the volume of the order. ### presignature\_events From b8649042a4adca1ad46cdaef10cf5ffaff7bedba Mon Sep 17 00:00:00 2001 From: ilya Date: Fri, 8 Mar 2024 13:06:03 +0000 Subject: [PATCH 13/13] Test fix --- crates/autopilot/src/database/fee_policies.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/autopilot/src/database/fee_policies.rs b/crates/autopilot/src/database/fee_policies.rs index 752e1f81c0..c691376554 100644 --- a/crates/autopilot/src/database/fee_policies.rs +++ b/crates/autopilot/src/database/fee_policies.rs @@ -137,7 +137,7 @@ mod tests { let fee_policy_4 = dto::FeePolicy { auction_id, order_uid, - kind: dto::fee_policy::FeePolicyKind::Surplus, + kind: dto::fee_policy::FeePolicyKind::PriceImprovement, surplus_factor: None, surplus_max_volume_factor: None, volume_factor: None,