From 314e0ee2c5dbc24bd22f08e483a16423d74327a1 Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Sun, 26 Nov 2023 05:09:00 -0500 Subject: [PATCH 01/11] Use redb and in Dockerfiles The motivation for redb was to remove the multiple rocksdb compile times from CI. --- common/db/Cargo.toml | 2 ++ common/db/src/lib.rs | 5 +++++ common/db/src/redb.rs | 46 +++++++++++++++++++++++++++++++++++++++ coordinator/Cargo.toml | 6 ++++- coordinator/src/main.rs | 13 +++++++++-- message-queue/Cargo.toml | 5 +++-- message-queue/src/main.rs | 12 +++++++++- processor/Cargo.toml | 6 +++-- processor/src/main.rs | 12 +++++++++- 9 files changed, 98 insertions(+), 9 deletions(-) create mode 100644 common/db/src/redb.rs diff --git a/common/db/Cargo.toml b/common/db/Cargo.toml index e2adffe17..7aad11270 100644 --- a/common/db/Cargo.toml +++ b/common/db/Cargo.toml @@ -14,7 +14,9 @@ all-features = true rustdoc-args = ["--cfg", "docsrs"] [dependencies] +redb = { version = "1", default-features = false, optional = true } rocksdb = { version = "0.21", default-features = false, features = ["lz4"], optional = true } [features] +redb = ["dep:redb"] rocksdb = ["dep:rocksdb"] diff --git a/common/db/src/lib.rs b/common/db/src/lib.rs index 031bcd4ed..df6f1394c 100644 --- a/common/db/src/lib.rs +++ b/common/db/src/lib.rs @@ -9,6 +9,11 @@ mod rocks; #[cfg(feature = "rocksdb")] pub use rocks::{RocksDB, new_rocksdb}; +#[cfg(feature = "redb")] +mod redb; +#[cfg(feature = "redb")] +pub use redb::{Redb, new_redb}; + /// An object implementing get. pub trait Get { fn get(&self, key: impl AsRef<[u8]>) -> Option>; diff --git a/common/db/src/redb.rs b/common/db/src/redb.rs new file mode 100644 index 000000000..c34ff4ad5 --- /dev/null +++ b/common/db/src/redb.rs @@ -0,0 +1,46 @@ +use std::sync::Arc; + +pub use ::redb::Database as Redb; +use ::redb::*; + +use crate::*; + +impl Get for WriteTransaction<'_> { + fn get(&self, key: impl AsRef<[u8]>) -> Option> { + let table = self.open_table::<&[u8], Vec>(TableDefinition::new("default")).unwrap(); + table.get(key.as_ref()).unwrap().map(|value| value.value().to_vec()) + } +} +impl DbTxn for WriteTransaction<'_> { + fn put(&mut self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) { + let mut table = self.open_table::<&[u8], &[u8]>(TableDefinition::new("default")).unwrap(); + table.insert(key.as_ref(), value.as_ref()).unwrap(); + } + fn del(&mut self, key: impl AsRef<[u8]>) { + let mut table = self.open_table::<&[u8], Vec>(TableDefinition::new("default")).unwrap(); + table.remove(key.as_ref()).unwrap(); + } + fn commit(self) { + self.commit().unwrap() + } +} + +impl Get for Arc { + fn get(&self, key: impl AsRef<[u8]>) -> Option> { + let txn = self.begin_read().unwrap(); + let Ok(table) = txn.open_table::<&[u8], Vec>(TableDefinition::new("default")) else { + return None; + }; + table.get(key.as_ref()).unwrap().map(|value| value.value().to_vec()) + } +} +impl Db for Arc { + type Transaction<'a> = WriteTransaction<'a>; + fn txn(&mut self) -> Self::Transaction<'_> { + self.begin_write().unwrap() + } +} + +pub fn new_redb(path: &str) -> Arc { + Arc::new(Redb::create(path).unwrap()) +} diff --git a/coordinator/Cargo.toml b/coordinator/Cargo.toml index 460e5d8bf..feb775ec6 100644 --- a/coordinator/Cargo.toml +++ b/coordinator/Cargo.toml @@ -30,7 +30,7 @@ frost-schnorrkel = { path = "../crypto/schnorrkel" } scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["std", "derive"] } -serai-db = { path = "../common/db", features = ["rocksdb"] } +serai-db = { path = "../common/db" } serai-env = { path = "../common/env" } processor-messages = { package = "serai-processor-messages", path = "../processor/messages" } @@ -55,3 +55,7 @@ futures = { version = "0.3", default-features = false, features = ["std"] } tributary = { package = "tributary-chain", path = "./tributary", features = ["tests"] } sp-application-crypto = { git = "https://github.com/serai-dex/substrate", default-features = false, features = ["std"] } sp-runtime = { git = "https://github.com/serai-dex/substrate", default-features = false, features = ["std"] } + +[features] +redb = ["serai-db/redb"] +rocksdb = ["serai-db/rocksdb"] diff --git a/coordinator/src/main.rs b/coordinator/src/main.rs index 4734b5194..2dd8f9b72 100644 --- a/coordinator/src/main.rs +++ b/coordinator/src/main.rs @@ -16,7 +16,6 @@ use schnorr::SchnorrSignature; use frost::Participant; use serai_db::{DbTxn, Db}; -use serai_env as env; use scale::Encode; use serai_client::{ @@ -1199,7 +1198,17 @@ async fn main() { log::info!("starting coordinator service..."); - let db = serai_db::new_rocksdb(&env::var("DB_PATH").expect("path to DB wasn't specified")); + #[allow(unused_variables, unreachable_code)] + let db = { + #[cfg(all(feature = "redb", feature = "rocksdb"))] + panic!("built with redb and rocksdb"); + #[cfg(all(feature = "redb", not(feature = "rocksdb")))] + let db = serai_db::new_redb(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); + #[cfg(feature = "rocksdb")] + let db = + serai_db::new_rocksdb(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); + db + }; let key = { let mut key_hex = serai_env::var("SERAI_KEY").expect("Serai key wasn't provided"); diff --git a/message-queue/Cargo.toml b/message-queue/Cargo.toml index 77fb5264e..6a7fd97ad 100644 --- a/message-queue/Cargo.toml +++ b/message-queue/Cargo.toml @@ -37,11 +37,12 @@ env_logger = { version = "0.10", default-features = false, features = ["humantim # Uses a single threaded runtime since this shouldn't ever be CPU-bound tokio = { version = "1", default-features = false, features = ["rt", "time", "io-util", "net", "macros"] } -serai-db = { path = "../common/db", features = ["rocksdb"], optional = true } +serai-db = { path = "../common/db", optional = true } serai-env = { path = "../common/env" } serai-primitives = { path = "../substrate/primitives", features = ["borsh"] } [features] -binaries = ["serai-db"] +redb = ["serai-db/redb"] +rocksdb = ["serai-db/rocksdb"] diff --git a/message-queue/src/main.rs b/message-queue/src/main.rs index c3ca70039..30867fddb 100644 --- a/message-queue/src/main.rs +++ b/message-queue/src/main.rs @@ -169,7 +169,17 @@ async fn main() { log::info!("Starting message-queue service..."); // Open the DB - let db = serai_db::new_rocksdb(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); + #[allow(unused_variables, unreachable_code)] + let db = { + #[cfg(all(feature = "redb", feature = "rocksdb"))] + panic!("built with redb and rocksdb"); + #[cfg(all(feature = "redb", not(feature = "rocksdb")))] + let db = serai_db::new_redb(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); + #[cfg(feature = "rocksdb")] + let db = + serai_db::new_rocksdb(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); + db + }; let read_key = |str| { let key = serai_env::var(str)?; diff --git a/processor/Cargo.toml b/processor/Cargo.toml index a95567f3f..efa8ee0c5 100644 --- a/processor/Cargo.toml +++ b/processor/Cargo.toml @@ -52,7 +52,7 @@ log = { version = "0.4", default-features = false, features = ["std"] } env_logger = { version = "0.10", default-features = false, features = ["humantime"], optional = true } tokio = { version = "1", default-features = false, features = ["rt-multi-thread", "sync", "time", "macros"] } -serai-db = { path = "../common/db", features = ["rocksdb"], optional = true } +serai-db = { path = "../common/db", optional = true } serai-env = { path = "../common/env", optional = true } # TODO: Replace with direct usage of primitives serai-client = { path = "../substrate/client", default-features = false } @@ -78,4 +78,6 @@ bitcoin = ["dep:secp256k1", "secp256k1", "bitcoin-serai", "serai-client/bitcoin" ed25519 = ["dalek-ff-group", "frost/ed25519"] monero = ["ed25519", "monero-serai", "serai-client/monero"] -binaries = ["env_logger", "serai-db", "serai-env", "messages", "message-queue"] +binaries = ["env_logger", "serai-env", "messages", "message-queue"] +redb = ["serai-db/redb"] +rocksdb = ["serai-db/rocksdb"] diff --git a/processor/src/main.rs b/processor/src/main.rs index 8c63b2ae2..cd022c72e 100644 --- a/processor/src/main.rs +++ b/processor/src/main.rs @@ -679,7 +679,17 @@ async fn main() { } env_logger::init(); - let db = serai_db::new_rocksdb(&env::var("DB_PATH").expect("path to DB wasn't specified")); + #[allow(unused_variables, unreachable_code)] + let db = { + #[cfg(all(feature = "redb", feature = "rocksdb"))] + panic!("built with redb and rocksdb"); + #[cfg(all(feature = "redb", not(feature = "rocksdb")))] + let db = serai_db::new_redb(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); + #[cfg(feature = "rocksdb")] + let db = + serai_db::new_rocksdb(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); + db + }; // Network configuration let url = { From 358dbf26efe5a061055403adebb3b6ee7045058a Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Sun, 26 Nov 2023 05:55:05 -0500 Subject: [PATCH 02/11] Correct feature flagging of coordinator and message-queue in Dockerfiles --- orchestration/coordinator/Dockerfile | 2 +- orchestration/coordinator/Dockerfile.coordinator | 2 +- orchestration/message-queue/Dockerfile | 2 +- orchestration/message-queue/Dockerfile.message-queue | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/orchestration/coordinator/Dockerfile b/orchestration/coordinator/Dockerfile index 44be31160..3a929270c 100644 --- a/orchestration/coordinator/Dockerfile +++ b/orchestration/coordinator/Dockerfile @@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cargo \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/serai/target \ mkdir /serai/bin && \ - cargo build -p serai-coordinator --all-features && \ + cargo build -p serai-coordinator --features redb && \ mv /serai/target/debug/serai-coordinator /serai/bin FROM debian:bookworm-slim as image diff --git a/orchestration/coordinator/Dockerfile.coordinator b/orchestration/coordinator/Dockerfile.coordinator index c3cbbb45c..7ef6129e6 100644 --- a/orchestration/coordinator/Dockerfile.coordinator +++ b/orchestration/coordinator/Dockerfile.coordinator @@ -1,2 +1,2 @@ - cargo build -p serai-coordinator --all-features && \ + cargo build -p serai-coordinator --features redb && \ mv /serai/target/debug/serai-coordinator /serai/bin diff --git a/orchestration/message-queue/Dockerfile b/orchestration/message-queue/Dockerfile index c5ea8a6d9..08336e96c 100644 --- a/orchestration/message-queue/Dockerfile +++ b/orchestration/message-queue/Dockerfile @@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cargo \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/serai/target \ mkdir /serai/bin && \ - cargo build --all-features -p serai-message-queue && \ + cargo build --features "binaries redb" -p serai-message-queue && \ mv /serai/target/debug/serai-message-queue /serai/bin FROM debian:bookworm-slim as image diff --git a/orchestration/message-queue/Dockerfile.message-queue b/orchestration/message-queue/Dockerfile.message-queue index beb697202..798502365 100644 --- a/orchestration/message-queue/Dockerfile.message-queue +++ b/orchestration/message-queue/Dockerfile.message-queue @@ -1,2 +1,2 @@ - cargo build --all-features -p serai-message-queue && \ + cargo build --features "binaries redb" -p serai-message-queue && \ mv /serai/target/debug/serai-message-queue /serai/bin From 1f6f08f51b462bfbf400198840f0e90a37abdb74 Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Sun, 26 Nov 2023 06:33:24 -0500 Subject: [PATCH 03/11] Correct message-queue DB type alias --- message-queue/src/main.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/message-queue/src/main.rs b/message-queue/src/main.rs index 30867fddb..e2e0795a6 100644 --- a/message-queue/src/main.rs +++ b/message-queue/src/main.rs @@ -26,6 +26,9 @@ mod binaries { pub(crate) use crate::queue::Queue; + #[cfg(all(feature = "redb", not(feature = "rocksdb")))] + pub(crate) type Db = Arc; + #[cfg(feature = "rocksdb")] pub(crate) type Db = serai_db::RocksDB; #[allow(clippy::type_complexity)] From 2ff0a7d49a483e19da517f77608fecd269969161 Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Sun, 26 Nov 2023 07:44:39 -0500 Subject: [PATCH 04/11] Use consistent table typing in redb --- common/db/src/redb.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/db/src/redb.rs b/common/db/src/redb.rs index c34ff4ad5..850f8436a 100644 --- a/common/db/src/redb.rs +++ b/common/db/src/redb.rs @@ -13,8 +13,8 @@ impl Get for WriteTransaction<'_> { } impl DbTxn for WriteTransaction<'_> { fn put(&mut self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) { - let mut table = self.open_table::<&[u8], &[u8]>(TableDefinition::new("default")).unwrap(); - table.insert(key.as_ref(), value.as_ref()).unwrap(); + let mut table = self.open_table::<&[u8], Vec>(TableDefinition::new("default")).unwrap(); + table.insert(key.as_ref(), &value.as_ref().to_vec()).unwrap(); } fn del(&mut self, key: impl AsRef<[u8]>) { let mut table = self.open_table::<&[u8], Vec>(TableDefinition::new("default")).unwrap(); From 8378a104abb7bc9f732b48203a6bf12555561dc5 Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Wed, 29 Nov 2023 00:32:01 -0500 Subject: [PATCH 05/11] Correct rebase artifacts --- Cargo.lock | 12 +++++++++++- orchestration/message-queue/Dockerfile | 2 +- orchestration/message-queue/Dockerfile.message-queue | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4236e699..03c73de6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4718,7 +4718,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c11e44798ad209ccdd91fc192f0526a369a01234f7373e1b141c96d7cee4f0e" dependencies = [ - "proc-macro-crate 1.3.1", + "proc-macro-crate 2.0.0", "proc-macro2", "quote", "syn 2.0.39", @@ -5714,6 +5714,15 @@ dependencies = [ "yasna", ] +[[package]] +name = "redb" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08837f9a129bde83c51953b8c96cbb3422b940166b730caa954836106eb1dfd2" +dependencies = [ + "libc", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -7372,6 +7381,7 @@ dependencies = [ name = "serai-db" version = "0.1.0" dependencies = [ + "redb", "rocksdb", ] diff --git a/orchestration/message-queue/Dockerfile b/orchestration/message-queue/Dockerfile index 08336e96c..6ececc607 100644 --- a/orchestration/message-queue/Dockerfile +++ b/orchestration/message-queue/Dockerfile @@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cargo \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/serai/target \ mkdir /serai/bin && \ - cargo build --features "binaries redb" -p serai-message-queue && \ + cargo build --features redb -p serai-message-queue && \ mv /serai/target/debug/serai-message-queue /serai/bin FROM debian:bookworm-slim as image diff --git a/orchestration/message-queue/Dockerfile.message-queue b/orchestration/message-queue/Dockerfile.message-queue index 798502365..4047339a4 100644 --- a/orchestration/message-queue/Dockerfile.message-queue +++ b/orchestration/message-queue/Dockerfile.message-queue @@ -1,2 +1,2 @@ - cargo build --features "binaries redb" -p serai-message-queue && \ + cargo build --features redb -p serai-message-queue && \ mv /serai/target/debug/serai-message-queue /serai/bin From 83a045dd29dcc2096afc70461f05f54da1fff7dc Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Wed, 29 Nov 2023 01:26:37 -0500 Subject: [PATCH 06/11] Correct removal of binaries feature from message-queue --- message-queue/src/main.rs | 255 ++++++++++++++++++-------------------- 1 file changed, 121 insertions(+), 134 deletions(-) diff --git a/message-queue/src/main.rs b/message-queue/src/main.rs index e2e0795a6..f6200793c 100644 --- a/message-queue/src/main.rs +++ b/message-queue/src/main.rs @@ -1,157 +1,149 @@ -#[cfg(feature = "binaries")] mod messages; -#[cfg(feature = "binaries")] mod queue; -#[cfg(feature = "binaries")] -mod binaries { - pub(crate) use std::{ - sync::{Arc, RwLock}, - collections::HashMap, - }; +pub(crate) use std::{ + sync::{Arc, RwLock}, + collections::HashMap, +}; - pub(crate) use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto}; - pub(crate) use schnorr_signatures::SchnorrSignature; +pub(crate) use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto}; +pub(crate) use schnorr_signatures::SchnorrSignature; - pub(crate) use serai_primitives::NetworkId; +pub(crate) use serai_primitives::NetworkId; - pub(crate) use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpListener, - }; +pub(crate) use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, +}; - use serai_db::{Get, DbTxn, Db as DbTrait}; +use serai_db::{Get, DbTxn, Db as DbTrait}; - pub(crate) use crate::messages::*; +pub(crate) use crate::messages::*; - pub(crate) use crate::queue::Queue; +pub(crate) use crate::queue::Queue; - #[cfg(all(feature = "redb", not(feature = "rocksdb")))] - pub(crate) type Db = Arc; - #[cfg(feature = "rocksdb")] - pub(crate) type Db = serai_db::RocksDB; +#[cfg(all(feature = "redb", not(feature = "rocksdb")))] +pub(crate) type Db = Arc; +#[cfg(feature = "rocksdb")] +pub(crate) type Db = serai_db::RocksDB; - #[allow(clippy::type_complexity)] - mod clippy { - use super::*; - use once_cell::sync::Lazy; - pub(crate) static KEYS: Lazy::G>>>> = - Lazy::new(|| Arc::new(RwLock::new(HashMap::new()))); - pub(crate) static QUEUES: Lazy>>>>> = - Lazy::new(|| Arc::new(RwLock::new(HashMap::new()))); +#[allow(clippy::type_complexity)] +mod clippy { + use super::*; + use once_cell::sync::Lazy; + pub(crate) static KEYS: Lazy::G>>>> = + Lazy::new(|| Arc::new(RwLock::new(HashMap::new()))); + pub(crate) static QUEUES: Lazy>>>>> = + Lazy::new(|| Arc::new(RwLock::new(HashMap::new()))); +} +pub(crate) use self::clippy::*; + +// queue RPC method +/* + Queues a message to be delivered from a processor to a coordinator, or vice versa. + + Messages are authenticated to be coming from the claimed service. Recipient services SHOULD + independently verify signatures. + + The metadata specifies an intent. Only one message, for a specified intent, will be delivered. + This allows services to safely send messages multiple times without them being delivered + multiple times. + + The message will be ordered by this service, with the order having no guarantees other than + successful ordering by the time this call returns. +*/ +pub(crate) fn queue_message( + db: &mut Db, + meta: Metadata, + msg: Vec, + sig: SchnorrSignature, +) { + { + let from = (*KEYS).read().unwrap()[&meta.from]; + assert!( + sig.verify(from, message_challenge(meta.from, from, meta.to, &meta.intent, &msg, sig.R)) + ); } - pub(crate) use self::clippy::*; - - // queue RPC method - /* - Queues a message to be delivered from a processor to a coordinator, or vice versa. - - Messages are authenticated to be coming from the claimed service. Recipient services SHOULD - independently verify signatures. - - The metadata specifies an intent. Only one message, for a specified intent, will be delivered. - This allows services to safely send messages multiple times without them being delivered - multiple times. - - The message will be ordered by this service, with the order having no guarantees other than - successful ordering by the time this call returns. - */ - pub(crate) fn queue_message( - db: &mut Db, - meta: Metadata, - msg: Vec, - sig: SchnorrSignature, - ) { - { - let from = (*KEYS).read().unwrap()[&meta.from]; - assert!( - sig.verify(from, message_challenge(meta.from, from, meta.to, &meta.intent, &msg, sig.R)) - ); - } - // Assert one, and only one of these, is the coordinator - assert!(matches!(meta.from, Service::Coordinator) ^ matches!(meta.to, Service::Coordinator)); + // Assert one, and only one of these, is the coordinator + assert!(matches!(meta.from, Service::Coordinator) ^ matches!(meta.to, Service::Coordinator)); - // Verify (from, to, intent) hasn't been prior seen - fn key(domain: &'static [u8], key: impl AsRef<[u8]>) -> Vec { - [&[u8::try_from(domain.len()).unwrap()], domain, key.as_ref()].concat() - } - fn intent_key(from: Service, to: Service, intent: &[u8]) -> Vec { - key(b"intent_seen", borsh::to_vec(&(from, to, intent)).unwrap()) - } - let mut txn = db.txn(); - let intent_key = intent_key(meta.from, meta.to, &meta.intent); - if Get::get(&txn, &intent_key).is_some() { - log::warn!( - "Prior queued message attempted to be queued again. From: {:?} To: {:?} Intent: {}", - meta.from, - meta.to, - hex::encode(&meta.intent) - ); - return; - } - DbTxn::put(&mut txn, intent_key, []); - - // Queue it - let id = (*QUEUES).read().unwrap()[&(meta.from, meta.to)].write().unwrap().queue_message( - &mut txn, - QueuedMessage { - from: meta.from, - // Temporary value which queue_message will override - id: u64::MAX, - msg, - sig: sig.serialize(), - }, + // Verify (from, to, intent) hasn't been prior seen + fn key(domain: &'static [u8], key: impl AsRef<[u8]>) -> Vec { + [&[u8::try_from(domain.len()).unwrap()], domain, key.as_ref()].concat() + } + fn intent_key(from: Service, to: Service, intent: &[u8]) -> Vec { + key(b"intent_seen", borsh::to_vec(&(from, to, intent)).unwrap()) + } + let mut txn = db.txn(); + let intent_key = intent_key(meta.from, meta.to, &meta.intent); + if Get::get(&txn, &intent_key).is_some() { + log::warn!( + "Prior queued message attempted to be queued again. From: {:?} To: {:?} Intent: {}", + meta.from, + meta.to, + hex::encode(&meta.intent) ); - - log::info!("Queued message. From: {:?} To: {:?} ID: {id}", meta.from, meta.to); - DbTxn::commit(txn); + return; } + DbTxn::put(&mut txn, intent_key, []); + + // Queue it + let id = (*QUEUES).read().unwrap()[&(meta.from, meta.to)].write().unwrap().queue_message( + &mut txn, + QueuedMessage { + from: meta.from, + // Temporary value which queue_message will override + id: u64::MAX, + msg, + sig: sig.serialize(), + }, + ); + + log::info!("Queued message. From: {:?} To: {:?} ID: {id}", meta.from, meta.to); + DbTxn::commit(txn); +} - // next RPC method - /* - Gets the next message in queue for the named services. - - This is not authenticated due to the fact every nonce would have to be saved to prevent - replays, or a challenge-response protocol implemented. Neither are worth doing when there - should be no sensitive data on this server. - */ - pub(crate) fn get_next_message(from: Service, to: Service) -> Option { - let queue_outer = (*QUEUES).read().unwrap(); - let queue = queue_outer[&(from, to)].read().unwrap(); - let next = queue.last_acknowledged().map(|i| i + 1).unwrap_or(0); - queue.get_message(next) - } +// next RPC method +/* + Gets the next message in queue for the named services. + + This is not authenticated due to the fact every nonce would have to be saved to prevent + replays, or a challenge-response protocol implemented. Neither are worth doing when there + should be no sensitive data on this server. +*/ +pub(crate) fn get_next_message(from: Service, to: Service) -> Option { + let queue_outer = (*QUEUES).read().unwrap(); + let queue = queue_outer[&(from, to)].read().unwrap(); + let next = queue.last_acknowledged().map(|i| i + 1).unwrap_or(0); + queue.get_message(next) +} - // ack RPC method - /* - Acknowledges a message as received and handled, meaning it'll no longer be returned as the next - message. - */ - pub(crate) fn ack_message(from: Service, to: Service, id: u64, sig: SchnorrSignature) { - { - let to_key = (*KEYS).read().unwrap()[&to]; - assert!(sig.verify(to_key, ack_challenge(to, to_key, from, id, sig.R))); - } +// ack RPC method +/* + Acknowledges a message as received and handled, meaning it'll no longer be returned as the next + message. +*/ +pub(crate) fn ack_message(from: Service, to: Service, id: u64, sig: SchnorrSignature) { + { + let to_key = (*KEYS).read().unwrap()[&to]; + assert!(sig.verify(to_key, ack_challenge(to, to_key, from, id, sig.R))); + } - // Is it: - // The acknowledged message should be > last acknowledged OR - // The acknowledged message should be >= - // It's the first if we save messages as acknowledged before acknowledging them - // It's the second if we acknowledge messages before saving them as acknowledged - // TODO: Check only a proper message is being acked + // Is it: + // The acknowledged message should be > last acknowledged OR + // The acknowledged message should be >= + // It's the first if we save messages as acknowledged before acknowledging them + // It's the second if we acknowledge messages before saving them as acknowledged + // TODO: Check only a proper message is being acked - log::info!("Acknowledging From: {:?} To: {:?} ID: {}", from, to, id); + log::info!("Acknowledging From: {:?} To: {:?} ID: {}", from, to, id); - (*QUEUES).read().unwrap()[&(from, to)].write().unwrap().ack_message(id) - } + (*QUEUES).read().unwrap()[&(from, to)].write().unwrap().ack_message(id) } -#[cfg(feature = "binaries")] #[tokio::main(flavor = "current_thread")] async fn main() { - use binaries::*; - // Override the panic handler with one which will panic if any tokio task panics { let existing = std::panic::take_hook(); @@ -285,8 +277,3 @@ async fn main() { }); } } - -#[cfg(not(feature = "binaries"))] -fn main() { - panic!("To run binaries, please build with `--feature binaries`."); -} From 3b7bc5af6f7e2dbc2c378ec328d740d3de49744c Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Wed, 29 Nov 2023 23:05:21 -0500 Subject: [PATCH 07/11] Correct processor feature flagging --- orchestration/processor/bitcoin/Dockerfile | 2 +- orchestration/processor/bitcoin/Dockerfile.processor.bitcoin | 2 +- orchestration/processor/monero/Dockerfile | 2 +- orchestration/processor/monero/Dockerfile.processor.monero | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/orchestration/processor/bitcoin/Dockerfile b/orchestration/processor/bitcoin/Dockerfile index f7d1c9c10..ffbde36cb 100644 --- a/orchestration/processor/bitcoin/Dockerfile +++ b/orchestration/processor/bitcoin/Dockerfile @@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cargo \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/serai/target \ mkdir /serai/bin && \ - cargo build --features "binaries bitcoin" -p serai-processor && \ + cargo build --features "binaries redb bitcoin" -p serai-processor && \ mv /serai/target/debug/serai-processor /serai/bin FROM debian:bookworm-slim as image diff --git a/orchestration/processor/bitcoin/Dockerfile.processor.bitcoin b/orchestration/processor/bitcoin/Dockerfile.processor.bitcoin index 9ed38e036..ab6eddf05 100644 --- a/orchestration/processor/bitcoin/Dockerfile.processor.bitcoin +++ b/orchestration/processor/bitcoin/Dockerfile.processor.bitcoin @@ -1,2 +1,2 @@ - cargo build --features "binaries bitcoin" -p serai-processor && \ + cargo build --features "binaries redb bitcoin" -p serai-processor && \ mv /serai/target/debug/serai-processor /serai/bin diff --git a/orchestration/processor/monero/Dockerfile b/orchestration/processor/monero/Dockerfile index cf531aa95..5355e4c75 100644 --- a/orchestration/processor/monero/Dockerfile +++ b/orchestration/processor/monero/Dockerfile @@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cargo \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/serai/target \ mkdir /serai/bin && \ - cargo build --features "binaries monero" -p serai-processor && \ + cargo build --features "binaries redb monero" -p serai-processor && \ mv /serai/target/debug/serai-processor /serai/bin FROM debian:bookworm-slim as image diff --git a/orchestration/processor/monero/Dockerfile.processor.monero b/orchestration/processor/monero/Dockerfile.processor.monero index 31d0a811f..f62a5cc49 100644 --- a/orchestration/processor/monero/Dockerfile.processor.monero +++ b/orchestration/processor/monero/Dockerfile.processor.monero @@ -1,2 +1,2 @@ - cargo build --features "binaries monero" -p serai-processor && \ + cargo build --features "binaries redb monero" -p serai-processor && \ mv /serai/target/debug/serai-processor /serai/bin From ca17342344ea8f304c4b1a99a1787907ba45fb93 Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Wed, 29 Nov 2023 23:54:53 -0500 Subject: [PATCH 08/11] Replace redb with parity-db It still has much better compile times yet doesn't block when creating multiple transactions. It also is actively maintained and doesn't grow our tree. The MPT aspects are irrelevant. --- Cargo.lock | 11 +---- common/db/Cargo.toml | 4 +- common/db/src/lib.rs | 8 ++-- common/db/src/parity_db.rs | 41 +++++++++++++++++ common/db/src/redb.rs | 46 ------------------- coordinator/Cargo.toml | 2 +- coordinator/src/main.rs | 9 ++-- message-queue/Cargo.toml | 2 +- message-queue/src/main.rs | 11 +++-- orchestration/coordinator/Dockerfile | 2 +- .../coordinator/Dockerfile.coordinator | 2 +- orchestration/message-queue/Dockerfile | 2 +- .../message-queue/Dockerfile.message-queue | 2 +- orchestration/processor/bitcoin/Dockerfile | 2 +- .../bitcoin/Dockerfile.processor.bitcoin | 2 +- orchestration/processor/monero/Dockerfile | 2 +- .../monero/Dockerfile.processor.monero | 2 +- processor/Cargo.toml | 2 +- processor/src/main.rs | 9 ++-- 19 files changed, 75 insertions(+), 86 deletions(-) create mode 100644 common/db/src/parity_db.rs delete mode 100644 common/db/src/redb.rs diff --git a/Cargo.lock b/Cargo.lock index 03c73de6e..8937c439f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5714,15 +5714,6 @@ dependencies = [ "yasna", ] -[[package]] -name = "redb" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08837f9a129bde83c51953b8c96cbb3422b940166b730caa954836106eb1dfd2" -dependencies = [ - "libc", -] - [[package]] name = "redox_syscall" version = "0.2.16" @@ -7381,7 +7372,7 @@ dependencies = [ name = "serai-db" version = "0.1.0" dependencies = [ - "redb", + "parity-db", "rocksdb", ] diff --git a/common/db/Cargo.toml b/common/db/Cargo.toml index 7aad11270..9833f755c 100644 --- a/common/db/Cargo.toml +++ b/common/db/Cargo.toml @@ -14,9 +14,9 @@ all-features = true rustdoc-args = ["--cfg", "docsrs"] [dependencies] -redb = { version = "1", default-features = false, optional = true } +parity-db = { version = "0.4", default-features = false, optional = true } rocksdb = { version = "0.21", default-features = false, features = ["lz4"], optional = true } [features] -redb = ["dep:redb"] +parity-db = ["dep:parity-db"] rocksdb = ["dep:rocksdb"] diff --git a/common/db/src/lib.rs b/common/db/src/lib.rs index df6f1394c..447542177 100644 --- a/common/db/src/lib.rs +++ b/common/db/src/lib.rs @@ -9,10 +9,10 @@ mod rocks; #[cfg(feature = "rocksdb")] pub use rocks::{RocksDB, new_rocksdb}; -#[cfg(feature = "redb")] -mod redb; -#[cfg(feature = "redb")] -pub use redb::{Redb, new_redb}; +#[cfg(feature = "parity_db")] +mod parity_db; +#[cfg(feature = "parity_db")] +pub use parity_db::{ParityDb, new_parity_db}; /// An object implementing get. pub trait Get { diff --git a/common/db/src/parity_db.rs b/common/db/src/parity_db.rs new file mode 100644 index 000000000..217482846 --- /dev/null +++ b/common/db/src/parity_db.rs @@ -0,0 +1,41 @@ +use std::sync::Arc; + +pub use ::parity_db::Db as ParityDb; +use ::redb::*; + +use crate::*; + +pub struct Transaction<'a>(&'a Arc, Vec<(u8, Vec, Option>)>); + +impl Get for Transaction<'_> { + fn get(&self, key: impl AsRef<[u8]>) -> Option> { + self.0.get(key) + } +} +impl DbTxn for Transaction<'_> { + fn put(&mut self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) { + self.1.push((0, key.as_ref().to_vec(), Some(value.as_ref().to_vec()))) + } + fn del(&mut self, key: impl AsRef<[u8]>) { + self.1.push((0, key.as_ref().to_vec(), None)) + } + fn commit(self) { + self.0.commit(self.1).unwrap() + } +} + +impl Get for Arc { + fn get(&self, key: impl AsRef<[u8]>) -> Option> { + ParityDb::get(&*self, 0, key.as_ref()).unwrap() + } +} +impl Db for Arc { + type Transaction<'a> = Transaction<'a>; + fn txn(&mut self) -> Self::Transaction<'_> { + Transaction(self, vec![]) + } +} + +pub fn new_parity_db(path: &str) -> Arc { + Arc::new(ParityDb::open_or_crate(Options::with_columns(path, 1)).unwrap()) +} diff --git a/common/db/src/redb.rs b/common/db/src/redb.rs deleted file mode 100644 index 850f8436a..000000000 --- a/common/db/src/redb.rs +++ /dev/null @@ -1,46 +0,0 @@ -use std::sync::Arc; - -pub use ::redb::Database as Redb; -use ::redb::*; - -use crate::*; - -impl Get for WriteTransaction<'_> { - fn get(&self, key: impl AsRef<[u8]>) -> Option> { - let table = self.open_table::<&[u8], Vec>(TableDefinition::new("default")).unwrap(); - table.get(key.as_ref()).unwrap().map(|value| value.value().to_vec()) - } -} -impl DbTxn for WriteTransaction<'_> { - fn put(&mut self, key: impl AsRef<[u8]>, value: impl AsRef<[u8]>) { - let mut table = self.open_table::<&[u8], Vec>(TableDefinition::new("default")).unwrap(); - table.insert(key.as_ref(), &value.as_ref().to_vec()).unwrap(); - } - fn del(&mut self, key: impl AsRef<[u8]>) { - let mut table = self.open_table::<&[u8], Vec>(TableDefinition::new("default")).unwrap(); - table.remove(key.as_ref()).unwrap(); - } - fn commit(self) { - self.commit().unwrap() - } -} - -impl Get for Arc { - fn get(&self, key: impl AsRef<[u8]>) -> Option> { - let txn = self.begin_read().unwrap(); - let Ok(table) = txn.open_table::<&[u8], Vec>(TableDefinition::new("default")) else { - return None; - }; - table.get(key.as_ref()).unwrap().map(|value| value.value().to_vec()) - } -} -impl Db for Arc { - type Transaction<'a> = WriteTransaction<'a>; - fn txn(&mut self) -> Self::Transaction<'_> { - self.begin_write().unwrap() - } -} - -pub fn new_redb(path: &str) -> Arc { - Arc::new(Redb::create(path).unwrap()) -} diff --git a/coordinator/Cargo.toml b/coordinator/Cargo.toml index feb775ec6..525ee1ba6 100644 --- a/coordinator/Cargo.toml +++ b/coordinator/Cargo.toml @@ -57,5 +57,5 @@ sp-application-crypto = { git = "https://github.com/serai-dex/substrate", defaul sp-runtime = { git = "https://github.com/serai-dex/substrate", default-features = false, features = ["std"] } [features] -redb = ["serai-db/redb"] +parity-db = ["serai-db/parity-db"] rocksdb = ["serai-db/rocksdb"] diff --git a/coordinator/src/main.rs b/coordinator/src/main.rs index 2dd8f9b72..203d29e72 100644 --- a/coordinator/src/main.rs +++ b/coordinator/src/main.rs @@ -1200,10 +1200,11 @@ async fn main() { #[allow(unused_variables, unreachable_code)] let db = { - #[cfg(all(feature = "redb", feature = "rocksdb"))] - panic!("built with redb and rocksdb"); - #[cfg(all(feature = "redb", not(feature = "rocksdb")))] - let db = serai_db::new_redb(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); + #[cfg(all(feature = "parity-db", feature = "rocksdb"))] + panic!("built with parity-db and rocksdb"); + #[cfg(all(feature = "parity-db", not(feature = "rocksdb")))] + let db = + serai_db::new_parity_db(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); #[cfg(feature = "rocksdb")] let db = serai_db::new_rocksdb(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); diff --git a/message-queue/Cargo.toml b/message-queue/Cargo.toml index 6a7fd97ad..b775d6a97 100644 --- a/message-queue/Cargo.toml +++ b/message-queue/Cargo.toml @@ -44,5 +44,5 @@ serai-env = { path = "../common/env" } serai-primitives = { path = "../substrate/primitives", features = ["borsh"] } [features] -redb = ["serai-db/redb"] +parity-db = ["serai-db/parity-db"] rocksdb = ["serai-db/rocksdb"] diff --git a/message-queue/src/main.rs b/message-queue/src/main.rs index f6200793c..bc96d6e80 100644 --- a/message-queue/src/main.rs +++ b/message-queue/src/main.rs @@ -22,7 +22,7 @@ pub(crate) use crate::messages::*; pub(crate) use crate::queue::Queue; -#[cfg(all(feature = "redb", not(feature = "rocksdb")))] +#[cfg(all(feature = "parity-db", not(feature = "rocksdb")))] pub(crate) type Db = Arc; #[cfg(feature = "rocksdb")] pub(crate) type Db = serai_db::RocksDB; @@ -166,10 +166,11 @@ async fn main() { // Open the DB #[allow(unused_variables, unreachable_code)] let db = { - #[cfg(all(feature = "redb", feature = "rocksdb"))] - panic!("built with redb and rocksdb"); - #[cfg(all(feature = "redb", not(feature = "rocksdb")))] - let db = serai_db::new_redb(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); + #[cfg(all(feature = "parity-db", feature = "rocksdb"))] + panic!("built with parity-db and rocksdb"); + #[cfg(all(feature = "parity-db", not(feature = "rocksdb")))] + let db = + serai_db::new_parity_db(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); #[cfg(feature = "rocksdb")] let db = serai_db::new_rocksdb(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); diff --git a/orchestration/coordinator/Dockerfile b/orchestration/coordinator/Dockerfile index 3a929270c..f941d8cad 100644 --- a/orchestration/coordinator/Dockerfile +++ b/orchestration/coordinator/Dockerfile @@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cargo \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/serai/target \ mkdir /serai/bin && \ - cargo build -p serai-coordinator --features redb && \ + cargo build -p serai-coordinator --features parity-db && \ mv /serai/target/debug/serai-coordinator /serai/bin FROM debian:bookworm-slim as image diff --git a/orchestration/coordinator/Dockerfile.coordinator b/orchestration/coordinator/Dockerfile.coordinator index 7ef6129e6..8fca56220 100644 --- a/orchestration/coordinator/Dockerfile.coordinator +++ b/orchestration/coordinator/Dockerfile.coordinator @@ -1,2 +1,2 @@ - cargo build -p serai-coordinator --features redb && \ + cargo build -p serai-coordinator --features parity-db && \ mv /serai/target/debug/serai-coordinator /serai/bin diff --git a/orchestration/message-queue/Dockerfile b/orchestration/message-queue/Dockerfile index 6ececc607..02c859bcf 100644 --- a/orchestration/message-queue/Dockerfile +++ b/orchestration/message-queue/Dockerfile @@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cargo \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/serai/target \ mkdir /serai/bin && \ - cargo build --features redb -p serai-message-queue && \ + cargo build --features parity-db -p serai-message-queue && \ mv /serai/target/debug/serai-message-queue /serai/bin FROM debian:bookworm-slim as image diff --git a/orchestration/message-queue/Dockerfile.message-queue b/orchestration/message-queue/Dockerfile.message-queue index 4047339a4..a40fad494 100644 --- a/orchestration/message-queue/Dockerfile.message-queue +++ b/orchestration/message-queue/Dockerfile.message-queue @@ -1,2 +1,2 @@ - cargo build --features redb -p serai-message-queue && \ + cargo build --features parity-db -p serai-message-queue && \ mv /serai/target/debug/serai-message-queue /serai/bin diff --git a/orchestration/processor/bitcoin/Dockerfile b/orchestration/processor/bitcoin/Dockerfile index ffbde36cb..b23e29138 100644 --- a/orchestration/processor/bitcoin/Dockerfile +++ b/orchestration/processor/bitcoin/Dockerfile @@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cargo \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/serai/target \ mkdir /serai/bin && \ - cargo build --features "binaries redb bitcoin" -p serai-processor && \ + cargo build --features "binaries parity-db bitcoin" -p serai-processor && \ mv /serai/target/debug/serai-processor /serai/bin FROM debian:bookworm-slim as image diff --git a/orchestration/processor/bitcoin/Dockerfile.processor.bitcoin b/orchestration/processor/bitcoin/Dockerfile.processor.bitcoin index ab6eddf05..685eaf7f9 100644 --- a/orchestration/processor/bitcoin/Dockerfile.processor.bitcoin +++ b/orchestration/processor/bitcoin/Dockerfile.processor.bitcoin @@ -1,2 +1,2 @@ - cargo build --features "binaries redb bitcoin" -p serai-processor && \ + cargo build --features "binaries parity-db bitcoin" -p serai-processor && \ mv /serai/target/debug/serai-processor /serai/bin diff --git a/orchestration/processor/monero/Dockerfile b/orchestration/processor/monero/Dockerfile index 5355e4c75..1fff6593b 100644 --- a/orchestration/processor/monero/Dockerfile +++ b/orchestration/processor/monero/Dockerfile @@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cargo \ --mount=type=cache,target=/usr/local/cargo/git \ --mount=type=cache,target=/serai/target \ mkdir /serai/bin && \ - cargo build --features "binaries redb monero" -p serai-processor && \ + cargo build --features "binaries parity-db monero" -p serai-processor && \ mv /serai/target/debug/serai-processor /serai/bin FROM debian:bookworm-slim as image diff --git a/orchestration/processor/monero/Dockerfile.processor.monero b/orchestration/processor/monero/Dockerfile.processor.monero index f62a5cc49..920097f12 100644 --- a/orchestration/processor/monero/Dockerfile.processor.monero +++ b/orchestration/processor/monero/Dockerfile.processor.monero @@ -1,2 +1,2 @@ - cargo build --features "binaries redb monero" -p serai-processor && \ + cargo build --features "binaries parity-db monero" -p serai-processor && \ mv /serai/target/debug/serai-processor /serai/bin diff --git a/processor/Cargo.toml b/processor/Cargo.toml index efa8ee0c5..33e910465 100644 --- a/processor/Cargo.toml +++ b/processor/Cargo.toml @@ -79,5 +79,5 @@ ed25519 = ["dalek-ff-group", "frost/ed25519"] monero = ["ed25519", "monero-serai", "serai-client/monero"] binaries = ["env_logger", "serai-env", "messages", "message-queue"] -redb = ["serai-db/redb"] +parity-db = ["serai-db/parity-db"] rocksdb = ["serai-db/rocksdb"] diff --git a/processor/src/main.rs b/processor/src/main.rs index cd022c72e..a80f93dc0 100644 --- a/processor/src/main.rs +++ b/processor/src/main.rs @@ -681,10 +681,11 @@ async fn main() { #[allow(unused_variables, unreachable_code)] let db = { - #[cfg(all(feature = "redb", feature = "rocksdb"))] - panic!("built with redb and rocksdb"); - #[cfg(all(feature = "redb", not(feature = "rocksdb")))] - let db = serai_db::new_redb(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); + #[cfg(all(feature = "parity-db", feature = "rocksdb"))] + panic!("built with parity-db and rocksdb"); + #[cfg(all(feature = "parity-db", not(feature = "rocksdb")))] + let db = + serai_db::new_parity_db(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); #[cfg(feature = "rocksdb")] let db = serai_db::new_rocksdb(&serai_env::var("DB_PATH").expect("path to DB wasn't specified")); From 76fa83c88215661b2131faa165295e3af443882a Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Thu, 30 Nov 2023 00:17:42 -0500 Subject: [PATCH 09/11] Correct stray Redb --- common/db/src/lib.rs | 4 ++-- common/db/src/parity_db.rs | 5 ++--- message-queue/src/main.rs | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/common/db/src/lib.rs b/common/db/src/lib.rs index 447542177..1c08fe3dc 100644 --- a/common/db/src/lib.rs +++ b/common/db/src/lib.rs @@ -9,9 +9,9 @@ mod rocks; #[cfg(feature = "rocksdb")] pub use rocks::{RocksDB, new_rocksdb}; -#[cfg(feature = "parity_db")] +#[cfg(feature = "parity-db")] mod parity_db; -#[cfg(feature = "parity_db")] +#[cfg(feature = "parity-db")] pub use parity_db::{ParityDb, new_parity_db}; /// An object implementing get. diff --git a/common/db/src/parity_db.rs b/common/db/src/parity_db.rs index 217482846..9859d4b8f 100644 --- a/common/db/src/parity_db.rs +++ b/common/db/src/parity_db.rs @@ -1,7 +1,6 @@ use std::sync::Arc; -pub use ::parity_db::Db as ParityDb; -use ::redb::*; +pub use ::parity_db::{Options, Db as ParityDb}; use crate::*; @@ -37,5 +36,5 @@ impl Db for Arc { } pub fn new_parity_db(path: &str) -> Arc { - Arc::new(ParityDb::open_or_crate(Options::with_columns(path, 1)).unwrap()) + Arc::new(ParityDb::open_or_create(&Options::with_columns(std::path::Path::new(path), 1)).unwrap()) } diff --git a/message-queue/src/main.rs b/message-queue/src/main.rs index bc96d6e80..25a36edfd 100644 --- a/message-queue/src/main.rs +++ b/message-queue/src/main.rs @@ -23,7 +23,7 @@ pub(crate) use crate::messages::*; pub(crate) use crate::queue::Queue; #[cfg(all(feature = "parity-db", not(feature = "rocksdb")))] -pub(crate) type Db = Arc; +pub(crate) type Db = Arc; #[cfg(feature = "rocksdb")] pub(crate) type Db = serai_db::RocksDB; From 566b1b48d53db8b9ad1312e41527d6dbab3a4bbe Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Thu, 30 Nov 2023 00:29:09 -0500 Subject: [PATCH 10/11] clippy warning --- common/db/src/parity_db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/db/src/parity_db.rs b/common/db/src/parity_db.rs index 9859d4b8f..ff13f0e37 100644 --- a/common/db/src/parity_db.rs +++ b/common/db/src/parity_db.rs @@ -25,7 +25,7 @@ impl DbTxn for Transaction<'_> { impl Get for Arc { fn get(&self, key: impl AsRef<[u8]>) -> Option> { - ParityDb::get(&*self, 0, key.as_ref()).unwrap() + ParityDb::get(self, 0, key.as_ref()).unwrap() } } impl Db for Arc { From 3f04bbd8f4f62e86fa1256aea100c00c9dd79b86 Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Thu, 30 Nov 2023 02:01:37 -0500 Subject: [PATCH 11/11] Correct txn get --- common/db/src/parity_db.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/common/db/src/parity_db.rs b/common/db/src/parity_db.rs index ff13f0e37..06fd0c7c0 100644 --- a/common/db/src/parity_db.rs +++ b/common/db/src/parity_db.rs @@ -8,7 +8,13 @@ pub struct Transaction<'a>(&'a Arc, Vec<(u8, Vec, Option>) impl Get for Transaction<'_> { fn get(&self, key: impl AsRef<[u8]>) -> Option> { - self.0.get(key) + let mut res = self.0.get(&key); + for change in &self.1 { + if change.1 == key.as_ref() { + res = change.2.clone(); + } + } + res } } impl DbTxn for Transaction<'_> {