Rewrite validator-sets with logarithmic algorithms

Uses the fact the underlying DB is sorted to achieve sorting of potential
validators by stake.

Removes release of deallocated stake for now.

Cargo.toml

@@ -46,6 +46,8 @@ members = [
   "substrate/validator-sets/primitives",
   "substrate/validator-sets/pallet",
 
+  "substrate/staking/pallet",
+
   "substrate/runtime",
   "substrate/node",
 

coordinator/src/main.rs

@@ -254,7 +254,10 @@ pub(crate) async fn scan_tributaries<
                         // TODO2: Differentiate connection errors from invariants
                         Err(e) => {
                           // Check if this failed because the keys were already set by someone else
-                          if matches!(serai.get_keys(spec.set()).await, Ok(Some(_))) {
+                          // TODO: hash_with_keys is latest, yet we'll remove old keys from storage
+                          let hash_with_keys = serai.get_latest_block_hash().await.unwrap();
+                          if matches!(serai.get_keys(spec.set(), hash_with_keys).await, Ok(Some(_)))
+                          {
                             log::info!("another coordinator set key pair for {:?}", set);
                             break;
                           }

coordinator/src/substrate/mod.rs

@@ -35,12 +35,14 @@ async fn in_set(
   key: &Zeroizing<<Ristretto as Ciphersuite>::F>,
   serai: &Serai,
   set: ValidatorSet,
+  block_hash: [u8; 32],
 ) -> Result<Option<bool>, SeraiError> {
-  let Some(data) = serai.get_validator_set(set).await? else {
+  let Some(participants) = serai.get_validator_set_participants(set.network, block_hash).await?
+  else {
     return Ok(None);
   };
   let key = (Ristretto::generator() * key.deref()).to_bytes();
-  Ok(Some(data.participants.iter().any(|(participant, _)| participant.0 == key)))
+  Ok(Some(participants.iter().any(|participant| participant.0 == key)))
 }
 
 async fn handle_new_set<D: Db, CNT: Clone + Fn(&mut D, TributarySpec)>(
@@ -51,10 +53,13 @@ async fn handle_new_set<D: Db, CNT: Clone + Fn(&mut D, TributarySpec)>(
   block: &Block,
   set: ValidatorSet,
 ) -> Result<(), SeraiError> {
-  if in_set(key, serai, set).await?.expect("NewSet for set which doesn't exist") {
+  if in_set(key, serai, set, block.hash()).await?.expect("NewSet for set which doesn't exist") {
     log::info!("present in set {:?}", set);
 
-    let set_data = serai.get_validator_set(set).await?.expect("NewSet for set which doesn't exist");
+    let set_participants = serai
+      .get_validator_set_participants(set.network, block.hash())
+      .await?
+      .expect("NewSet for set which doesn't exist");
 
     let time = if let Ok(time) = block.time() {
       time
@@ -77,7 +82,7 @@ async fn handle_new_set<D: Db, CNT: Clone + Fn(&mut D, TributarySpec)>(
     const SUBSTRATE_TO_TRIBUTARY_TIME_DELAY: u64 = 120;
     let time = time + SUBSTRATE_TO_TRIBUTARY_TIME_DELAY;
 
-    let spec = TributarySpec::new(block.hash(), time, set, set_data);
+    let spec = TributarySpec::new(block.hash(), time, set, set_participants);
     create_new_tributary(db, spec.clone());
   } else {
     log::info!("not present in set {:?}", set);

coordinator/src/tests/tributary/chain.rs

@@ -15,8 +15,8 @@ use ciphersuite::{
 use sp_application_crypto::sr25519;
 
 use serai_client::{
-  primitives::{NETWORKS, NetworkId, Amount},
-  validator_sets::primitives::{Session, ValidatorSet, ValidatorSetData},
+  primitives::NetworkId,
+  validator_sets::primitives::{Session, ValidatorSet},
 };
 
 use tokio::time::sleep;
@@ -52,20 +52,12 @@ pub fn new_spec<R: RngCore + CryptoRng>(
 
   let set = ValidatorSet { session: Session(0), network: NetworkId::Bitcoin };
 
-  let set_data = ValidatorSetData {
-    bond: Amount(100),
-    network: NETWORKS[&NetworkId::Bitcoin].clone(),
-    participants: keys
-      .iter()
-      .map(|key| {
-        (sr25519::Public((<Ristretto as Ciphersuite>::generator() * **key).to_bytes()), Amount(100))
-      })
-      .collect::<Vec<_>>()
-      .try_into()
-      .unwrap(),
-  };
+  let set_participants = keys
+    .iter()
+    .map(|key| sr25519::Public((<Ristretto as Ciphersuite>::generator() * **key).to_bytes()))
+    .collect::<Vec<_>>();
 
-  let res = TributarySpec::new(serai_block, start_time, set, set_data);
+  let res = TributarySpec::new(serai_block, start_time, set, set_participants);
   assert_eq!(TributarySpec::read::<&[u8]>(&mut res.serialize().as_ref()).unwrap(), res);
   res
 }

coordinator/src/tributary/mod.rs

@@ -17,8 +17,8 @@ use frost::Participant;
 use scale::{Encode, Decode};
 
 use serai_client::{
-  primitives::NetworkId,
-  validator_sets::primitives::{Session, ValidatorSet, ValidatorSetData},
+  primitives::{NetworkId, PublicKey},
+  validator_sets::primitives::{Session, ValidatorSet},
 };
 
 #[rustfmt::skip]
@@ -51,16 +51,16 @@ impl TributarySpec {
     serai_block: [u8; 32],
     start_time: u64,
     set: ValidatorSet,
-    set_data: ValidatorSetData,
+    set_participants: Vec<PublicKey>,
   ) -> TributarySpec {
     let mut validators = vec![];
-    for (participant, amount) in set_data.participants {
+    for participant in set_participants {
       // TODO: Ban invalid keys from being validators on the Serai side
       // (make coordinator key a session key?)
       let participant = <Ristretto as Ciphersuite>::read_G::<&[u8]>(&mut participant.0.as_ref())
         .expect("invalid key registered as participant");
-      // Give one weight on Tributary per bond instance
-      validators.push((participant, amount.0 / set_data.bond.0));
+      // TODO: Give one weight on Tributary per bond instance
+      validators.push((participant, 1));
     }
 
     Self { serai_block, start_time, set, validators }

substrate/client/src/serai/validator_sets.rs

@@ -1,8 +1,8 @@
-use sp_core::sr25519::Signature;
+use sp_core::sr25519::{Public, Signature};
 
 use serai_runtime::{validator_sets, ValidatorSets, Runtime};
 pub use validator_sets::primitives;
-use primitives::{ValidatorSet, ValidatorSetData, KeyPair};
+use primitives::{ValidatorSet, KeyPair};
 
 use subxt::utils::Encoded;
 
@@ -31,39 +31,29 @@ impl Serai {
       .await
   }
 
-  pub async fn get_validator_set(
+  pub async fn get_validator_set_participants(
     &self,
-    set: ValidatorSet,
-  ) -> Result<Option<ValidatorSetData>, SeraiError> {
-    self
-      .storage(
-        PALLET,
-        "ValidatorSets",
-        Some(vec![scale_value(set)]),
-        self.get_latest_block_hash().await?,
-      )
-      .await
+    network: NetworkId,
+    at_hash: [u8; 32],
+  ) -> Result<Option<Vec<Public>>, SeraiError> {
+    self.storage(PALLET, "Participants", Some(vec![scale_value(network)]), at_hash).await
   }
 
   pub async fn get_validator_set_musig_key(
     &self,
     set: ValidatorSet,
+    at_hash: [u8; 32],
   ) -> Result<Option<[u8; 32]>, SeraiError> {
-    self
-      .storage(
-        PALLET,
-        "MuSigKeys",
-        Some(vec![scale_value(set)]),
-        self.get_latest_block_hash().await?,
-      )
-      .await
+    self.storage(PALLET, "MuSigKeys", Some(vec![scale_value(set)]), at_hash).await
   }
 
   // TODO: Store these separately since we almost never need both at once?
-  pub async fn get_keys(&self, set: ValidatorSet) -> Result<Option<KeyPair>, SeraiError> {
-    self
-      .storage(PALLET, "Keys", Some(vec![scale_value(set)]), self.get_latest_block_hash().await?)
-      .await
+  pub async fn get_keys(
+    &self,
+    set: ValidatorSet,
+    at_hash: [u8; 32],
+  ) -> Result<Option<KeyPair>, SeraiError> {
+    self.storage(PALLET, "Keys", Some(vec![scale_value(set)]), at_hash).await
   }
 
   pub fn set_validator_set_keys(

substrate/client/tests/common/in_instructions.rs

@@ -26,7 +26,9 @@ pub async fn provide_batch(batch: Batch) -> [u8; 32] {
   // TODO: Get the latest session
   let set = ValidatorSet { session: Session(0), network: batch.network };
   let pair = insecure_pair_from_name(&format!("ValidatorSet {:?}", set));
-  let keys = if let Some(keys) = serai.get_keys(set).await.unwrap() {
+  let keys = if let Some(keys) =
+    serai.get_keys(set, serai.get_latest_block_hash().await.unwrap()).await.unwrap()
+  {
     keys
   } else {
     let keys = (pair.public(), vec![].try_into().unwrap());

substrate/client/tests/common/validator_sets.rs

@@ -28,7 +28,11 @@ pub async fn set_validator_set_keys(set: ValidatorSet, key_pair: KeyPair) -> [u8
   let serai = serai().await;
   let public_key = <Ristretto as Ciphersuite>::read_G::<&[u8]>(&mut public.0.as_ref()).unwrap();
   assert_eq!(
-    serai.get_validator_set_musig_key(set).await.unwrap().unwrap(),
+    serai
+      .get_validator_set_musig_key(set, serai.get_latest_block_hash().await.unwrap())
+      .await
+      .unwrap()
+      .unwrap(),
     musig_key(set, &[public]).0
   );
 
@@ -40,7 +44,11 @@ pub async fn set_validator_set_keys(set: ValidatorSet, key_pair: KeyPair) -> [u8
   let threshold_keys =
     musig::<Ristretto>(&musig_context(set), &Zeroizing::new(secret_key), &[public_key]).unwrap();
   assert_eq!(
-    serai.get_validator_set_musig_key(set).await.unwrap().unwrap(),
+    serai
+      .get_validator_set_musig_key(set, serai.get_latest_block_hash().await.unwrap())
+      .await
+      .unwrap()
+      .unwrap(),
     threshold_keys.group_key().to_bytes()
   );
 
@@ -66,7 +74,7 @@ pub async fn set_validator_set_keys(set: ValidatorSet, key_pair: KeyPair) -> [u8
     serai.get_key_gen_events(block).await.unwrap(),
     vec![ValidatorSetsEvent::KeyGen { set, key_pair: key_pair.clone() }]
   );
-  assert_eq!(serai.get_keys(set).await.unwrap(), Some(key_pair));
+  assert_eq!(serai.get_keys(set, block).await.unwrap(), Some(key_pair));
 
   block
 }

substrate/client/tests/validator_sets.rs

@@ -3,7 +3,7 @@ use rand_core::{RngCore, OsRng};
 use sp_core::{sr25519::Public, Pair};
 
 use serai_client::{
-  primitives::{NETWORKS, NetworkId, insecure_pair_from_name},
+  primitives::{NetworkId, insecure_pair_from_name},
   validator_sets::{
     primitives::{Session, ValidatorSet, musig_key},
     ValidatorSetsEvent,
@@ -47,12 +47,19 @@ serai_test!(
         .collect::<Vec<_>>(),
     );
 
-    let set_data = serai.get_validator_set(set).await.unwrap().unwrap();
-    assert_eq!(set_data.network, NETWORKS[&NetworkId::Bitcoin]);
-    let participants_ref: &[_] = set_data.participants.as_ref();
-    assert_eq!(participants_ref, [(public, set_data.bond)].as_ref());
+    let participants = serai
+      .get_validator_set_participants(set.network, serai.get_latest_block_hash().await.unwrap())
+      .await
+      .unwrap()
+      .unwrap();
+    let participants_ref: &[_] = participants.as_ref();
+    assert_eq!(participants_ref, [public].as_ref());
     assert_eq!(
-      serai.get_validator_set_musig_key(set).await.unwrap().unwrap(),
+      serai
+        .get_validator_set_musig_key(set, serai.get_latest_block_hash().await.unwrap())
+        .await
+        .unwrap()
+        .unwrap(),
       musig_key(set, &[public]).0
     );
 
@@ -64,6 +71,6 @@ serai_test!(
       serai.get_key_gen_events(block).await.unwrap(),
       vec![ValidatorSetsEvent::KeyGen { set, key_pair: key_pair.clone() }]
     );
-    assert_eq!(serai.get_keys(set).await.unwrap(), Some(key_pair));
+    assert_eq!(serai.get_keys(set, block).await.unwrap(), Some(key_pair));
   }
 );

substrate/node/src/chain_spec.rs

@@ -26,6 +26,7 @@ fn testnet_genesis(
     (
       key,
       key,
+      // TODO: Properly diversify these?
       SessionKeys { babe: key.into(), grandpa: key.into(), authority_discovery: key.into() },
     )
   };
@@ -54,13 +55,9 @@ fn testnet_genesis(
     },
 
     validator_sets: ValidatorSetsConfig {
-      bond: Amount(1_000_000 * 10_u64.pow(8)),
-      networks: vec![
-        (NetworkId::Serai, NETWORKS[&NetworkId::Serai].clone()),
-        (NetworkId::Bitcoin, NETWORKS[&NetworkId::Bitcoin].clone()),
-        (NetworkId::Ethereum, NETWORKS[&NetworkId::Ethereum].clone()),
-        (NetworkId::Monero, NETWORKS[&NetworkId::Monero].clone()),
-      ],
+      stake: Amount(1_000_000 * 10_u64.pow(8)),
+      // TODO: Array of these in primitives
+      networks: vec![NetworkId::Serai, NetworkId::Bitcoin, NetworkId::Ethereum, NetworkId::Monero],
       participants: validators.iter().map(|name| account_from_name(name)).collect(),
     },
     session: SessionConfig { keys: validators.iter().map(|name| session_key(*name)).collect() },

substrate/primitives/Cargo.toml

@@ -12,8 +12,6 @@ all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 
 [dependencies]
-lazy_static = { version = "1", optional = true }
-
 zeroize = { version = "^1.5", features = ["derive"], optional = true }
 
 serde = { version = "1", default-features = false, features = ["derive", "alloc"] }
@@ -26,5 +24,5 @@ sp-core = { git = "https://github.com/serai-dex/substrate", default-features = f
 sp-runtime = { git = "https://github.com/serai-dex/substrate", default-features = false }
 
 [features]
-std = ["lazy_static", "zeroize", "scale/std", "serde/std", "scale-info/std", "sp-core/std", "sp-runtime/std"]
+std = ["zeroize", "scale/std", "serde/std", "scale-info/std", "sp-core/std", "sp-runtime/std"]
 default = ["std"]

substrate/primitives/src/networks.rs

@@ -1,6 +1,3 @@
-#[cfg(feature = "std")]
-use std::collections::HashMap;
-
 #[cfg(feature = "std")]
 use zeroize::Zeroize;
 
@@ -120,13 +117,3 @@ impl Network {
     &self.coins
   }
 }
-
-#[cfg(feature = "std")]
-lazy_static::lazy_static! {
-  pub static ref NETWORKS: HashMap<NetworkId, Network> = HashMap::from([
-    (NetworkId::Serai, Network::new(vec![Coin::Serai]).unwrap()),
-    (NetworkId::Bitcoin, Network::new(vec![Coin::Bitcoin]).unwrap()),
-    (NetworkId::Ethereum, Network::new(vec![Coin::Ether, Coin::Dai]).unwrap()),
-    (NetworkId::Monero, Network::new(vec![Coin::Monero]).unwrap()),
-  ]);
-}