refactor roll vs apply

examples/sync-mainnet/dolos.toml

@@ -0,0 +1,21 @@
+[upstream]
+peer_address = "relays-new.cardano-mainnet.iohk.io:3001"
+network_magic = 764824073
+
+[rolldb]
+path = "./tmp/rolldb"
+k_param = 1000
+
+[applydb]
+path = "./tmp/applydb"
+
+[serve.grpc]
+listen_address = "[::]:50051"
+
+[serve.ouroboros]
+listen_address = "localhost:30013"
+magic = 0
+
+[byron]
+path = "./byron.json"
+hash = "xxx"

examples/sync-preview/byron.json

@@ -0,0 +1,91 @@
+{
+  "bootStakeholders": {
+    "021e737009040bf7f1e7b1bcc148f29d748d4a6b561902c95e4a9f36": 1,
+    "0bc82ced9544980b9ffe7f64b1538bbda6804a5cc32c8035485e184b": 1,
+    "18ed9844deef98cf9ba8b39791dede0538d2d2fa79bf67ef37dcc826": 1,
+    "66cfa84ad0ee5ca8586244c8393007cf3d9622d77cfa03fd4f35065b": 1,
+    "76c4d6c68c0ef81ae364411a84e52ce66089ed006ca29adfc0227901": 1,
+    "8cc6b89fec65cc83d34b7bab2e6494db631d8476a86625767dd0c2a0": 1,
+    "e90060fdc085ac9f63cdb3b32ba1d84e0f7eb98561687b213b4c8770": 1
+  },
+  "heavyDelegation": {
+    "021e737009040bf7f1e7b1bcc148f29d748d4a6b561902c95e4a9f36": {
+      "omega": 0,
+      "issuerPk": "6hSFCotivD08t02n43RMiaF9LzwtYVrFMu/WX6ShfEsxfdXFL5Y6c+DwHSZOCywU0RJz5er2icIO03UytC9NTg==",
+      "delegatePk": "JEnSVQTPGriTx1+lAMkKhCNsMBDNPGw+NiEvNPh4ui6IdvxrO+WkQPTy5U865XB4VFvi/zb7d+H1bilnztQNBg==",
+      "cert": "558952d17442e8cc73f0c7dd606e329b38ed2ec0c1f83fe2567d28b21ef2223d2d23640cd0531f75832b50e519631c48643fcfaa7168851645dce07b90d87f0e"
+    },
+    "0bc82ced9544980b9ffe7f64b1538bbda6804a5cc32c8035485e184b": {
+      "omega": 0,
+      "issuerPk": "MJ7IskKU8GKk0Eeg3zhfSOK1DDVXOMHD2V/zhEpODUtL9YB0Y7sXnbZfg3+Df05hskP5Jz+dZvdC6DH/dP9jmQ==",
+      "delegatePk": "hwO7NJL7LfAk5e/QG61FKcdORoK60tvprE3063Muh4EQKrWA6l7t23B2GziK8D0hRO0j5W1Gzpn8WW69XLIlKA==",
+      "cert": "2bccf50d0c3cbb03dd29cfba817e8ba615db3d7722b41b264ad08722e548cfe83d069b29d13e490823d7519ecdd9940ea49573f6027056c4bd58da1adf75020e"
+    },
+    "18ed9844deef98cf9ba8b39791dede0538d2d2fa79bf67ef37dcc826": {
+      "omega": 0,
+      "issuerPk": "pXbW4Jak8maeuWiosvrurykKnqDSHswUjroonSDS3fTnWS+BKe+vjT4zZJNKhQ33KbagiHVJ5CJUNggfsCtG2g==",
+      "delegatePk": "rbJAZp3kWCUvp8dnLR6qsgpGU+qKAFow4NHYKWiKCkfm1qFCFONob50N1IbNWCGWAhg38ZPTvBazTasjsfj6yQ==",
+      "cert": "89e1638e31fd3d402cecb897ba773d8c2c11c2d3cff2462b266e21461539b1a4fe8fb528e159b9af473799b51e49aa5b5816a88f10c484aa7cef7ad12850830a"
+    },
+    "66cfa84ad0ee5ca8586244c8393007cf3d9622d77cfa03fd4f35065b": {
+      "omega": 0,
+      "issuerPk": "/LGZjmmcAMRisP7Rf454GM2QUKgj2aAyqE+iQo2PIEhcistFOlT+idtbLTceZAnQcwwPJDtTcNi+EnPQyscZOg==",
+      "delegatePk": "rinFUiKKCPPFY0ULEKn1SPRgLVmOS3jdTXDtrxK6VI1I11G3uBS1Olxi0mQSN3kf+B3hm/xHkuUDVNaSXNiBeQ==",
+      "cert": "3e7f30bb68c5bc4d23c2a730ac154a188a1fd45aac3f438efd380303171443d2ca4f50e5a1ff66b40ae3da64697f2599956ae06c21b73fa828b8c0dc9fb27302"
+    },
+    "76c4d6c68c0ef81ae364411a84e52ce66089ed006ca29adfc0227901": {
+      "omega": 0,
+      "issuerPk": "9EE85tTLdSSR4T1Xoy6n9wr6jlbavCdfp9oQKusskO3DSSyNqRYS7QzYQ96j/WnphUey63082YkKijMfF9A4eA==",
+      "delegatePk": "dvyHDkXg8LFtb0K6Sitl8OGSEZPvfCVQYLDR6Au6t6/ROvlerMKQ8uri4fG7hQQzbHKtdKWgv94t+zuFJTQ1fw==",
+      "cert": "5ec0ed46ae7e575bdb089f1bceca3b2689b13a7162fe08578fe60ba64607fffaa507412a97652c3c81cc0ef93ff404cf809a628ae19faba1a035fca0505c1d04"
+    },
+    "8cc6b89fec65cc83d34b7bab2e6494db631d8476a86625767dd0c2a0": {
+      "omega": 0,
+      "issuerPk": "Hr5S5PAxf9HSB4FzmtZzaFcXrNrctrI5XUrDrnCkOUTX6rhbtOMkXU3sWVDOvU6LNSSr3/Ws2+iCYZIr7LmTWg==",
+      "delegatePk": "FaLH2b5H/XS31YRnm98N6fP4Etx6m+GbniVAXMwOp8KhYXPKBJBsX/EjIy3pSkvRBhGCjsycB0yrDxWMi5ZsIQ==",
+      "cert": "10f06304cceb42071605ebba67b308c7568e5e6fe0d773c58f7e8c13bc8d8a340f70a4fd5e1b4a1c1db1de5c7646802bbc929d6c82d7adb8a77cb6ad77eac50a"
+    },
+    "e90060fdc085ac9f63cdb3b32ba1d84e0f7eb98561687b213b4c8770": {
+      "omega": 0,
+      "issuerPk": "B2R+VXzy3c8bxncdOpQ2Z/tblxRNQO8AXQ0OsJDQvZYnLeGQcLD78kyYLpi3nfuS4SfnLar23NV4yiEVwaw+Yw==",
+      "delegatePk": "nACHGIBacymrKwn07iW/a5ZKJCPZ2cKQqeXw3ivR7WOYVUuufWhZlCoUTZ7rtBqoDaexblUQwkC7hA7AmNA3FA==",
+      "cert": "b5440daa05f7fae557df46e4f1b7c5802b86f465daad1137e315abf6e72f1c877207276abb8dcba86e18e42d39b34c2f0fa82ba2919944cdc8e2e5264baa450b"
+    }
+  },
+  "startTime": 1666656000,
+  "nonAvvmBalances": {
+    "FHnt4NL7yPXjpZtYj1YUiX9QYYUZGXDT9gA2PJXQFkTSMx3EgawXK5BUrCHdhe2": "0",
+    "FHnt4NL7yPXk7D87qAWEmfnL7wSQ9AzBU2mjZt3eM48NSCbygxgzAU6vCGiRZEW": "0",
+    "FHnt4NL7yPXpazQsTdJ3Gp1twQUo4N5rrgGbRNSzchjchPiApc1k4CvqDMcdd7H": "0",
+    "FHnt4NL7yPXtNo1wLCLZyGTMfAvB14h8onafiYkM7B69ZwvGgXeUyQWfi7FPrif": "0",
+    "FHnt4NL7yPXtmi4mAjD43V3NB3shDs1gCuHNcMLPsRWjaw1b2yRV2xad8S8V6aq": "0",
+    "FHnt4NL7yPXvDWHa8bVs73UEUdJd64VxWXSFNqetECtYfTd9TtJguJ14Lu3feth": "30000000000000000",
+    "FHnt4NL7yPXvNSRpCYydjRr7koQCrsTtkovk5uYMimgqMJX2DyrEEBqiXaTd8rG": "0",
+    "FHnt4NL7yPY9rTvdsCeyRnsbzp4bN7XdmAZeU5PzA1qR2asYmN6CsdxJw4YoDjG": "0"
+  },
+  "blockVersionData": {
+    "scriptVersion": 0,
+    "slotDuration": "20000",
+    "maxBlockSize": "2000000",
+    "maxHeaderSize": "2000000",
+    "maxTxSize": "4096",
+    "maxProposalSize": "700",
+    "mpcThd": "20000000000000",
+    "heavyDelThd": "300000000000",
+    "updateVoteThd": "1000000000000",
+    "updateProposalThd": "100000000000000",
+    "updateImplicit": "10000",
+    "softforkRule": {
+      "initThd": "900000000000000",
+      "minThd": "600000000000000",
+      "thdDecrement": "50000000000000"
+    },
+    "txFeePolicy": {
+      "summand": "155381000000000",
+      "multiplier": "43946000000"
+    },
+    "unlockStakeEpoch": "18446744073709551615"
+  },
+  "protocolConsts": { "k": 432, "protocolMagic": 2 },
+  "avvmDistr": {}
+}

examples/sync-preview/dolos.toml

@@ -15,3 +15,7 @@ listen_address = "[::]:50051"
 [serve.ouroboros]
 listen_address = "localhost:30013"
 magic = 0
+
+[byron]
+path = "./byron.json"
+hash = "xxx"

src/bin/dolos/daemon.rs

@@ -1,4 +1,5 @@
 use std::path::Path;
+use tracing::info;
 
 use dolos::{
     prelude::*,
@@ -41,15 +42,9 @@ pub async fn run(
 
     let applydb = ApplyDB::open(applydb_path).map_err(Error::storage)?;
 
-    if applydb.is_empty() {
-        applydb
-            .insert_genesis_utxos(&byron_genesis)
-            .map_err(Error::storage)?;
-    }
-
     let server = tokio::spawn(dolos::serve::serve(config.serve, rolldb.clone()));
 
-    dolos::sync::pipeline(&config.upstream, rolldb, applydb, policy)
+    dolos::sync::pipeline(&config.upstream, rolldb, applydb, byron_genesis, policy)
         .unwrap()
         .block();
 

src/bin/dolos/sync.rs

@@ -29,6 +29,9 @@ pub fn run(
     let rolldb =
         RollDB::open(rolldb_path, config.rolldb.k_param.unwrap_or(1000)).map_err(Error::storage)?;
 
+    let byron_genesis =
+        pallas::ledger::configs::byron::from_file(&config.byron.path).map_err(Error::config)?;
+
     let applydb_path = config
         .applydb
         .path
@@ -37,7 +40,7 @@ pub fn run(
 
     let applydb = ApplyDB::open(applydb_path).map_err(Error::storage)?;
 
-    dolos::sync::pipeline(&config.upstream, rolldb, applydb, policy)
+    dolos::sync::pipeline(&config.upstream, rolldb, applydb, byron_genesis, policy)
         .unwrap()
         .block();
 

src/model.rs

@@ -26,4 +26,5 @@ pub enum RollEvent {
     Apply(BlockSlot, BlockHash, RawBlock),
     Undo(BlockSlot, BlockHash, RawBlock),
     Reset(Point),
+    Origin,
 }

src/serve/grpc/sync.rs

@@ -33,6 +33,8 @@ fn roll_to_tip_response(
         action: match evt.action() {
             WalAction::Apply => follow_tip_response::Action::Apply(raw_to_anychain(block)).into(),
             WalAction::Undo => follow_tip_response::Action::Undo(raw_to_anychain(block)).into(),
+            // TODO: shouldn't we have a u5c event for origin?
+            WalAction::Origin => None,
             WalAction::Mark => None,
         },
     }
@@ -124,7 +126,7 @@ impl chain_sync_service_server::ChainSyncService for ChainSyncServiceImpl {
         &self,
         _request: Request<FollowTipRequest>,
     ) -> Result<Response<Self::FollowTipStream>, tonic::Status> {
-        let s = crate::storage::rolldb::stream::RollStream::start_with_block(self.0.clone(), None)
+        let s = crate::storage::rolldb::stream::RollStream::start_after_block(self.0.clone(), None)
             .map(|(evt, block)| Ok(roll_to_tip_response(evt, &block)));
 
         Ok(Response::new(Box::pin(s)))

src/storage/applydb/genesis.rs

@@ -2,6 +2,7 @@ use pallas::ledger::addresses::ByronAddress;
 use pallas::ledger::configs::byron::GenesisUtxo;
 use pallas::ledger::primitives::byron::TxOut;
 use rocksdb::WriteBatch;
+use tracing::info;
 
 use crate::{
     prelude::*,
@@ -43,6 +44,7 @@ impl ApplyDB {
             .collect::<Result<Vec<_>, _>>()?
             .into_iter()
             .fold(WriteBatch::default(), |mut batch, (k, v)| {
+                info!(tx = %k.0 .0, "inserting genesis utxo");
                 UtxoKV::stage_upsert(&self.db, k, v, &mut batch);
                 batch
             });
@@ -84,11 +86,12 @@ mod tests {
     }
 
     #[test]
-    fn test_genesis_utxos() {
+    fn test_mainnet_genesis_utxos() {
         with_tmp_db(|db| {
             let path = std::path::PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap())
-                .join("test_data")
-                .join("mainnet-byron-genesis.json");
+                .join("examples")
+                .join("sync-mainnet")
+                .join("byron.json");
 
             let byron = pallas::ledger::configs::byron::from_file(&path).unwrap();
             db.insert_genesis_utxos(&byron).unwrap();
@@ -101,4 +104,24 @@ mod tests {
             )
         });
     }
+
+    #[test]
+    fn test_preview_genesis_utxos() {
+        with_tmp_db(|db| {
+            let path = std::path::PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap())
+                .join("examples")
+                .join("sync-preview")
+                .join("byron.json");
+
+            let byron = pallas::ledger::configs::byron::from_file(&path).unwrap();
+            db.insert_genesis_utxos(&byron).unwrap();
+
+            assert_genesis_utxo_exists(
+                &db,
+                "4843cf2e582b2f9ce37600e5ab4cc678991f988f8780fed05407f9537f7712bd",
+                "FHnt4NL7yPXvDWHa8bVs73UEUdJd64VxWXSFNqetECtYfTd9TtJguJ14Lu3feth",
+                30_000_000_000_000_000,
+            );
+        });
+    }
 }

src/storage/applydb/mod.rs

@@ -3,6 +3,7 @@ pub mod genesis;
 use pallas::crypto::hash::Hash;
 use serde::{Deserialize, Serialize};
 use std::{path::Path, sync::Arc};
+use tracing::{error, info};
 
 use rocksdb::{Options, WriteBatch, DB};
 
@@ -55,12 +56,20 @@ impl<'a> BlockWriteBatch<'a> {
         )
     }
 
-    pub fn spend_utxo(&mut self, tx: TxHash, output: OutputIndex) -> Result<(), Error> {
-        let k = DBSerde(UtxoRef(tx, output));
-        let v = UtxoKV::get_by_key(self.0, k.clone())?.ok_or(Error::NotFound)?;
+    pub fn spend_utxo(&mut self, tx: TxHash, idx: OutputIndex) -> Result<(), Error> {
+        let k = DBSerde(UtxoRef(tx, idx));
+        //let v = UtxoKV::get_by_key(self.0, k.clone())?.ok_or(Error::NotFound)?;
+
+        let v = UtxoKV::get_by_key(self.0, k.clone())?.ok_or_else(|| {
+            error!(%tx, idx, "utxo not found");
+            Error::NotFound
+        })?;
+
         StxiKV::stage_upsert(self.0, k.clone(), v, &mut self.2);
         UtxoKV::stage_delete(self.0, k, &mut self.2);
 
+        info!(%tx, idx, "spending utxo");
+
         Ok(())
     }
 

src/storage/kvtable.rs

@@ -266,6 +266,15 @@ where
         db.cf_handle(Self::CF_NAME).unwrap()
     }
 
+    fn reset(db: &rocksdb::DB) -> Result<(), Error> {
+        db.drop_cf(Self::CF_NAME).map_err(|_| Error::IO)?;
+
+        db.create_cf(Self::CF_NAME, &rocksdb::Options::default())
+            .map_err(|_| Error::IO)?;
+
+        Ok(())
+    }
+
     fn get_by_key(db: &rocksdb::DB, k: K) -> Result<Option<V>, Error> {
         let cf = Self::cf(db);
         let raw_key = Box::<[u8]>::from(k);
@@ -297,7 +306,7 @@ where
         // iterator and see if we have at least one value. If someone know a better way
         // to accomplish this, please refactor.
         let mut iter = Self::iter_keys(db, rocksdb::IteratorMode::Start);
-        iter.next().is_some()
+        iter.next().is_none()
     }
 
     fn iter_keys<'a>(db: &'a rocksdb::DB, mode: rocksdb::IteratorMode) -> KeyIterator<'a, K> {