Ab/leaf2 migration

Cargo.toml

@@ -70,7 +70,7 @@ marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketpla
 marketplace-builder-shared = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.1.59" }
 hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.57" }
 hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.83" }
-hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "v0.1.76" }
+hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", branch = "ab/leaf2-migration" }
 hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.83" }
 hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" }
 hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.83" }

sequencer/api/migrations/postgres/V501__epoch_tables.sql

@@ -0,0 +1,54 @@
+CREATE TABLE anchor_leaf2 (
+    view   BIGINT PRIMARY KEY,
+    leaf   BYTEA,
+    qc   BYTEA
+);
+
+
+CREATE TABLE da_proposal2 (
+    view BIGINT PRIMARY KEY,
+    payload_hash VARCHAR,
+    data BYTEA
+);
+
+CREATE TABLE vid_share2 (
+    view BIGINT PRIMARY KEY,
+    payload_hash VARCHAR,
+    data BYTEA
+);
+
+
+CREATE TABLE undecided_state2 (
+    -- The ID is always set to 0. Setting it explicitly allows us to enforce with every insert or
+    -- update that there is only a single entry in this table: the latest known state.
+    id INT PRIMARY KEY,
+
+    leaves BYTEA NOT NULL,
+    state  BYTEA NOT NULL
+);
+
+
+CREATE TABLE quorum_proposals2 (
+    view BIGINT PRIMARY KEY,
+    leaf_hash VARCHAR,
+    data BYTEA
+);
+
+CREATE UNIQUE INDEX quorum_proposals2_leaf_hash_idx ON quorum_proposals (leaf_hash);
+CREATE INDEX da_proposal2_payload_hash_idx ON da_proposal (payload_hash);
+CREATE INDEX vid_share2_payload_hash_idx ON vid_share (payload_hash);
+
+CREATE TABLE quorum_certificate2 (
+    view BIGINT PRIMARY KEY,
+    leaf_hash VARCHAR NOT NULL,
+    data BYTEA NOT NULL
+);
+
+CREATE INDEX quorum_certificate2_leaf_hash_idx ON quorum_certificate (leaf_hash);
+
+CREATE TABLE epoch_migration (
+    table_name TEXT PRIMARY KEY,
+    completed bool DEFAULT FALSE
+);
+
+INSERT INTO epoch_migration (table_name) VALUES ('anchor_leaf'), ('da_proposal'), ('vid_share'), ('undecided_state'), ('quorum_proposals'), ('quorum_certificate');

sequencer/api/migrations/sqlite/V301__epoch_tables.sql

@@ -0,0 +1,54 @@
+CREATE TABLE anchor_leaf2 (
+    view   BIGINT PRIMARY KEY,
+    leaf  BLOB,
+    qc   BLOB
+);
+
+
+CREATE TABLE da_proposal2 (
+    view BIGINT PRIMARY KEY,
+    payload_hash VARCHAR,
+    data BLOB
+);
+
+CREATE TABLE vid_share2 (
+    view BIGINT PRIMARY KEY,
+    payload_hash VARCHAR,
+    data BLOB
+);
+
+
+CREATE TABLE undecided_state2 (
+    -- The ID is always set to 0. Setting it explicitly allows us to enforce with every insert or
+    -- update that there is only a single entry in this table: the latest known state.
+    id INT PRIMARY KEY,
+
+    leaves BLOB NOT NULL,
+    state  BLOB NOT NULL
+);
+
+
+CREATE TABLE quorum_proposals2 (
+    view BIGINT PRIMARY KEY,
+    leaf_hash VARCHAR,
+    data BLOB
+);
+
+CREATE UNIQUE INDEX quorum_proposals2_leaf_hash_idx ON quorum_proposals (leaf_hash);
+CREATE INDEX da_proposal2_payload_hash_idx ON da_proposal (payload_hash);
+CREATE INDEX vid_share2_payload_hash_idx ON vid_share (payload_hash);
+
+CREATE TABLE quorum_certificate2 (
+    view BIGINT PRIMARY KEY,
+    leaf_hash VARCHAR NOT NULL,
+    data BLOB NOT NULL
+);
+
+CREATE INDEX quorum_certificate2_leaf_hash_idx ON quorum_certificate (leaf_hash);
+
+CREATE TABLE epoch_migration (
+    table_name TEXT PRIMARY KEY,
+    completed bool NOT NULL DEFAULT FALSE
+);
+
+INSERT INTO epoch_migration (table_name) VALUES ('anchor_leaf'), ('da_proposal'), ('vid_share'), ('undecided_state'), ('quorum_proposals'), ('quorum_certificate');

sequencer/src/api.rs

@@ -1065,8 +1065,10 @@ mod api_tests {
     use hotshot_query_service::availability::{
         AvailabilityDataSource, BlockQueryData, VidCommonQueryData,
     };
+    use hotshot_types::data::{DaProposal2, EpochNumber, VidDisperseShare2};
+    use hotshot_types::simple_certificate::QuorumCertificate2;
     use hotshot_types::{
-        data::{DaProposal, QuorumProposal2, VidDisperseShare},
+        data::QuorumProposal2,
         event::LeafInfo,
         message::Proposal,
         simple_certificate::QuorumCertificate,
@@ -1246,31 +1248,29 @@ mod api_tests {
         // Create two non-consecutive leaf chains.
         let mut chain1 = vec![];
 
-        let genesis = Leaf::genesis(&Default::default(), &NodeState::mock()).await;
+        let genesis = Leaf2::genesis(&Default::default(), &NodeState::mock()).await;
         let payload = genesis.block_payload().unwrap();
         let payload_bytes_arc = payload.encode();
         let disperse = vid_scheme(2).disperse(payload_bytes_arc.clone()).unwrap();
         let payload_commitment = disperse.commit;
         let mut quorum_proposal = QuorumProposal2::<SeqTypes> {
             block_header: genesis.block_header().clone(),
             view_number: ViewNumber::genesis(),
-            justify_qc: QuorumCertificate::genesis::<MockSequencerVersions>(
+            justify_qc: QuorumCertificate2::genesis::<MockSequencerVersions>(
                 &ValidatedState::default(),
                 &NodeState::mock(),
             )
-            .await
-            .to_qc2(),
+            .await,
             upgrade_certificate: None,
             view_change_evidence: None,
             next_drb_result: None,
             next_epoch_justify_qc: None,
         };
-        let mut qc = QuorumCertificate::genesis::<MockSequencerVersions>(
+        let mut qc = QuorumCertificate2::genesis::<MockSequencerVersions>(
             &ValidatedState::default(),
             &NodeState::mock(),
         )
-        .await
-        .to_qc2();
+        .await;
 
         let mut justify_qc = qc.clone();
         for i in 0..5 {
@@ -1288,7 +1288,7 @@ mod api_tests {
                 PubKey::sign(&privkey, &bincode::serialize(&quorum_proposal).unwrap())
                     .expect("Failed to sign quorum_proposal");
             persistence
-                .append_quorum_proposal(&Proposal {
+                .append_quorum_proposal2(&Proposal {
                     data: quorum_proposal.clone(),
                     signature: quorum_proposal_signature,
                     _pd: Default::default(),
@@ -1297,33 +1297,37 @@ mod api_tests {
                 .unwrap();
 
             // Include VID information for each leaf.
-            let share = VidDisperseShare::<SeqTypes> {
+            let share = VidDisperseShare2::<SeqTypes> {
                 view_number: leaf.view_number(),
                 payload_commitment,
                 share: disperse.shares[0].clone(),
                 common: disperse.common.clone(),
                 recipient_key: pubkey,
+                epoch: EpochNumber::new(0),
+                target_epoch: EpochNumber::new(0),
+                data_epoch_payload_commitment: None,
             };
             persistence
-                .append_vid(&share.to_proposal(&privkey).unwrap())
+                .append_vid2(&share.to_proposal(&privkey).unwrap())
                 .await
                 .unwrap();
 
             // Include payload information for each leaf.
             let block_payload_signature =
                 PubKey::sign(&privkey, &payload_bytes_arc).expect("Failed to sign block payload");
-            let da_proposal_inner = DaProposal::<SeqTypes> {
+            let da_proposal_inner = DaProposal2::<SeqTypes> {
                 encoded_transactions: payload_bytes_arc.clone(),
                 metadata: payload.ns_table().clone(),
                 view_number: leaf.view_number(),
+                epoch: EpochNumber::new(0),
             };
             let da_proposal = Proposal {
                 data: da_proposal_inner,
                 signature: block_payload_signature,
                 _pd: Default::default(),
             };
             persistence
-                .append_da(&da_proposal, payload_commitment)
+                .append_da2(&da_proposal, payload_commitment)
                 .await
                 .unwrap();
         }
@@ -1369,8 +1373,8 @@ mod api_tests {
         for (leaf, qc) in chain1.iter().chain(&chain2) {
             tracing::info!(height = leaf.height(), "check archive");
             let qd = data_source.get_leaf(leaf.height() as usize).await.await;
-            let stored_leaf: Leaf2 = qd.leaf().clone().into();
-            let stored_qc = qd.qc().clone().to_qc2();
+            let stored_leaf: Leaf2 = qd.leaf().clone();
+            let stored_qc = qd.qc().clone();
             assert_eq!(&stored_leaf, leaf);
             assert_eq!(&stored_qc, qc);
 
@@ -1489,10 +1493,7 @@ mod api_tests {
             .unwrap();
 
         // Check that we still processed the leaf.
-        assert_eq!(
-            leaf,
-            data_source.get_leaf(1).await.await.leaf().clone().into()
-        );
+        assert_eq!(leaf, data_source.get_leaf(1).await.await.leaf().clone());
         assert!(data_source.get_vid_common(1).await.is_pending());
         assert!(data_source.get_block(1).await.is_pending());
     }

sequencer/src/api/sql.rs

@@ -4,7 +4,7 @@ use committable::{Commitment, Committable};
 use espresso_types::{
     get_l1_deposits,
     v0_99::{ChainConfig, IterableFeeInfo},
-    BlockMerkleTree, FeeAccount, FeeMerkleTree, Leaf, Leaf2, NodeState, ValidatedState,
+    BlockMerkleTree, FeeAccount, FeeMerkleTree, Leaf2, NodeState, ValidatedState,
 };
 use hotshot::traits::ValidatedState as _;
 use hotshot_query_service::{
@@ -21,7 +21,7 @@ use hotshot_query_service::{
     Resolvable,
 };
 use hotshot_types::{
-    data::{QuorumProposal, ViewNumber},
+    data::{QuorumProposal2, ViewNumber},
     message::Proposal,
     traits::node_implementation::ConsensusTime,
 };
@@ -257,7 +257,7 @@ async fn load_accounts<Mode: TransactionMode>(
         }
     }
 
-    Ok((snapshot, leaf.leaf().clone().into()))
+    Ok((snapshot, leaf.leaf().clone()))
 }
 
 async fn load_chain_config<Mode: TransactionMode>(
@@ -286,7 +286,7 @@ async fn reconstruct_state<Mode: TransactionMode>(
         .get_leaf((from_height as usize).into())
         .await
         .context(format!("leaf {from_height} not available"))?;
-    let from_leaf: Leaf2 = from_leaf.leaf().clone().into();
+    let from_leaf: Leaf2 = from_leaf.leaf().clone();
     ensure!(
         from_leaf.view_number() < to_view,
         "state reconstruction: starting state {:?} must be before ending state {to_view:?}",
@@ -440,13 +440,13 @@ where
     P: Type<Db> + for<'q> Encode<'q, Db>,
 {
     let (data,) = query_as::<(Vec<u8>,)>(&format!(
-        "SELECT data FROM quorum_proposals WHERE {where_clause} LIMIT 1",
+        "SELECT data FROM quorum_proposals2 WHERE {where_clause} LIMIT 1",
     ))
     .bind(param)
     .fetch_one(tx.as_mut())
     .await?;
-    let proposal: Proposal<SeqTypes, QuorumProposal<SeqTypes>> = bincode::deserialize(&data)?;
-    Ok(Leaf::from_quorum_proposal(&proposal.data).into())
+    let proposal: Proposal<SeqTypes, QuorumProposal2<SeqTypes>> = bincode::deserialize(&data)?;
+    Ok(Leaf2::from_quorum_proposal(&proposal.data))
 }
 
 #[cfg(any(test, feature = "testing"))]