[WEEKLY RELEASE] HotShot - rc-0.5.79 (#2230)

Closes #<ISSUE_NUMBER>
<!-- These comments should help create a useful PR message, please
delete any remaining comments before opening the PR. -->
<!-- If there is no issue number make sure to describe clearly *why*
this PR is necessary. -->
<!-- Mention open questions, remaining TODOs, if any -->

### This PR:
<!-- Describe what this PR adds to this repo and why -->
<!-- E.g. -->
<!-- * Implements feature 1 -->
<!-- * Fixes bug 3 -->

### This PR does not:
<!-- Describe what is out of scope for this PR, if applicable. Leave
this section blank if it's not applicable -->
<!-- This section helps avoid the reviewer having to needlessly point
out missing parts -->
<!-- * Implement feature 3 because that feature is blocked by Issue 4
-->
<!-- * Implement xyz because that is tracked in issue #123. -->
<!-- * Address xzy for which I opened issue #456 -->

### Key places to review:
<!-- Describe key places for reviewers to pay close attention to -->
<!-- * file.rs, `add_integers` function -->
<!-- Or directly comment on those files/lines to make it easier for the
reviewers -->

<!-- ### How to test this PR:  -->
<!-- Optional, uncomment the above line if this is relevant to your PR
-->
<!-- If your PR is fully tested through CI there is no need to add this
section -->
<!-- * E.g. `just test` -->

<!-- ### Things tested -->
<!-- Anything that was manually tested (that is not tested in CI). -->
<!-- E.g. building/running of docker containers. Changes to docker demo,
... -->
<!-- Especially mention anything untested, with reasoning and link an
issue to resolve this. -->

<!-- Complete the following items before creating this PR -->
<!-- [ ] Issue linked or PR description mentions why this change is
necessary. -->
<!-- [ ] PR description is clear enough for reviewers. -->
<!-- [ ] Documentation for changes (additions) has been updated (added).
-->
<!-- [ ] If this is a draft it is marked as "draft".  -->

<!-- To make changes to this template edit
https://github.com/EspressoSystems/.github/blob/main/PULL_REQUEST_TEMPLATE.md
-->

Cargo.toml

@@ -57,24 +57,24 @@ dyn-clone = "1.0"
 ethers = { version = "2.0", features = ["solc"] }
 futures = "0.3"
 
-hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.78-patch4" }
+hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.79" }
 # Hotshot imports
-hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.78-patch4" }
-hotshot-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.1.53" }
-marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.1.53" }
-marketplace-builder-shared = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.1.53" }
-hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.51" }
-hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.78-patch4" }
-hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.66" }
-hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.78-patch4" }
+hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "0.5.79" }
+hotshot-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.1.54" }
+marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.1.54" }
+marketplace-builder-shared = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "0.1.54" }
+hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.52" }
+hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.79" }
+hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "0.1.68" }
+hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.79" }
 hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" }
-hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.78-patch4" }
-hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.78-patch4" }
-hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.78-patch4" }
-libp2p-networking = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.78-patch4" }
+hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.79" }
+hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.79" }
+hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.79" }
+libp2p-networking = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.79" }
 hotshot-contract-adapter = { version = "0.1.0", path = "contracts/rust/adapter" }
 # Temporary, used to pull in the mock auction results provider
-hotshot-example-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.78-patch4" }
+hotshot-example-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "0.5.79" }
 
 # Push CDN imports
 cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", features = [
@@ -155,15 +155,3 @@ debug = false
 strip = "debuginfo"
 [profile.dev.package."*"]
 opt-level = 3
-
-[patch."https://github.com/EspressoSystems/HotShot.git"]
-hotshot = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.78-patch5" }
-hotshot-builder-api = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.78-patch5" }
-hotshot-orchestrator = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.78-patch5" }
-hotshot-stake-table = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.78-patch5" }
-hotshot-task ={ git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.78-patch5" }
-hotshot-task-impls ={ git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.78-patch5" }
-hotshot-testing ={ git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.78-patch5" }
-hotshot-types = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.78-patch5" }
-libp2p-networking = { git = "https://github.com/EspressoSystems//HotShot.git", tag = "0.5.78-patch5" }
-hotshot-example-types = { git = "https://www.github.com/EspressoSystems/HotShot.git", tag = "0.5.78-patch5" }

builder/src/lib.rs

@@ -73,7 +73,7 @@ pub mod testing {
             node_implementation::{ConsensusTime, NodeType, Versions},
             signature_key::BuilderSignatureKey as _,
         },
-        ExecutionType, HotShotConfig, PeerConfig, ValidatorConfig,
+        HotShotConfig, PeerConfig, ValidatorConfig,
     };
     use sequencer::{context::Consensus, network, SequencerApiVersion};
     use std::{
@@ -110,28 +110,16 @@ pub mod testing {
             let (
                 priv_keys_non_staking_nodes,
                 non_staking_nodes_state_key_pairs,
-                known_nodes_without_stake,
+                _known_nodes_without_stake,
             ) = generate_stake_table_entries(num_nodes_without_stake as u64, 0);
 
-            // get the pub key out of the stake table entry for the non-staking nodes
-            // Only pass the pub keys to the hotshot config
-            let known_nodes_without_stake_pub_keys = known_nodes_without_stake
-                .iter()
-                .map(|x| <BLSPubKey as SignatureKey>::public_key(&x.stake_table_entry))
-                .collect::<Vec<_>>();
-
             let builder_url = hotshot_builder_url();
 
             let config: HotShotConfig<PubKey> = HotShotConfig {
-                execution_type: ExecutionType::Continuous,
                 num_nodes_with_stake: NonZeroUsize::new(num_nodes_with_stake).unwrap(),
                 known_da_nodes: known_nodes_with_stake.clone(),
                 known_nodes_with_stake: known_nodes_with_stake.clone(),
-                known_nodes_without_stake: known_nodes_without_stake_pub_keys,
                 next_view_timeout: Duration::from_secs(5).as_millis() as u64,
-                timeout_ratio: (10, 11),
-                round_start_delay: Duration::from_millis(1).as_millis() as u64,
-                start_delay: Duration::from_millis(1).as_millis() as u64,
                 num_bootstrap: 1usize,
                 da_staked_committee_size: num_nodes_with_stake,
                 my_own_validator_config: Default::default(),
@@ -233,7 +221,9 @@ pub mod testing {
                 }
             } else {
                 ValidatorConfig {
-                    public_key: self.config.known_nodes_without_stake[i],
+                    public_key: self.config.known_nodes_with_stake[i]
+                        .stake_table_entry
+                        .stake_key,
                     private_key: self.priv_keys_non_staking_nodes[i].clone(),
                     stake_value: 0,
                     state_key_pair: self.non_staking_nodes_state_key_pairs[i].clone(),

sequencer/src/api/data_source.rs

@@ -24,7 +24,7 @@ use hotshot_types::{
     light_client::StateSignatureRequestBody,
     network::NetworkConfig,
     traits::{network::ConnectedNetwork, node_implementation::Versions},
-    ExecutionType, HotShotConfig, PeerConfig, ValidatorConfig,
+    HotShotConfig, PeerConfig, ValidatorConfig,
 };
 use serde::{Deserialize, Serialize};
 use tide_disco::Url;
@@ -212,20 +212,15 @@ impl From<ValidatorConfig<PubKey>> for PublicValidatorConfig {
 /// Hotshot config has sensitive information like private keys and such fields are excluded from this struct.
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct PublicHotShotConfig {
-    execution_type: ExecutionType,
     start_threshold: (u64, u64),
     num_nodes_with_stake: NonZeroUsize,
     known_nodes_with_stake: Vec<PeerConfig<PubKey>>,
     known_da_nodes: Vec<PeerConfig<PubKey>>,
-    known_nodes_without_stake: Vec<PubKey>,
     my_own_validator_config: PublicValidatorConfig,
     da_staked_committee_size: usize,
     fixed_leader_for_gpuvid: usize,
     next_view_timeout: u64,
     view_sync_timeout: Duration,
-    timeout_ratio: (u64, u64),
-    round_start_delay: u64,
-    start_delay: u64,
     num_bootstrap: usize,
     builder_timeout: Duration,
     data_request_delay: Duration,
@@ -247,20 +242,15 @@ impl From<HotShotConfig<PubKey>> for PublicHotShotConfig {
         // if new fields are added to HotShotConfig. This makes sure that we handle
         // all fields appropriately and do not miss any updates.
         let HotShotConfig::<PubKey> {
-            execution_type,
             start_threshold,
             num_nodes_with_stake,
             known_nodes_with_stake,
             known_da_nodes,
-            known_nodes_without_stake,
             my_own_validator_config,
             da_staked_committee_size,
             fixed_leader_for_gpuvid,
             next_view_timeout,
             view_sync_timeout,
-            timeout_ratio,
-            round_start_delay,
-            start_delay,
             num_bootstrap,
             builder_timeout,
             data_request_delay,
@@ -277,20 +267,15 @@ impl From<HotShotConfig<PubKey>> for PublicHotShotConfig {
         } = v;
 
         Self {
-            execution_type,
             start_threshold,
             num_nodes_with_stake,
             known_nodes_with_stake,
             known_da_nodes,
-            known_nodes_without_stake,
             my_own_validator_config: my_own_validator_config.into(),
             da_staked_committee_size,
             fixed_leader_for_gpuvid,
             next_view_timeout,
             view_sync_timeout,
-            timeout_ratio,
-            round_start_delay,
-            start_delay,
             num_bootstrap,
             builder_timeout,
             data_request_delay,
@@ -314,20 +299,15 @@ impl PublicHotShotConfig {
         my_own_validator_config: ValidatorConfig<PubKey>,
     ) -> HotShotConfig<PubKey> {
         HotShotConfig {
-            execution_type: self.execution_type,
             start_threshold: self.start_threshold,
             num_nodes_with_stake: self.num_nodes_with_stake,
             known_nodes_with_stake: self.known_nodes_with_stake,
             known_da_nodes: self.known_da_nodes,
-            known_nodes_without_stake: self.known_nodes_without_stake,
             my_own_validator_config,
             da_staked_committee_size: self.da_staked_committee_size,
             fixed_leader_for_gpuvid: self.fixed_leader_for_gpuvid,
             next_view_timeout: self.next_view_timeout,
             view_sync_timeout: self.view_sync_timeout,
-            timeout_ratio: self.timeout_ratio,
-            round_start_delay: self.round_start_delay,
-            start_delay: self.start_delay,
             num_bootstrap: self.num_bootstrap,
             builder_timeout: self.builder_timeout,
             data_request_delay: self.data_request_delay,
@@ -359,7 +339,6 @@ pub struct PublicNetworkConfig {
     node_index: u64,
     seed: [u8; 32],
     transaction_size: usize,
-    start_delay_seconds: u64,
     key_type_name: String,
     libp2p_config: Option<Libp2pConfig>,
     config: PublicHotShotConfig,
@@ -385,7 +364,6 @@ impl From<NetworkConfig<PubKey>> for PublicNetworkConfig {
             node_index: cfg.node_index,
             seed: cfg.seed,
             transaction_size: cfg.transaction_size,
-            start_delay_seconds: cfg.start_delay_seconds,
             key_type_name: cfg.key_type_name,
             libp2p_config: cfg.libp2p_config,
             config: cfg.config.into(),
@@ -426,7 +404,6 @@ impl PublicNetworkConfig {
             node_index,
             seed: self.seed,
             transaction_size: self.transaction_size,
-            start_delay_seconds: self.start_delay_seconds,
             key_type_name: self.key_type_name,
             libp2p_config: self.libp2p_config,
             config: self.config.into_hotshot_config(my_own_validator_config),

sequencer/src/bin/orchestrator.rs

@@ -118,7 +118,6 @@ async fn main() {
     args.logging.init();
 
     let mut config = NetworkConfig::<PubKey> {
-        start_delay_seconds: args.start_delay.as_secs(),
         manual_start_password: args.manual_start_password,
         indexed_da: false,
         ..Default::default()
@@ -132,13 +131,9 @@ async fn main() {
     config.config.num_nodes_with_stake = args.num_nodes;
     config.config.known_nodes_with_stake = vec![Default::default(); args.num_nodes.get()];
     config.config.known_da_nodes = Vec::new();
-    config.config.known_nodes_without_stake = vec![];
     config.config.next_view_timeout = args.next_view_timeout.as_millis() as u64;
     config.libp2p_config = Some(libp2p_config);
-    config.config.timeout_ratio = args.timeout_ratio.into();
     config.config.start_threshold = args.start_threshold.into();
-    config.config.round_start_delay = args.round_start_delay.as_millis() as u64;
-    config.config.start_delay = args.start_delay.as_millis() as u64;
     config.config.da_staked_committee_size = args.num_nodes.get();
     config.config.builder_urls = Vec1::try_from_vec(args.builder_urls).unwrap();
     config.config.builder_timeout = args.builder_timeout;

sequencer/src/block/full_payload/payload.rs

@@ -102,13 +102,8 @@ impl Payload {
         // add each tx to its namespace
         let mut ns_builders = BTreeMap::<NamespaceId, NsPayloadBuilder>::new();
         for tx in transactions.into_iter() {
-            // accounting for block byte length limit
-            block_byte_len += tx.payload().len() + NsPayloadBuilder::tx_table_entry_byte_len();
-            if !ns_builders.contains_key(&tx.namespace()) {
-                // each new namespace adds overhead
-                block_byte_len +=
-                    NsTableBuilder::entry_byte_len() + NsPayloadBuilder::tx_table_header_byte_len();
-            }
+            // accounting for block byte length limit            
+            block_byte_len += tx.size_in_block(!ns_builders.contains_key(&tx.namespace()));
             if block_byte_len > max_block_byte_len {
                 tracing::warn!("transactions truncated to fit in maximum block byte length {max_block_byte_len}");
                 break;

sequencer/src/lib.rs

@@ -563,7 +563,7 @@ pub mod testing {
             node_implementation::ConsensusTime,
             stake_table::StakeTableScheme,
         },
-        ExecutionType, HotShotConfig, PeerConfig,
+        HotShotConfig, PeerConfig,
     };
     use marketplace_builder_core::{
         builder_state::BuilderState,
@@ -798,15 +798,10 @@ pub mod testing {
 
             let config: HotShotConfig<PubKey> = HotShotConfig {
                 fixed_leader_for_gpuvid: 0,
-                execution_type: ExecutionType::Continuous,
                 num_nodes_with_stake: num_nodes.try_into().unwrap(),
                 known_da_nodes: known_nodes_with_stake.clone(),
                 known_nodes_with_stake: known_nodes_with_stake.clone(),
-                known_nodes_without_stake: vec![],
                 next_view_timeout: Duration::from_secs(5).as_millis() as u64,
-                timeout_ratio: (10, 11),
-                round_start_delay: Duration::from_millis(1).as_millis() as u64,
-                start_delay: Duration::from_millis(1).as_millis() as u64,
                 num_bootstrap: 1usize,
                 da_staked_committee_size: num_nodes,
                 my_own_validator_config: Default::default(),

sequencer/src/persistence/sql.rs

@@ -28,7 +28,6 @@ use hotshot_types::{
     utils::View,
     vote::HasViewNumber,
 };
-use sqlx::Postgres;
 use sqlx::Row;
 use sqlx::{query, Executor};
 use std::{collections::BTreeMap, time::Duration};
@@ -119,7 +118,7 @@ pub struct Options {
     pub(crate) archive: bool,
 }
 
-impl TryFrom<Options> for Config<Postgres> {
+impl TryFrom<Options> for Config {
     type Error = anyhow::Error;
 
     fn try_from(opt: Options) -> Result<Self, Self::Error> {

sequencer/src/restart_tests.rs

@@ -816,7 +816,6 @@ fn start_orchestrator(port: u16, nodes: &[NodeParams], builder_port: u16) -> Joi
     config.config.da_staked_committee_size = num_nodes;
     config.config.known_nodes_with_stake = vec![];
     config.config.known_da_nodes = vec![];
-    config.config.known_nodes_without_stake = vec![];
     config.config.next_view_timeout = view_timeout.as_millis() as u64;
     config.config.builder_timeout = builder_timeout;
     config.config.builder_urls = vec1![format!("http://localhost:{builder_port}").parse().unwrap()];

types/src/v0/impls/block/full_payload/payload.rs

@@ -13,11 +13,12 @@ use jf_vid::VidScheme;
 use sha2::Digest;
 use thiserror::Error;
 
+use crate::Transaction;
 use crate::{
     v0::impls::{NodeState, ValidatedState},
     v0_1::ChainConfig,
     Index, Iter, NamespaceId, NsIndex, NsPayload, NsPayloadBuilder, NsPayloadRange, NsTable,
-    NsTableBuilder, Payload, PayloadByteLen, SeqTypes, Transaction, TxProof,
+    NsTableBuilder, Payload, PayloadByteLen, SeqTypes, TxProof,
 };
 
 #[derive(serde::Deserialize, serde::Serialize, Error, Debug, Eq, PartialEq)]
@@ -79,23 +80,16 @@ impl Payload {
         <Self as BlockPayload<SeqTypes>>::Error,
     > {
         // accounting for block byte length limit
-        let max_block_byte_len: usize = u64::from(chain_config.max_block_size)
-            .try_into()
-            .expect("too large max block size for architecture");
-        let mut block_byte_len = NsTableBuilder::header_byte_len();
+        let max_block_byte_len = u64::from(chain_config.max_block_size);
+        let mut block_byte_len = NsTableBuilder::header_byte_len() as u64;
 
         // add each tx to its namespace
         let mut ns_builders = BTreeMap::<NamespaceId, NsPayloadBuilder>::new();
         for tx in transactions.into_iter() {
-            let mut tx_size = tx.payload().len() + NsPayloadBuilder::tx_table_entry_byte_len();
-            if !ns_builders.contains_key(&tx.namespace()) {
-                // each new namespace adds overhead
-                tx_size +=
-                    NsTableBuilder::entry_byte_len() + NsPayloadBuilder::tx_table_header_byte_len();
-            }
+            let tx_size = tx.size_in_block(!ns_builders.contains_key(&tx.namespace()));
 
             if tx_size > max_block_byte_len {
-                // skip this transaction since it excceds the block size limit
+                // skip this transaction since it exceeds the block size limit
                 tracing::warn!(
                     "skip the transaction to fit in maximum block byte length {max_block_byte_len}, transaction size {tx_size}"
                 );

types/src/v0/impls/transaction.rs

@@ -5,6 +5,8 @@ use serde::{de::Error, Deserialize, Deserializer};
 
 use crate::{NamespaceId, Transaction};
 
+use super::{NsPayloadBuilder, NsTableBuilder};
+
 impl From<u32> for NamespaceId {
     fn from(value: u32) -> Self {
         Self(value as u64)
@@ -60,6 +62,16 @@ impl Transaction {
         self.payload
     }
 
+    pub fn size_in_block(&self, new_ns: bool) -> u64 {
+        if new_ns {
+            // each new namespace adds overhead
+            // here self.minimum_block_size() = `self.payload().len() + NsPayloadBuilder::tx_table_entry_byte_len() + NsTableBuilder::entry_byte_len() + NsPayloadBuilder::tx_table_header_byte_len()`
+            self.minimum_block_size()
+        } else {
+            (self.payload().len() + NsPayloadBuilder::tx_table_entry_byte_len()) as u64
+        }
+    }
+
     #[cfg(any(test, feature = "testing"))]
     pub fn random(rng: &mut dyn rand::RngCore) -> Self {
         use rand::Rng;
@@ -79,7 +91,15 @@ impl Transaction {
     }
 }
 
-impl HotShotTransaction for Transaction {}
+impl HotShotTransaction for Transaction {
+    fn minimum_block_size(&self) -> u64 {
+        let len = self.payload().len()
+            + NsPayloadBuilder::tx_table_entry_byte_len()
+            + NsTableBuilder::entry_byte_len()
+            + NsPayloadBuilder::tx_table_header_byte_len();
+        len as u64
+    }
+}
 
 impl Committable for Transaction {
     fn commit(&self) -> Commitment<Self> {