feat(sp): Add batch events (#612)

Co-authored-by: Konrad Stepniak <[email protected]>

docs/src/architecture/pallets/storage-provider.md

@@ -326,9 +326,9 @@ The Storage Provider Pallet emits the following events:
     - `sector_number` - The sector number that is proven.
     - `partition_number` - The partition number the proven sector is in.
     - `deadline_idx` - The deadline index assigned to the proven sector.
-- `SectorSlashed` - A previously pre-committed sector, but not proven, has been slashed by the system because it has expired.
+- `SectorsSlashed` - Previously pre-committed sectors, but not proven, have been slashed by the system because it has expired.
   - `owner` - SS58 address of the storage provider.
-  - `sector_number` - The sector number that has been slashed because of expiry.
+  - `sector_numbers` - The sector numbers that have been slashed because of expiry.
 - `ValidPoStSubmitted` - A valid PoSt has been submitted by a storage provider.
   - `owner` - SS58 address of the storage provider.
 - `FaultsDeclared` - A storage provider has declared some sectors as faulty.
@@ -343,10 +343,9 @@ The Storage Provider Pallet emits the following events:
     - `deadline` - The deadline to which the recovered sectors are assigned.
     - `partition` - Partition number within the deadline containing the recovered sectors.
     - `sectors` - Sectors in the partition being declared as recovered.
-- `PartitionFaulty` - It was detected that a storage provider has not submitted their PoSt on time and has marked some sectors as faulty.
+- `PartitionsFaulty` - It was detected that a storage provider has not submitted their PoSt on time and has marked some partitions and sectors as faulty.
   - `owner` - SS58 address of the storage provider.
-  - `partition` - Partition number for which the PoSt was missed.
-  - `sectors` - The sectors in the partition declared faulty by the system.
+  - `faulty_partitions` - A map with partition numbers and the sectors associated with the partition number that are faulty.
 - `SectorsTerminated` - A storage provider has terminated some sectors.
   - `owner` - SS58 address of the storage provider.
   - `terminations` - An array with information about the terminated sectors. This information includes:

pallets/storage-provider/src/lib.rs

@@ -35,7 +35,7 @@ pub mod pallet {
 
     extern crate alloc;
 
-    use alloc::{vec, vec::Vec};
+    use alloc::{collections::BTreeMap, vec, vec::Vec};
     use core::fmt::Debug;
 
     use cid::Cid;
@@ -71,7 +71,7 @@ pub mod pallet {
             DeclareFaultsParams, DeclareFaultsRecoveredParams, FaultDeclaration,
             RecoveryDeclaration,
         },
-        partition::PartitionNumber,
+        partition::{PartitionNumber, MAX_PARTITIONS_PER_DEADLINE},
         proofs::{assign_proving_period_offset, SubmitWindowedPoStParams},
         sector::{
             ProveCommitResult, ProveCommitSector, SectorOnChainInfo, SectorPreCommitInfo,
@@ -276,9 +276,10 @@ pub mod pallet {
             sectors: BoundedVec<ProveCommitResult, ConstU32<MAX_SECTORS_PER_CALL>>,
         },
         /// Emitted when a sector was pre-committed, but not proven, so it got slashed in the pre-commit hook.
-        SectorSlashed {
+        SectorsSlashed {
             owner: T::AccountId,
-            sector_number: SectorNumber,
+            // No need for a bounded collection as we produce the output ourselves.
+            sector_numbers: BoundedVec<SectorNumber, ConstU32<MAX_SECTORS>>,
         },
         /// Emitted when an SP submits a valid PoSt
         ValidPoStSubmitted { owner: T::AccountId },
@@ -293,10 +294,13 @@ pub mod pallet {
             recoveries: BoundedVec<RecoveryDeclaration, ConstU32<DECLARATIONS_MAX>>,
         },
         /// Emitted when an SP doesn't submit Windowed PoSt in time and PoSt hook marks partitions as faulty
-        PartitionFaulty {
+        PartitionsFaulty {
             owner: T::AccountId,
-            partition: PartitionNumber,
-            sectors: BoundedBTreeSet<SectorNumber, ConstU32<MAX_SECTORS>>,
+            faulty_partitions: BoundedBTreeMap<
+                PartitionNumber,
+                BoundedBTreeSet<SectorNumber, ConstU32<MAX_SECTORS>>,
+                ConstU32<MAX_PARTITIONS_PER_DEADLINE>,
+            >,
         },
         /// Emitted when an SP terminates some sectors.
         SectorsTerminated {
@@ -1100,18 +1104,16 @@ pub mod pallet {
                     return;
                 }
 
+                let mut removed_sectors = BoundedVec::new();
                 log::info!(target: LOG_TARGET, "found {} expired pre committed sectors for {:?}", expired.len(), storage_provider);
                 for sector_number in expired {
                     // Expired sectors should be removed, because in other case they'd be processed twice in the next block.
-                    let Ok(()) = state.remove_pre_committed_sector(sector_number) else {
+                    if let Ok(()) = state.remove_pre_committed_sector(sector_number) {
+                        removed_sectors.force_push(sector_number)
+                    } else {
                         log::error!(target: LOG_TARGET, "catastrophe, failed to remove sector {} for {:?}", sector_number, storage_provider);
                         continue;
                     };
-
-                    Self::deposit_event(Event::<T>::SectorSlashed {
-                        sector_number,
-                        owner: storage_provider.clone(),
-                    });
                 }
 
                 let Some(slashed_deposits) = state.pre_commit_deposits.checked_sub(&slash_amount)
@@ -1128,6 +1130,10 @@ pub mod pallet {
                 };
 
                 StorageProviders::<T>::insert(&storage_provider, state);
+                Self::deposit_event(Event::<T>::SectorsSlashed {
+                    owner: storage_provider,
+                    sector_numbers: removed_sectors,
+                })
             }
         }
 
@@ -1231,9 +1237,9 @@ pub mod pallet {
                 }
 
                 log::info!(target: LOG_TARGET, "block: {:?}, checking storage provider {:?} deadline: {:?}",
-                   current_block,
-                   storage_provider,
-                   current_deadline.idx,
+                    current_block,
+                    storage_provider,
+                    current_deadline.idx,
                 );
 
                 let Ok(deadline) =
@@ -1244,7 +1250,12 @@ pub mod pallet {
                     continue;
                 };
 
-                let mut faulty_partitions = 0;
+                let mut faulty_partitions_amount = 0;
+                // Create collection for fault partitions, 1 event per SP
+                let mut faulty_partitions: BTreeMap<
+                    PartitionNumber,
+                    BoundedBTreeSet<SectorNumber, ConstU32<MAX_SECTORS>>,
+                > = BTreeMap::new();
                 for (partition_number, partition) in deadline.partitions.iter_mut() {
                     if partition.sectors.len() == 0 {
                         continue;
@@ -1278,24 +1289,23 @@ pub mod pallet {
                     current_block, storage_provider, partition_number, new_faults.len());
 
                     if new_faults.len() > 0 {
-                        Self::deposit_event(Event::PartitionFaulty {
-                            owner: storage_provider.clone(),
-                            partition: *partition_number,
-                            sectors: new_faults.try_into()
-                                .expect("new_faults.len() <= MAX_SECTORS, cannot be more new faults than all of the sectors in partition"),
-                        });
-                        faulty_partitions += 1;
+                        faulty_partitions.insert(*partition_number, new_faults.try_into().expect("should be able to create BoundedBTreeSet due to input being bounded"));
+                        faulty_partitions_amount += 1;
                     }
                 }
 
                 // TODO(@th7nder,[#106,#187],08/08/2024): figure out slashing amounts (for continued faults, new faults).
-                if faulty_partitions > 0 {
+                if faulty_partitions_amount > 0 {
                     log::warn!(target: LOG_TARGET, "block: {:?}, sp: {:?}, deadline: {:?} - should have slashed {} partitions...",
                         current_block,
                         storage_provider,
                         current_deadline.idx,
-                        faulty_partitions,
+                        faulty_partitions_amount,
                     );
+                    Self::deposit_event(Event::PartitionsFaulty {
+                        owner: storage_provider.clone(),
+                        faulty_partitions: faulty_partitions.try_into().expect("should be able to create a BTreeMap with a MAX_PARTITIONS_PER_DEADLINE bound after iterating over a map with the same bound"),
+                    })
                 } else {
                     log::info!(target: LOG_TARGET, "block: {:?}, sp: {:?}, deadline: {:?} - all proofs submitted on time.",
                         current_block,

pallets/storage-provider/src/tests/post_hook.rs

@@ -1,8 +1,7 @@
-extern crate alloc;
-
 use frame_support::{assert_ok, pallet_prelude::Get};
 use primitives_proofs::{DealId, SectorNumber};
 use sp_core::bounded_vec;
+use sp_runtime::{BoundedBTreeMap, BoundedBTreeSet};
 
 use super::new_test_ext;
 use crate::{
@@ -68,16 +67,25 @@ fn marks_partitions_as_faulty() {
         let partition = &deadline.partitions[&0];
         let expected_sectors =
             sector_set::<MAX_SECTORS>(&[first_sector_number, second_sector_number]);
-
+        let mut expected_faulty_sectors = BoundedBTreeSet::new();
+        expected_faulty_sectors
+            .try_insert(SectorNumber::new(first_sector_number).unwrap())
+            .unwrap();
+        expected_faulty_sectors
+            .try_insert(SectorNumber::new(second_sector_number).unwrap())
+            .unwrap();
         assert_eq!(partition.faults.len(), 2);
         assert_eq!(expected_sectors, partition.faults);
+        let mut expected_faulty_partitions = BoundedBTreeMap::new();
+        expected_faulty_partitions
+            .try_insert(0, expected_faulty_sectors)
+            .unwrap();
         assert_eq!(
             events(),
             [RuntimeEvent::StorageProvider(
-                Event::<Test>::PartitionFaulty {
+                Event::<Test>::PartitionsFaulty {
                     owner: account(storage_provider),
-                    partition: 0,
-                    sectors: expected_sectors.clone()
+                    faulty_partitions: expected_faulty_partitions,
                 }
             ),]
         );

pallets/storage-provider/src/tests/pre_commit_sector_hook.rs

@@ -1,15 +1,15 @@
-extern crate alloc;
-
+use primitives_proofs::SectorNumber;
 use sp_core::bounded_vec;
+use sp_runtime::{BoundedBTreeMap, BoundedBTreeSet};
 
 use super::new_test_ext;
 use crate::{
     pallet::{Event, StorageProviders},
     sector::ProveCommitSector,
     tests::{
-        account, events, publish_deals, register_storage_provider, run_to_block, sector_set,
-        Balances, RuntimeEvent, RuntimeOrigin, SectorPreCommitInfoBuilder, StorageProvider, System,
-        Test, CHARLIE,
+        account, events, publish_deals, register_storage_provider, run_to_block, Balances,
+        RuntimeEvent, RuntimeOrigin, SectorPreCommitInfoBuilder, StorageProvider, System, Test,
+        CHARLIE,
     },
 };
 
@@ -76,17 +76,20 @@ fn pre_commit_hook_slashed_deal() {
             Balances::reserved_balance(account(storage_provider)),
             deal_precommit_deposit
         );
+        let mut expected_faulty_sectors = BoundedBTreeSet::new();
+        expected_faulty_sectors
+            .try_insert(SectorNumber::new(2).unwrap())
+            .unwrap();
+        let mut expected_faulty_partitions = BoundedBTreeMap::new();
+        expected_faulty_partitions
+            .try_insert(0, expected_faulty_sectors)
+            .unwrap();
         assert_eq!(
             events(),
             [
-                RuntimeEvent::StorageProvider(Event::<Test>::PartitionFaulty {
-                    owner: account(storage_provider),
-                    partition: 0,
-                    sectors: sector_set(&[2])
-                }),
-                RuntimeEvent::StorageProvider(Event::<Test>::SectorSlashed {
+                RuntimeEvent::StorageProvider(Event::<Test>::PartitionsFaulty {
                     owner: account(storage_provider),
-                    sector_number: 1.into(),
+                    faulty_partitions: expected_faulty_partitions,
                 }),
                 // the slash -> withdraw is related to the usage of slash_and_burn
                 // when slashing the SP for a failed pre_commit
@@ -105,6 +108,10 @@ fn pre_commit_hook_slashed_deal() {
                 RuntimeEvent::Balances(pallet_balances::Event::<Test>::Rescinded {
                     amount: deal_precommit_deposit
                 }),
+                RuntimeEvent::StorageProvider(Event::<Test>::SectorsSlashed {
+                    owner: account(storage_provider),
+                    sector_numbers: bounded_vec![1.into()],
+                }),
             ]
         );
     });

storagext/lib/src/runtime/display/storage_provider.rs

@@ -87,12 +87,17 @@ impl std::fmt::Display for Event {
                 "Sectors Proven: {{ owner: {}, sectors: {:?} }}",
                 owner, sectors,
             )),
-            Event::SectorSlashed {
+            Event::SectorsSlashed {
                 owner,
-                sector_number,
+                sector_numbers,
             } => f.write_fmt(format_args!(
-                "Sector Slashed: {{ owner: {}, sector_number: {} }}",
-                owner, sector_number,
+                "Sectors Slashed: {{ owner: {}, sector_numbers: {} }}",
+                owner,
+                itertools::Itertools::intersperse(
+                    sector_numbers.0.iter().map(ToString::to_string),
+                    ", ".to_string()
+                )
+                .collect::<String>(),
             )),
             Event::ValidPoStSubmitted { owner } => {
                 f.write_fmt(format_args!("Valid PoSt Submitted: {{ owner: {} }}", owner,))
@@ -115,16 +120,25 @@ impl std::fmt::Display for Event {
                 )
                 .collect::<String>()
             )),
-            Event::PartitionFaulty {
+            Event::PartitionsFaulty {
                 owner,
-                partition,
-                sectors,
+                faulty_partitions,
             } => f.write_fmt(format_args!(
-                "Faulty Partition: {{ owner: {}, partition: {}, sectors: [{}] }}",
+                "Faulty Partitions: {{ owner: {}, faulty_partitions: [{}] }}",
                 owner,
-                partition,
                 itertools::Itertools::intersperse(
-                    sectors.0.iter().map(|recovery| format!("{}", recovery)),
+                    faulty_partitions
+                        .0
+                        .iter()
+                        .map(|(partition, sectors)| format!(
+                            "{{ partition: {}, sectors: {} }}",
+                            partition,
+                            itertools::Itertools::intersperse(
+                                sectors.0.iter().map(ToString::to_string),
+                                ", ".to_string()
+                            )
+                            .collect::<String>()
+                        )),
                     ", ".to_string()
                 )
                 .collect::<String>()

storagext/lib/src/runtime/mod.rs

@@ -72,6 +72,10 @@ pub mod display;
         path = "bounded_collections::bounded_btree_set::BoundedBTreeSet",
         derive = "::serde::Serialize"
     ),
+    derive_for_type(
+        path = "bounded_collections::bounded_btree_map::BoundedBTreeMap",
+        derive = "::serde::Serialize"
+    ),
     derive_for_type(
         path = "pallet_market::pallet::SettledDealData",
         derive = "::serde::Serialize"