From e9f7a07004919a61abe931364b3d6afad11cf6f8 Mon Sep 17 00:00:00 2001 From: Benjamin Saunders Date: Mon, 8 Jan 2024 22:44:12 -0800 Subject: [PATCH 1/2] Define a simpler row-major entity encoding for saves Nodes are unlikely to ever have enough entities in them for column-major representation to be worth the trouble to implement. As a bonus, this representation should be compatible with postcard. We can easily switch this up again in the future if needed thanks to protobuf schema evolution. --- save/src/protos.proto | 20 ++++------------- save/src/protos.rs | 24 +++++--------------- server/Cargo.toml | 2 +- server/src/postcard_helpers.rs | 7 ++++++ server/src/sim.rs | 41 +++++++++++++++++----------------- 5 files changed, 38 insertions(+), 56 deletions(-) diff --git a/save/src/protos.proto b/save/src/protos.proto index db099797..7dcff1ce 100644 --- a/save/src/protos.proto +++ b/save/src/protos.proto @@ -13,21 +13,9 @@ message Character { } message EntityNode { - // Entities whose origins lie within this node - repeated Archetype archetypes = 1; -} - -// A set of entities, all of which have the same components -message Archetype { - // Entity IDs - repeated fixed64 entities = 1; - - // Type of components stored in each column - repeated ComponentType component_types = 2; - - // Each data represents a dense column of component values of the type identified by the - // component_type at the same index as the column - repeated bytes component_data = 3; + // Entities whose origins lie within this node, each encoded as: + // { entity: u64, component_count: varint, components: [{ type: varint, length: varint, data: [u8] }] } + repeated bytes entities = 1; } message VoxelNode { @@ -46,6 +34,6 @@ message Chunk { enum ComponentType { // 4x4 matrix of f32s POSITION = 0; - // Varint length tag followed by UTF-8 text + // UTF-8 text NAME = 1; } diff --git a/save/src/protos.rs b/save/src/protos.rs index 89a01b0c..15246e2a 100644 --- a/save/src/protos.rs +++ b/save/src/protos.rs @@ -15,24 +15,10 @@ pub struct Character { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct EntityNode { - /// Entities whose origins lie within this node - #[prost(message, repeated, tag = "1")] - pub archetypes: ::prost::alloc::vec::Vec, -} -/// A set of entities, all of which have the same components -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, PartialEq, ::prost::Message)] -pub struct Archetype { - /// Entity IDs - #[prost(fixed64, repeated, tag = "1")] - pub entities: ::prost::alloc::vec::Vec, - /// Type of components stored in each column - #[prost(enumeration = "ComponentType", repeated, tag = "2")] - pub component_types: ::prost::alloc::vec::Vec, - /// Each data represents a dense column of component values of the type identified by the - /// component_type at the same index as the column - #[prost(bytes = "vec", repeated, tag = "3")] - pub component_data: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, + /// Entities whose origins lie within this node, each encoded as: + /// { entity: u64, component_count: varint, components: \[{ type: varint, length: varint, data: [u8\] }] } + #[prost(bytes = "vec", repeated, tag = "1")] + pub entities: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -56,7 +42,7 @@ pub struct Chunk { pub enum ComponentType { /// 4x4 matrix of f32s Position = 0, - /// Varint length tag followed by UTF-8 text + /// UTF-8 text Name = 1, } impl ComponentType { diff --git a/server/Cargo.toml b/server/Cargo.toml index 702c1cfd..77472014 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -9,7 +9,7 @@ license = "Apache-2.0 OR Zlib" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -postcard = { version = "1.0.4", default-features = false } +postcard = { version = "1.0.4", default-features = false, features = ["use-std"] } common = { path = "../common" } tracing = "0.1.10" tokio = { version = "1.18.2", features = ["rt-multi-thread", "time", "macros", "sync"] } diff --git a/server/src/postcard_helpers.rs b/server/src/postcard_helpers.rs index 097b616e..b8a7924e 100644 --- a/server/src/postcard_helpers.rs +++ b/server/src/postcard_helpers.rs @@ -25,3 +25,10 @@ impl postcard::ser_flavors::Flavor for ExtendVec<'_> { Ok(()) } } + +#[derive(serde::Serialize, serde::Deserialize)] +pub struct SaveEntity { + /// [`EntityId`] value, represented as an array to avoid wastefully varint encoding random bytes + pub entity: [u8; 8], + pub components: Vec<(u64, Vec)>, +} diff --git a/server/src/sim.rs b/server/src/sim.rs index 53a2dec9..fd8b608d 100644 --- a/server/src/sim.rs +++ b/server/src/sim.rs @@ -6,6 +6,7 @@ use fxhash::{FxHashMap, FxHashSet}; use hecs::Entity; use rand::rngs::SmallRng; use rand::{Rng, SeedableRng}; +use save::ComponentType; use tracing::{error_span, info, trace}; use common::{ @@ -22,7 +23,7 @@ use common::{ EntityId, SimConfig, Step, }; -use crate::postcard_helpers; +use crate::postcard_helpers::{self, SaveEntity}; pub struct Sim { cfg: Arc, @@ -96,12 +97,8 @@ impl Sim { } fn snapshot_node(&self, node: NodeId) -> save::EntityNode { - let mut ids = Vec::new(); - let mut character_transforms = Vec::new(); - let mut character_names = Vec::new(); - let entities = self.graph_entities.get(node); - - for &entity in entities { + let mut entities = Vec::new(); + for &entity in self.graph_entities.get(node) { // TODO: Handle entities other than characters let mut q = self .world @@ -110,21 +107,25 @@ impl Sim { let Some((id, pos, ch)) = q.get() else { continue; }; - ids.push(id.to_bits()); - postcard_helpers::serialize(pos.local.as_ref(), &mut character_transforms).unwrap(); - postcard_helpers::serialize(&ch.name, &mut character_names).unwrap(); + let mut repr = Vec::new(); + postcard_helpers::serialize( + &SaveEntity { + entity: id.to_bits().to_le_bytes(), + components: vec![ + ( + ComponentType::Position as u64, + postcard::to_stdvec(pos.local.as_ref()).unwrap(), + ), + (ComponentType::Name as u64, ch.name.as_bytes().into()), + ], + }, + &mut repr, + ) + .unwrap(); + entities.push(repr); } - save::EntityNode { - archetypes: vec![save::Archetype { - entities: ids, - component_types: vec![ - save::ComponentType::Position.into(), - save::ComponentType::Name.into(), - ], - component_data: vec![character_transforms, character_names], - }], - } + save::EntityNode { entities } } pub fn spawn_character(&mut self, hello: ClientHello) -> (EntityId, Entity) { From 866b3e5ebbdc042b46c788bea5f9464fd8c3fc70 Mon Sep 17 00:00:00 2001 From: Benjamin Saunders Date: Mon, 8 Jan 2024 23:19:31 -0800 Subject: [PATCH 2/2] Save non-character entities --- server/src/sim.rs | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/server/src/sim.rs b/server/src/sim.rs index fd8b608d..8ab95608 100644 --- a/server/src/sim.rs +++ b/server/src/sim.rs @@ -7,7 +7,7 @@ use hecs::Entity; use rand::rngs::SmallRng; use rand::{Rng, SeedableRng}; use save::ComponentType; -use tracing::{error_span, info, trace}; +use tracing::{error, error_span, info, trace}; use common::{ character_controller, dodeca, @@ -99,25 +99,28 @@ impl Sim { fn snapshot_node(&self, node: NodeId) -> save::EntityNode { let mut entities = Vec::new(); for &entity in self.graph_entities.get(node) { - // TODO: Handle entities other than characters - let mut q = self - .world - .query_one::<(&EntityId, &Position, &Character)>(entity) - .unwrap(); - let Some((id, pos, ch)) = q.get() else { + let Ok(entity) = self.world.entity(entity) else { + error!("stale graph entity {:?}", entity); + continue; + }; + let Some(id) = entity.get::<&EntityId>() else { continue; }; + let mut components = Vec::new(); + if let Some(pos) = entity.get::<&Position>() { + components.push(( + ComponentType::Position as u64, + postcard::to_stdvec(pos.local.as_ref()).unwrap(), + )); + } + if let Some(ch) = entity.get::<&Character>() { + components.push((ComponentType::Name as u64, ch.name.as_bytes().into())); + } let mut repr = Vec::new(); postcard_helpers::serialize( &SaveEntity { entity: id.to_bits().to_le_bytes(), - components: vec![ - ( - ComponentType::Position as u64, - postcard::to_stdvec(pos.local.as_ref()).unwrap(), - ), - (ComponentType::Name as u64, ch.name.as_bytes().into()), - ], + components, }, &mut repr, )