diff --git a/rust/demo-app/move/Move.lock b/rust/demo-app/move/Move.lock index 443de1b4..70de3ec4 100644 --- a/rust/demo-app/move/Move.lock +++ b/rust/demo-app/move/Move.lock @@ -3,7 +3,7 @@ [move] version = 0 manifest_digest = "C7E0EBC4DD41B277F44A5FA805716E002069FC359A3BE24E541A14E868DA561C" -deps_digest = "112928C94A84031C09CD9B9D1D44B149B73FC0EEA5FA8D8B2D7CA4D91936335A" +deps_digest = "F8BBB0CCB2491CA29A3DF03D6F92277A4F3574266507ACD77214D37ECA3F3082" dependencies = [ { name = "Sui" }, diff --git a/rust/suibase/.cargo/config.toml b/rust/suibase/.cargo/config.toml new file mode 100644 index 00000000..97e4b5d0 --- /dev/null +++ b/rust/suibase/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.'cfg(all())'] +rustflags = ["--cfg", "uuid_unstable"] diff --git a/rust/suibase/rustfmt.toml b/rust/suibase/.rustfmt.toml similarity index 50% rename from rust/suibase/rustfmt.toml rename to rust/suibase/.rustfmt.toml index 55797efb..9c0edece 100644 --- a/rust/suibase/rustfmt.toml +++ b/rust/suibase/.rustfmt.toml @@ -1,2 +1,5 @@ edition = "2021" use_field_init_shorthand = true +tab_spaces = 4 +max_width = 100 +hard_tabs = false diff --git a/rust/suibase/Cargo.lock b/rust/suibase/Cargo.lock index 8f89e578..c6bc72be 100644 --- a/rust/suibase/Cargo.lock +++ b/rust/suibase/Cargo.lock @@ -125,6 +125,12 @@ dependencies = [ "syn 2.0.22", ] +[[package]] +name = "atomic" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59bdb34bc650a32731b31bd8f0829cc15d24a708ee31559e0bb34f2bc320cba" + [[package]] name = "atty" version = "0.2.14" @@ -578,6 +584,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fstr" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c395dd73333fdeddb92b9d9de4d63e265233a91206de34ca9536473be4c2b5b" + [[package]] name = "futures" version = "0.3.28" @@ -2142,6 +2154,8 @@ dependencies = [ "tower", "tower-http", "twox-hash", + "uuid", + "uuid7", ] [[package]] @@ -2554,11 +2568,25 @@ dependencies = [ [[package]] name = "uuid" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d023da39d1fde5a8a3fe1f3e01ca9632ada0a63e9797de55a879d6e2236277be" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" dependencies = [ + "atomic", "getrandom", + "rand", +] + +[[package]] +name = "uuid7" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcba843a27a44d6bfbc8852222af425ebd04da999d898df30fd2ed16abe469ac" +dependencies = [ + "fstr", + "rand", + "rand_chacha", + "uuid", ] [[package]] diff --git a/rust/suibase/Cargo.toml b/rust/suibase/Cargo.toml index 11055409..7ce6839a 100644 --- a/rust/suibase/Cargo.toml +++ b/rust/suibase/Cargo.toml @@ -44,6 +44,9 @@ tower-http = { version = "0.3.4", features = [ "propagate-header", ] } +uuid = { version = "1.4.1", features = ["v4","v7","fast-rng"] } +uuid7 = { version= "0.7.0", features = [ "uuid" ] } + anyhow = { version = "1.0.71", features = ["backtrace"] } thiserror = "1.0" diff --git a/rust/suibase/crates/suibase-daemon/Cargo.toml b/rust/suibase/crates/suibase-daemon/Cargo.toml index f7cf169a..52368b1f 100644 --- a/rust/suibase/crates/suibase-daemon/Cargo.toml +++ b/rust/suibase/crates/suibase-daemon/Cargo.toml @@ -19,6 +19,7 @@ mime = "0.3.16" memchr = "2.5.0" + tokio.workspace = true tokio-graceful-shutdown.workspace = true @@ -27,6 +28,8 @@ notify = { version = "6.0", default-features = false, features = [ "macos_kqueue", ] } +uuid.workspace = true +uuid7.workspace = true anyhow.workspace = true clap.workspace = true diff --git a/rust/suibase/crates/suibase-daemon/src/api/api_server.rs b/rust/suibase/crates/suibase-daemon/src/api/api_server.rs index d318572a..17374e14 100644 --- a/rust/suibase/crates/suibase-daemon/src/api/api_server.rs +++ b/rust/suibase/crates/suibase-daemon/src/api/api_server.rs @@ -88,7 +88,7 @@ impl JSONRPCServer { } async fn run_server(self, _subsys: &SubsystemHandle) -> Result<()> { - // Refrence: + // Reference: // https://github.com/paritytech/jsonrpsee/blob/master/examples/examples/cors_server.rs let cors = CorsLayer::new() // Allow `POST` when accessing the resource diff --git a/rust/suibase/crates/suibase-daemon/src/api/json_rpc_api.rs b/rust/suibase/crates/suibase-daemon/src/api/json_rpc_api.rs index 741ed943..0e769e31 100644 --- a/rust/suibase/crates/suibase-daemon/src/api/json_rpc_api.rs +++ b/rust/suibase/crates/suibase-daemon/src/api/json_rpc_api.rs @@ -1,3 +1,4 @@ +use hyper::header; // Defines the JSON-RPC API. // // Intended Design (WIP) @@ -11,6 +12,8 @@ // emit a message toward the AdminController describing the action needed. The AdminController perform the // modification and provides the response with a returning tokio OneShot channel. // +// All *successful" JSON responses have a required "header" section. +// use jsonrpsee::core::RpcResult; use jsonrpsee_proc_macros::rpc; @@ -18,6 +21,37 @@ use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_with::serde_as; +#[serde_as] +#[derive(Clone, Default, Debug, JsonSchema, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct Header { + // Header fields + // ============= + // - method: + // A string echoing the method of the request. + // + // - key: + // A string echoing one of the "key" parameter of the request (e.g. the workdir requested). + // This field is optional and its interpretation depends on the method. + // + // - data_uuid: + // A sortable hex 64 bytes (UUID v7). Increments with every data modification. + // + // - server_uuid: + // A hex 64 bytes that changes every time the server detects that + // a data_uuid is unexpectedly lower than the previous one (e.g. system + // time went backward) or the PID of the process changes. This is to + // complement data_version for added reliability. + // + pub method: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub server_uuid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub data_uuid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub key: Option, +} + #[serde_as] #[derive(Clone, Default, Debug, JsonSchema, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] @@ -75,6 +109,8 @@ impl LinksSummary { #[derive(Clone, Debug, JsonSchema, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct LinksResponse { + pub header: Header, + pub status: String, // This is a single word combined "Multi-Link status". Either "OK" or "DOWN". pub info: String, // More details about the status (e.g. '50% degraded', 'all servers down', etc...) @@ -100,6 +136,7 @@ pub struct LinksResponse { impl LinksResponse { pub fn new() -> Self { Self { + header: Header::default(), status: "DISABLED".to_string(), info: "INITIALIZING".to_string(), summary: None, @@ -114,12 +151,14 @@ impl LinksResponse { #[derive(Clone, Debug, JsonSchema, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct InfoResponse { + pub header: Header, pub info: String, // "Success" or info on failure. } impl InfoResponse { pub fn new() -> Self { Self { + header: Header::default(), info: "Unknown Error".to_string(), } } diff --git a/rust/suibase/crates/suibase-daemon/src/api/proxy_api.rs b/rust/suibase/crates/suibase-daemon/src/api/proxy_api.rs index 14b65715..27aa286b 100644 --- a/rust/suibase/crates/suibase-daemon/src/api/proxy_api.rs +++ b/rust/suibase/crates/suibase-daemon/src/api/proxy_api.rs @@ -1,5 +1,6 @@ use core::fmt; use std::fmt::Display; +use tokio::sync::Mutex; use axum::async_trait; @@ -11,21 +12,77 @@ use crate::admin_controller::{ AdminControllerMsg, AdminControllerTx, EVENT_NOTIF_CONFIG_FILE_CHANGE, }; use crate::basic_types::TargetServerIdx; -use crate::shared_types::{Globals, ServerStats, TargetServer}; +use crate::shared_types::{Globals, ServerStats, SingleThreadUUID, TargetServer}; use super::{InfoResponse, ProxyApiServer}; use super::{LinkStats, LinksResponse, LinksSummary, RpcInputError}; +struct GetLinksInput { + pub target_servers_stats: Option>, + pub all_servers_stats: Option, + pub selection_vectors: Option>>, + pub input_port_found: bool, + pub proxy_enabled: bool, + pub user_request_start: bool, + pub uuid: Option, +} + +impl GetLinksInput { + fn new(uuid: Option) -> Self { + Self { + target_servers_stats: None, + all_servers_stats: None, + selection_vectors: None, + input_port_found: false, + proxy_enabled: false, + user_request_start: false, + uuid, + } + } + + fn clone_from(&mut self, other: &Self) { + self.target_servers_stats = other.target_servers_stats.clone(); + self.all_servers_stats = other.all_servers_stats.clone(); + self.selection_vectors = other.selection_vectors.clone(); + self.input_port_found = other.input_port_found; + self.proxy_enabled = other.proxy_enabled; + self.user_request_start = other.user_request_start; + + // If the other Uuid is Some, then clone it, otherwise + // increment the self.uuid "in-place". + if let Some(other_uuid) = &other.uuid { + self.uuid = Some(other_uuid.clone()); + } else { + self.uuid.as_mut().unwrap().increment(); + } + } +} + +impl PartialEq for GetLinksInput { + fn eq(&self, other: &Self) -> bool { + // Note: uuid is not compared. Allow to compare other when not yet "versioned". + self.all_servers_stats.as_ref() == other.all_servers_stats.as_ref() + && self.target_servers_stats == other.target_servers_stats + && self.selection_vectors == other.selection_vectors + && self.input_port_found == other.input_port_found + && self.proxy_enabled == other.proxy_enabled + && self.user_request_start == other.user_request_start + } +} + pub struct ProxyApiImpl { pub globals: Globals, pub admctrl_tx: AdminControllerTx, + prev_get_links_input: Mutex, } impl ProxyApiImpl { pub fn new(globals: Globals, admctrl_tx: AdminControllerTx) -> Self { + let prev_get_links_input = Mutex::new(GetLinksInput::new(Some(SingleThreadUUID::new()))); Self { globals, admctrl_tx, + prev_get_links_input, } } @@ -72,18 +129,18 @@ impl ProxyApiImpl { // // Expected input range is "0" or "0.0" to "100.0" // - // Empty or invalid input is formated as " -" + // Empty or invalid input is formatted as " -" // // Any value above "100" is ignored. // Any non-numeric value is ignored. // Only one decimal is displayed (rounding applies). // // Examples: - // "0" is interpreted as x == 0 and formated as " 0.0" - // "0.0" is interpreted as 0 < x < 0.1 and formated as " <0.1" - // "100" is interpreted as x == 100 and formated as "100.0" - // "105.28" is interpreted as x == 100 and formated as "100.0" - // "0.19" is rounded to 0.2 and formated as " 0.2" + // "0" is interpreted as x == 0 and formatted as " 0.0" + // "0.0" is interpreted as 0 < x < 0.1 and formatted as " <0.1" + // "100" is interpreted as x == 100 and formatted as "100.0" + // "105.28" is interpreted as x == 100 and formatted as "100.0" + // "0.19" is rounded to 0.2 and formatted as " 0.2" let value = input .chars() .filter(|c| c.is_ascii_digit() || *c == '.') @@ -188,6 +245,10 @@ impl ProxyApiServer for ProxyApiImpl { ) -> RpcResult { let mut resp = LinksResponse::new(); + // Initialize some of the header fields. + resp.header.method = "getLinks".to_string(); + resp.header.key = Some(workdir.clone()); + // "Unwrap" all the options to booleans. // // Summary/links is the enabling of group of metrics. @@ -211,12 +272,7 @@ impl ProxyApiServer for ProxyApiImpl { let mut debug_out = String::new(); // Variables initialized during the read lock. - let mut target_servers_stats: Option> = None; - let mut all_servers_stats: Option = None; - let mut selection_vectors: Option>> = None; - let mut input_port_found = false; - let mut proxy_enabled = false; - let mut user_request_start = false; + let mut inputs = GetLinksInput::new(None); { // Get read lock access to the globals and just quickly copy what is needed. // Most parsing and processing is done outside the lock. @@ -224,21 +280,21 @@ impl ProxyApiServer for ProxyApiImpl { let globals = &*globals_read_guard; if let Some(input_port) = globals.find_input_port_by_name(&workdir) { - input_port_found = true; - proxy_enabled = input_port.is_proxy_enabled(); - user_request_start = input_port.is_user_request_start(); + inputs.input_port_found = true; + inputs.proxy_enabled = input_port.is_proxy_enabled(); + inputs.user_request_start = input_port.is_user_request_start(); - all_servers_stats = Some(input_port.all_servers_stats.clone()); + inputs.all_servers_stats = Some(input_port.all_servers_stats.clone()); let target_servers = &input_port.target_servers; - target_servers_stats = Some( + inputs.target_servers_stats = Some( target_servers .iter() .map(|(idx, target_server)| (idx, target_server.stats.clone())) .collect(), ); - selection_vectors = Some(input_port.selection_vectors.clone()); + inputs.selection_vectors = Some(input_port.selection_vectors.clone()); } // If debug, then extensively add more info to the output. @@ -246,6 +302,26 @@ impl ProxyApiServer for ProxyApiImpl { if debug { debug_out.push_str(&format!("{:?}", globals)); } + + // If data, then handle potential UUID increment. + if data { + // To avoid race condition, prev_get_links_input is lock and modified only here. + // Outside the lock, use 'inputs' within this thread. + let prev_get_links_input = &mut *self.prev_get_links_input.lock().await; + if inputs != *prev_get_links_input { + // Because input.uuid is None, this will clone all the data AND increment + // the existing uuid. + prev_get_links_input.clone_from(&inputs); + } + + // Make sure prev_get_links_input.uuid is initialized and inputs.uuid is same. + if let Some(prev_uuid) = &prev_get_links_input.uuid { + inputs.uuid = Some(prev_uuid.clone()); + } else { + prev_get_links_input.uuid = Some(SingleThreadUUID::new()); + inputs.uuid = prev_get_links_input.uuid.clone(); + } + } } // Release the read lock. // Map the target_servers_stats into the API LinkStats. @@ -253,21 +329,21 @@ impl ProxyApiServer for ProxyApiImpl { let mut neutral_health_count: usize = 0; let mut link_stats: Vec = Vec::new(); let mut load_distribution_depth = 0; - if let Some(target_servers_stats) = target_servers_stats { + if let Some(target_servers_stats) = inputs.target_servers_stats { let mut total_request: u64 = 0; let mut link_n_request: Vec = Vec::with_capacity(target_servers_stats.len()); // Prepare LinkStats, which is the "metrics" portion of the API. // // The "display/debug" portion is built from the "metrics" portion. // - // The design seems a bit innefficient (extra string conversion), but the + // The design seems a bit inefficient (extra string conversion), but the // intention is to give more opportunity to catch bugs by using (earlier // than typical) the least visible (but crucial) metrics portion. // Use a vector of indices to drive the display order. // Also find which selections were assigned for load distribution (if any). let mut indices: Vec = Vec::new(); - if let Some(selection_vectors) = selection_vectors { + if let Some(selection_vectors) = inputs.selection_vectors { if !selection_vectors.is_empty() { load_distribution_depth = selection_vectors[0].len(); } @@ -352,7 +428,7 @@ impl ProxyApiServer for ProxyApiImpl { // Map the all_servers_stats into the API LinksSummary. let mut summary_stats = LinksSummary::new(); - if let Some(all_servers_stats) = all_servers_stats { + if let Some(all_servers_stats) = inputs.all_servers_stats { summary_stats.success_on_first_attempt = all_servers_stats.success_on_first_attempt(); summary_stats.success_on_retry = all_servers_stats.success_on_retry(); all_servers_stats.get_classified_failure( @@ -362,7 +438,7 @@ impl ProxyApiServer for ProxyApiImpl { ); } - if !input_port_found { + if !inputs.input_port_found { return Err(RpcInputError::InvalidParams("workdir".to_string(), workdir).into()); } @@ -374,9 +450,9 @@ impl ProxyApiServer for ProxyApiImpl { }; let server_count = link_stats.len(); - (resp.status, resp.info) = if !proxy_enabled { + (resp.status, resp.info) = if !inputs.proxy_enabled { ("DOWN".to_string(), "proxy not enabled".to_string()) - } else if !user_request_start { + } else if !inputs.user_request_start { ("DOWN".to_string(), format!("{} not started", workdir)) } else if server_count == 0 { ("DOWN".to_string(), "no links in suibase.yaml".to_string()) @@ -385,7 +461,12 @@ impl ProxyApiServer for ProxyApiImpl { } else if healthy_server_count == 0 { ("DOWN".to_string(), "no servers available".to_string()) } else if healthy_server_count * 100 / server_count > 50 { - ("OK".to_string(), format!("protected{}", load_balance_str)) + let resp_info = if workdir == "localnet" { + load_balance_str + } else { + format!("protected{}", load_balance_str) + }; + ("OK".to_string(), resp_info) } else { ( "OK".to_string(), @@ -398,16 +479,21 @@ impl ProxyApiServer for ProxyApiImpl { if display { // User requested human-friendly display. if summary { + let resp_info = if resp.info.is_empty() { + String::new() + } else { + format!(" ( {} )", resp.info) + }; display_out.push_str(&format!( - "multi-link RPC: {} ( {} )\n\n\ - Cummulative Request Stats\n\ + "multi-link RPC: {}{}\n\n\ + Cumulative Request Stats\n\ -------------------------\n\ Success first attempt {:>9}\n\ Success after retry {:>9}\n\ Failure bad request {:>9}\n\ Failure others {:>9}\n\n", resp.status, - resp.info, + resp_info, summary_stats.success_on_first_attempt, summary_stats.success_on_retry, summary_stats.fail_bad_request, @@ -471,6 +557,12 @@ impl ProxyApiServer for ProxyApiImpl { if links { resp.links = Some(link_stats); } + + if let Some(uuid) = inputs.uuid { + let (server_uuid, data_uuid) = uuid.get(); + resp.header.data_uuid = Some(data_uuid.to_string()); + resp.header.server_uuid = Some(server_uuid.to_string()); + } } Ok(resp) @@ -479,6 +571,9 @@ impl ProxyApiServer for ProxyApiImpl { async fn fs_change(&self, path: String) -> RpcResult { let mut resp = InfoResponse::new(); + // Initialize some of the header fields. + resp.header.method = "fsChange".to_string(); + // Inform the AdminController that something changed... let mut msg = AdminControllerMsg::new(); msg.event_id = EVENT_NOTIF_CONFIG_FILE_CHANGE; diff --git a/rust/suibase/crates/suibase-daemon/src/shared_types/input_port.rs b/rust/suibase/crates/suibase-daemon/src/shared_types/input_port.rs index dd7a8747..58f85c36 100644 --- a/rust/suibase/crates/suibase-daemon/src/shared_types/input_port.rs +++ b/rust/suibase/crates/suibase-daemon/src/shared_types/input_port.rs @@ -11,7 +11,7 @@ use twox_hash::XxHash32; pub struct InputPort { idx: Option, - // The name of the workdir (e.g. localnet). Set once at contruction. + // The name of the workdir (e.g. localnet). Set once at construction. workdir_name: String, // The workdir idx (from AdminController context). Set once at construction. @@ -43,7 +43,7 @@ pub struct InputPort { pub all_servers_stats: ServerStats, // The "TargetServer" selection vectors are updated periodically by - // the NetworkMonitor. They help the handler to very quicly pick + // the NetworkMonitor. They help the handler to very quickly pick // a set of TargetServer to try. // // Design: @@ -55,7 +55,7 @@ pub struct InputPort { pub selection_vectors: Vec>, // All remaining TargetServerIdx that are not in selection_vectors because - // not in OK state (could be fine rigt now, but not yet known). These are the + // not in OK state (could be fine right now, but not yet known). These are the // fallback attempts on initialization or hard recovery (least worst first). pub selection_worst: Vec, } @@ -286,7 +286,7 @@ impl InputPort { // Build a vector of idx() of the elements of target_servers. // At same time, find one currently OK with the best latency_avg(). - // Isolate immediatly all down target servers in selection_worst. + // Isolate immediately all down target servers in selection_worst. let mut ok_idx_vec: Vec = Vec::new(); let mut best_latency_avg: f64 = f64::MAX; let mut best_latency_avg_idx: Option = None; diff --git a/rust/suibase/crates/suibase-daemon/src/shared_types/mod.rs b/rust/suibase/crates/suibase-daemon/src/shared_types/mod.rs index c354aa85..77a82d06 100644 --- a/rust/suibase/crates/suibase-daemon/src/shared_types/mod.rs +++ b/rust/suibase/crates/suibase-daemon/src/shared_types/mod.rs @@ -1,4 +1,4 @@ -// This is for shared variables (used by multipled thread). +// This is for shared variables (used by more than one thread). // // This is a submodule specific to suibase-daemon. // @@ -7,10 +7,12 @@ pub(crate) use self::globals::*; pub(crate) use self::input_port::*; pub(crate) use self::server_stats::*; pub(crate) use self::target_server::*; +pub(crate) use self::uuid::*; pub(crate) use self::workdirs::*; mod globals; mod input_port; mod server_stats; mod target_server; +mod uuid; mod workdirs; diff --git a/rust/suibase/crates/suibase-daemon/src/shared_types/server_stats.rs b/rust/suibase/crates/suibase-daemon/src/shared_types/server_stats.rs index 9e8ec111..20ad00a4 100644 --- a/rust/suibase/crates/suibase-daemon/src/shared_types/server_stats.rs +++ b/rust/suibase/crates/suibase-daemon/src/shared_types/server_stats.rs @@ -43,7 +43,7 @@ pub const SEND_FAILED_LAST_REASON: u8 = SEND_FAILED_UNSPECIFIED_STATUS; // Do not touch this. pub const SEND_FAILED_VEC_SIZE: usize = SEND_FAILED_LAST_REASON as usize + 1; -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct ServerStats { // Keep a copy of the server alias here because it is very // practical later while processing API calls. @@ -74,7 +74,7 @@ pub struct ServerStats { // Theses are specific failure counts for request. // // There could be multiple send failure (retries) per - // request so these are counted seperatly. + // request so these are counted separately. // // unknown_reason are for when something out-of-range // is reported (that would be a bug). @@ -187,7 +187,7 @@ impl ServerStats { // Sum all the request failures. let total = self.get_accum_failure(); - // Now isolate a few noteable one for the caller. + // Now isolate a few notable one for the caller. *network_down = self.req_failure_reasons[REQUEST_FAILED_NETWORK_DOWN as usize]; *bad_request = self.req_failure_reasons[REQUEST_FAILED_BAD_REQUEST_HTTP as usize]; *other_failures = total - (*network_down + *bad_request); @@ -281,7 +281,7 @@ impl ServerStats { self.send_failure_reasons[reason as usize] += 1; match reason { SEND_FAILED_UNSPECIFIED_ERROR => { - self.error_info = Some("Server Unreacheable".to_string()) + self.error_info = Some("Server Unreachable".to_string()) } SEND_FAILED_RESP_HTTP_STATUS => { let status_code = http::StatusCode::from_u16(status); diff --git a/rust/suibase/crates/suibase-daemon/src/shared_types/uuid.rs b/rust/suibase/crates/suibase-daemon/src/shared_types/uuid.rs new file mode 100644 index 00000000..a02168ac --- /dev/null +++ b/rust/suibase/crates/suibase-daemon/src/shared_types/uuid.rs @@ -0,0 +1,128 @@ +// MTSafeUUID provides: +// - a get() function that returns both a server_id (UUID v4) and a data_version (UUID v7). +// - The server_id is initialized once on process startup and changes whenever +// a data_version is unexpectedly not higher than the previous one generated. +// - get() is multi-thread safe (Mutex protected). +// +// SingleThreadUUID is same, except the user is responsible for Mutex access. +// +use std::sync::{Arc, Mutex}; +use uuid::{Uuid, Variant, Version}; +use uuid7::{uuid7, V7Generator}; + +#[cfg(not(test))] +use log::{info, warn}; + +#[cfg(test)] +use std::{println as info, println as warn}; + +#[derive(Clone, Debug)] +pub struct SingleThreadUUID { + server_id: Uuid, + data_version: Uuid, +} + +impl SingleThreadUUID { + pub fn new() -> Self { + Self { + server_id: Uuid::new_v4(), + data_version: uuid7::new_v7(), + } + } + + pub fn get(&self) -> (Uuid, Uuid) { + (self.server_id, self.data_version) + } + + pub fn set(&mut self, other: &Self) { + self.server_id = other.server_id; + self.data_version = other.data_version; + } + + pub fn increment(&mut self) { + let new_data_version: Uuid = uuid7::new_v7(); + + //info!("data_version: {}", new_data_version.to_string()); + //info!("server_id: {}", self.server_id.to_string()); + /* + if let Some(version) = new_data_version.get_version() { + if version != Version::SortRand { + warn!("WARNING: UUID data_version is not random"); + } + }*/ + if new_data_version <= self.data_version { + self.server_id = Uuid::new_v4(); + } + self.data_version = new_data_version; + } +} + +impl PartialEq for SingleThreadUUID { + fn eq(&self, other: &Self) -> bool { + self.server_id == other.server_id && self.data_version == other.data_version + } +} + +pub type MTSafeUUID = Arc>; + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_one_thread_uuid() { + let single_thread_uuid = SingleThreadUUID::new(); + let mt_safe_uuid = Arc::new(tokio::sync::Mutex::new(single_thread_uuid)); + let mut locked_uuid = mt_safe_uuid.lock().await; + let mut prev_data_version = locked_uuid.data_version; + let initial_server_id = locked_uuid.server_id; + for _ in 0..100000 { + locked_uuid.increment(); + let (server_id, data_version) = locked_uuid.get(); + + assert_eq!(server_id, initial_server_id); + assert!(data_version > prev_data_version); + + prev_data_version = data_version; + } + } + + #[tokio::test] + async fn test_two_threads_uuid() { + let single_thread_uuid = SingleThreadUUID::new(); + let mt_safe_uuid = Arc::new(tokio::sync::Mutex::new(single_thread_uuid)); + let mt_safe_uuid_clone = mt_safe_uuid.clone(); + + let (initial_server_id, mut prev_data_version) = { + let locked_uuid = mt_safe_uuid.lock().await; + (locked_uuid.server_id, locked_uuid.data_version) + }; + + let (_result1, _result2) = tokio::join!( + async move { + for _ in 0..100000 { + let mut locked_uuid = mt_safe_uuid.lock().await; + locked_uuid.increment(); + let (server_id, data_version) = locked_uuid.get(); + + assert_eq!(server_id, initial_server_id); + assert!(data_version > prev_data_version); + + prev_data_version = data_version; + } + }, + async move { + for _ in 0..100000 { + let mut locked_uuid = mt_safe_uuid_clone.lock().await; + locked_uuid.increment(); + let (server_id, data_version) = locked_uuid.get(); + + assert_eq!(server_id, initial_server_id); + assert!(data_version > prev_data_version); + + prev_data_version = data_version; + } + } + ); + } +} diff --git a/rust/suibase/crates/suibase-daemon/src/workdirs_watcher.rs b/rust/suibase/crates/suibase-daemon/src/workdirs_watcher.rs index d5c0e466..122b119c 100644 --- a/rust/suibase/crates/suibase-daemon/src/workdirs_watcher.rs +++ b/rust/suibase/crates/suibase-daemon/src/workdirs_watcher.rs @@ -9,8 +9,8 @@ use crate::admin_controller::{ }; use crate::shared_types::{Workdir, Workdirs}; -use notify::{Watcher, PollWatcher}; use notify::{Error, Event, RecommendedWatcher, RecursiveMode}; +use notify::{PollWatcher, Watcher}; pub struct WorkdirsWatcher { workdirs: Workdirs, @@ -272,10 +272,9 @@ impl WorkdirsWatcher { .unwrap(); let poll_watcher_config = notify::Config::default(); - - let mut poll_watcher = - // notify::recommended_watcher(move |res: Result| match res { - PollWatcher::new(move |res: Result| match res { + + let mut poll_watcher = PollWatcher::new( + move |res: Result| match res { Ok(event) => { //log::info!("watcher step 1 event {:?}", event); let event_to_spawned_fn = event; //.clone(); @@ -290,7 +289,9 @@ impl WorkdirsWatcher { Err(e) => { log::error!("watcher error: {:?}", e); } - }, poll_watcher_config.with_poll_interval(std::time::Duration::from_secs(15)) )?; + }, + poll_watcher_config.with_poll_interval(std::time::Duration::from_secs(15)), + )?; { let workdirs_guard = self.workdirs.read().await; diff --git a/typescript/vscode-extension/package.json b/typescript/vscode-extension/package.json index 76a42f80..2629dd09 100644 --- a/typescript/vscode-extension/package.json +++ b/typescript/vscode-extension/package.json @@ -84,6 +84,7 @@ "typescript": "^5.1.6" }, "dependencies": { + "rpc-websockets": "^7.6.0", "ws": "^8.14.2" } } diff --git a/typescript/vscode-extension/pnpm-lock.yaml b/typescript/vscode-extension/pnpm-lock.yaml index 434b6fce..9adb7f41 100644 --- a/typescript/vscode-extension/pnpm-lock.yaml +++ b/typescript/vscode-extension/pnpm-lock.yaml @@ -5,9 +5,12 @@ settings: excludeLinksFromLockfile: false dependencies: + rpc-websockets: + specifier: ^7.6.0 + version: 7.6.0 ws: specifier: ^8.14.2 - version: 8.14.2 + version: 8.14.2(bufferutil@4.0.7)(utf-8-validate@5.0.10) devDependencies: '@types/mocha': @@ -51,6 +54,13 @@ packages: engines: {node: '>=0.10.0'} dev: true + /@babel/runtime@7.23.1: + resolution: {integrity: sha512-hC2v6p8ZSI/W0HUzh3V8C5g+NwSKzKPtJwSpTjwl0o297GP9+ZLQSkdvHz46CM3LqyoXxq+5G9komY+eSqSO0g==} + engines: {node: '>=6.9.0'} + dependencies: + regenerator-runtime: 0.14.0 + dev: false + /@eslint-community/eslint-utils@4.4.0(eslint@8.49.0): resolution: {integrity: sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -431,6 +441,14 @@ packages: resolution: {integrity: sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==} dev: true + /bufferutil@4.0.7: + resolution: {integrity: sha512-kukuqc39WOHtdxtw4UScxF/WVnMFVSQVKhtx3AjZJzhd0RGZZldcrfSEbVsWWe6KNH253574cq5F+wpv0G9pJw==} + engines: {node: '>=6.14.2'} + requiresBuild: true + dependencies: + node-gyp-build: 4.6.1 + dev: false + /callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} @@ -655,6 +673,10 @@ packages: engines: {node: '>=0.10.0'} dev: true + /eventemitter3@4.0.7: + resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} + dev: false + /fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} dev: true @@ -1108,6 +1130,12 @@ packages: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} dev: true + /node-gyp-build@4.6.1: + resolution: {integrity: sha512-24vnklJmyRS8ViBNI8KbtK/r/DmXQMRiOMXTNz2nrTnAYUwjmEEbnnpB/+kt+yWRv73bPsSPRFddrcIbAxSiMQ==} + hasBin: true + requiresBuild: true + dev: false + /normalize-path@3.0.0: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} @@ -1232,6 +1260,10 @@ packages: picomatch: 2.3.1 dev: true + /regenerator-runtime@0.14.0: + resolution: {integrity: sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA==} + dev: false + /require-directory@2.1.1: resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} engines: {node: '>=0.10.0'} @@ -1254,6 +1286,18 @@ packages: glob: 7.2.3 dev: true + /rpc-websockets@7.6.0: + resolution: {integrity: sha512-Jgcs8q6t8Go98dEulww1x7RysgTkzpCMelVxZW4hvuyFtOGpeUz9prpr2KjUa/usqxgFCd9Tu3+yhHEP9GVmiQ==} + dependencies: + '@babel/runtime': 7.23.1 + eventemitter3: 4.0.7 + uuid: 8.3.2 + ws: 8.14.2(bufferutil@4.0.7)(utf-8-validate@5.0.10) + optionalDependencies: + bufferutil: 4.0.7 + utf-8-validate: 5.0.10 + dev: false + /run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} dependencies: @@ -1409,10 +1453,23 @@ packages: punycode: 2.3.0 dev: true + /utf-8-validate@5.0.10: + resolution: {integrity: sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==} + engines: {node: '>=6.14.2'} + requiresBuild: true + dependencies: + node-gyp-build: 4.6.1 + dev: false + /util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} dev: true + /uuid@8.3.2: + resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + hasBin: true + dev: false + /which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -1447,7 +1504,7 @@ packages: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} dev: true - /ws@8.14.2: + /ws@8.14.2(bufferutil@4.0.7)(utf-8-validate@5.0.10): resolution: {integrity: sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==} engines: {node: '>=10.0.0'} peerDependencies: @@ -1458,6 +1515,9 @@ packages: optional: true utf-8-validate: optional: true + dependencies: + bufferutil: 4.0.7 + utf-8-validate: 5.0.10 dev: false /y18n@5.0.8: