Merge branch 'master' into parquet-loader

# Conflicts: # examples/rust/src/bin/btc/main.rs # examples/rust/src/bin/lotr/main.rs # raphtory/src/io/csv_loader.rs
Pometry · Jun 26, 2024 · 4c7695f · 4c7695f
2 parents 07d12f3 + d0a25c3
commit 4c7695f
Show file tree

Hide file tree

Showing 32 changed files with 103 additions and 169 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -167,7 +167,7 @@ traitlets==5.14.3
     #   nbsphinx
 typing-extensions==4.12.2
     # via pydata-sphinx-theme
-urllib3==2.2.1
+urllib3==2.2.2
     # via requests
 webencodings==0.5.1
     # via

diff --git a/examples/rust/src/bin/btc/main.rs b/examples/rust/src/bin/btc/main.rs
@@ -2,7 +2,7 @@
 #![allow(dead_code)]
 
 use chrono::{DateTime, Utc};
-use raphtory::{core::utils::hashing, io::csv_loader::CsvLoader, prelude::*};
+use raphtory::{io::csv_loader::CsvLoader, prelude::*};
 use regex::Regex;
 use serde::Deserialize;
 use std::{
@@ -54,7 +54,7 @@ fn main() {
         panic!("Missing data dir = {}", data_dir.to_str().unwrap())
     }
 
-    let test_v = hashing::calculate_hash(&"139eeGkMGR6F9EuJQ3qYoXebfkBbNAsLtV:btc");
+    let test_v = "139eeGkMGR6F9EuJQ3qYoXebfkBbNAsLtV:btc".id();
 
     // If data_dir/graphdb.bincode exists, use bincode to load the graph from binary encoded data files
     // otherwise load the graph from csv data files
@@ -82,8 +82,8 @@ fn main() {
         CsvLoader::new(data_dir)
             .with_filter(Regex::new(r".+(sent|received)").unwrap())
             .load_into_graph(&g, |sent: Sent, g: &Graph| {
-                let src = hashing::calculate_hash(&sent.addr);
-                let dst = hashing::calculate_hash(&sent.txn);
+                let src = sent.addr.id();
+                let dst = sent.txn.id();
                 let time = sent.time.timestamp();
 
                 if src == test_v || dst == test_v {

diff --git a/examples/rust/src/bin/lotr/main.rs b/examples/rust/src/bin/lotr/main.rs
@@ -1,5 +1,5 @@
 use raphtory::{
-    algorithms::pathing::temporal_reachability::temporally_reachable_nodes, core::utils::hashing,
+    algorithms::pathing::temporal_reachability::temporally_reachable_nodes,
     io::csv_loader::CsvLoader, prelude::*,
 };
 use serde::Deserialize;
@@ -99,7 +99,7 @@ fn main() {
     assert_eq!(graph.count_nodes(), 139);
     assert_eq!(graph.count_edges(), 701);
 
-    let gandalf = hashing::calculate_hash(&"Gandalf");
+    let gandalf = "Gandalf".id();
 
     assert_eq!(gandalf, 2760374808085341115);
     assert!(graph.has_node(gandalf));

diff --git a/pometry-storage-private b/pometry-storage-private
diff --git a/raphtory-api/Cargo.toml b/raphtory-api/Cargo.toml
@@ -26,6 +26,10 @@ rayon = { workspace = true }
 rand = { workspace = true }
 quickcheck = { workspace = true }
 quickcheck_macros = { workspace = true }
+twox-hash.workspace = true
+
+[dev-dependencies]
+proptest.workspace = true
 
 [features]
 default = []

diff --git a/...ory/src/core/entities/nodes/input_node.rs → raphtory-api/src/core/input/input_node.rs b/...ory/src/core/entities/nodes/input_node.rs → raphtory-api/src/core/input/input_node.rs
@@ -5,6 +5,7 @@
 //! `u64`, `&str`, and `String`.
 
 use crate::core::utils::hashing;
+
 const MAX_U64_BYTES: [u8; 20] = [
     49, 56, 52, 52, 54, 55, 52, 52, 48, 55, 51, 55, 48, 57, 53, 53, 49, 54, 49, 53,
 ];
@@ -85,7 +86,7 @@ impl InputNode for String {
 
 #[cfg(test)]
 mod test {
-    use crate::core::entities::nodes::input_node::{parse_u64_strict, InputNode};
+    use crate::core::input::input_node::{parse_u64_strict, InputNode};
     use proptest::prelude::*;
 
     #[test]
@@ -109,6 +110,8 @@ mod test {
             let res = parse_u64_strict(&s);
             if let Some(n) = res {
                 assert_eq!(n.to_string(), s)
+            } else {
+                assert_ne!(s.id().to_string(), s)
             }
         });
     }

diff --git a/raphtory-api/src/core/input/mod.rs b/raphtory-api/src/core/input/mod.rs
@@ -0,0 +1 @@
+pub mod input_node;
diff --git a/raphtory-api/src/core/mod.rs b/raphtory-api/src/core/mod.rs
@@ -1,5 +1,7 @@
 pub mod entities;
+pub mod input;
 pub mod storage;
+pub mod utils;
 
 /// Denotes the direction of an edge. Can be incoming, outgoing or both.
 #[derive(

diff --git a/raphtory-api/src/core/storage/dict_mapper.rs b/raphtory-api/src/core/storage/dict_mapper.rs
@@ -67,14 +67,11 @@ impl DictMapper {
 
 #[cfg(test)]
 mod test {
-    use std::{collections::HashMap, sync::Arc, thread};
-
     use crate::core::storage::dict_mapper::DictMapper;
     use quickcheck_macros::quickcheck;
     use rand::seq::SliceRandom;
     use rayon::prelude::*;
-
-    use super::*;
+    use std::collections::HashMap;
 
     #[test]
     fn test_dict_mapper() {

diff --git a/raphtory-api/src/core/utils/hashing.rs b/raphtory-api/src/core/utils/hashing.rs
@@ -0,0 +1,10 @@
+//! Utility functions used throughout the modules.
+
+use std::hash::{Hash, Hasher};
+use twox_hash::XxHash64;
+
+pub fn calculate_hash<T: Hash + ?Sized>(t: &T) -> u64 {
+    let mut s = XxHash64::default();
+    t.hash(&mut s);
+    s.finish()
+}
diff --git a/raphtory-api/src/core/utils/mod.rs b/raphtory-api/src/core/utils/mod.rs
@@ -0,0 +1 @@
+pub mod hashing;
diff --git a/raphtory-graphql/src/lib.rs b/raphtory-graphql/src/lib.rs
@@ -49,10 +49,7 @@ mod graphql_test {
         prelude::*,
     };
     use serde_json::json;
-    use std::{
-        collections::{HashMap, HashSet},
-        path::Path,
-    };
+    use std::collections::{HashMap, HashSet};
     use tempfile::{tempdir, TempDir};
 
     #[tokio::test]

diff --git a/raphtory/src/algorithms/pathing/temporal_reachability.rs b/raphtory/src/algorithms/pathing/temporal_reachability.rs
@@ -1,11 +1,8 @@
 use crate::{
     algorithms::algorithm_result::AlgorithmResult,
-    core::{
-        entities::nodes::input_node::InputNode,
-        state::{
-            accumulator_id::accumulators::{hash_set, min, or},
-            compute_state::ComputeStateVec,
-        },
+    core::state::{
+        accumulator_id::accumulators::{hash_set, min, or},
+        compute_state::ComputeStateVec,
     },
     db::{
         api::view::StaticGraphViewOps,
@@ -20,6 +17,7 @@ use crate::{
 };
 use itertools::Itertools;
 use num_traits::Zero;
+use raphtory_api::core::input::input_node::InputNode;
 use std::{collections::HashMap, ops::Add};
 
 #[derive(Eq, Hash, PartialEq, Clone, Debug, Default)]

diff --git a/raphtory/src/core/entities/graph/tgraph.rs b/raphtory/src/core/entities/graph/tgraph.rs
@@ -6,7 +6,7 @@ use crate::{
                 tgraph_storage::GraphStorage,
                 timer::{MaxCounter, MinCounter, TimeCounterTrait},
             },
-            nodes::{input_node::InputNode, node_ref::NodeRef, node_store::NodeStore},
+            nodes::{node_ref::NodeRef, node_store::NodeStore},
             properties::{graph_meta::GraphMeta, props::Meta, tprop::TProp},
             LayerIds, EID, VID,
         },
@@ -24,8 +24,11 @@ use crate::{
     },
     prelude::DeletionOps,
 };
-use dashmap::{DashMap, DashSet};
-use raphtory_api::core::storage::{arc_str::ArcStr, locked_vec::ArcReadLockedVec, FxDashMap};
+use dashmap::DashSet;
+use raphtory_api::core::{
+    input::input_node::InputNode,
+    storage::{arc_str::ArcStr, locked_vec::ArcReadLockedVec, FxDashMap},
+};
 use rustc_hash::FxHasher;
 use serde::{Deserialize, Serialize};
 use std::{

diff --git a/raphtory/src/core/entities/nodes/mod.rs b/raphtory/src/core/entities/nodes/mod.rs
@@ -1,4 +1,3 @@
-pub mod input_node;
 pub mod node_ref;
 pub mod node_store;
 pub mod structure;
diff --git a/raphtory/src/core/entities/properties/props.rs b/raphtory/src/core/entities/properties/props.rs
@@ -10,13 +10,12 @@ use crate::{
     },
     db::api::storage::tprop_storage_ops::TPropOps,
 };
-use lock_api;
 use parking_lot::RwLock;
 use raphtory_api::core::storage::{
     arc_str::ArcStr, dict_mapper::DictMapper, locked_vec::ArcReadLockedVec,
 };
 use serde::{Deserialize, Serialize};
-use std::{borrow::Borrow, fmt::Debug, hash::Hash, ops::Deref, sync::Arc};
+use std::{fmt::Debug, hash::Hash, ops::Deref, sync::Arc};
 
 #[derive(Serialize, Deserialize, Default, Debug, PartialEq)]
 pub struct Props {
@@ -319,7 +318,6 @@ impl PropMapper {
 #[cfg(test)]
 mod test {
     use super::*;
-    use rayon::prelude::*;
     use std::{sync::Arc, thread};
 
     #[test]

diff --git a/raphtory/src/core/mod.rs b/raphtory/src/core/mod.rs
@@ -29,16 +29,15 @@ use crate::{
     prelude::GraphViewOps,
 };
 use chrono::{DateTime, NaiveDateTime, Utc};
+use raphtory_api::core::storage::arc_str::ArcStr;
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 use std::{
-    borrow::Borrow,
     cmp::Ordering,
     collections::HashMap,
     fmt,
     fmt::{Display, Formatter},
     hash::{Hash, Hasher},
-    ops::Deref,
     sync::Arc,
 };
 
@@ -53,9 +52,6 @@ pub mod state;
 pub mod storage;
 pub mod utils;
 
-// this is here because Arc<str> annoyingly doesn't implement all the expected comparisons
-
-use raphtory_api::core::storage::arc_str::ArcStr;
 pub use raphtory_api::core::*;
 
 #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Hash)]

diff --git a/raphtory/src/core/utils/hashing.rs b/raphtory/src/core/utils/hashing.rs
diff --git a/raphtory/src/core/utils/mod.rs b/raphtory/src/core/utils/mod.rs
@@ -1,3 +1,2 @@
 pub mod errors;
-pub mod hashing;
 pub mod time;
diff --git a/raphtory/src/db/api/mutation/addition_ops.rs b/raphtory/src/db/api/mutation/addition_ops.rs
@@ -1,6 +1,6 @@
 use crate::{
     core::{
-        entities::{edges::edge_ref::EdgeRef, nodes::input_node::InputNode},
+        entities::edges::edge_ref::EdgeRef,
         utils::{errors::GraphError, time::IntoTimeWithFormat},
         Prop,
     },
@@ -12,6 +12,7 @@ use crate::{
         graph::{edge::EdgeView, node::NodeView},
     },
 };
+use raphtory_api::core::input::input_node::InputNode;
 
 use super::time_from_input;
 

diff --git a/raphtory/src/db/api/mutation/deletion_ops.rs b/raphtory/src/db/api/mutation/deletion_ops.rs
@@ -1,13 +1,11 @@
 use crate::{
-    core::{
-        entities::nodes::input_node::InputNode,
-        utils::{errors::GraphError, time::IntoTimeWithFormat},
-    },
+    core::utils::{errors::GraphError, time::IntoTimeWithFormat},
     db::api::mutation::{
         internal::{InternalAdditionOps, InternalDeletionOps},
         TryIntoInputTime,
     },
 };
+use raphtory_api::core::input::input_node::InputNode;
 
 use super::time_from_input;