Fix/performance

.github/workflows/benchmark.yml

@@ -44,10 +44,6 @@ jobs:
           toolchain: 1.77.0
           override: true
           components: rustfmt, clippy
-      - name: Cargo update
-        uses: actions-rs/cargo@v1
-        with:
-          command: update
       - name: Run benchmark (Unix)
         run: |
           set -o pipefail

.github/workflows/test_python_workflow.yml

@@ -59,8 +59,8 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
           cache: 'pip'
-      - name: Flip raphtory-arrow in Cargo.toml
-        run: python ./scripts/flip_ra.py Cargo.toml
+      - name: Activate raphtory-arrow in Cargo.toml
+        run: make pull-arrow
       - name: Run Maturin develop
         uses: PyO3/maturin-action@v1
         with:

.github/workflows/test_rust_workflow.yml

@@ -52,8 +52,8 @@ jobs:
         name: Cargo cache
         with:
           cache-all-crates: true
-      - name: Flip raphtory-arrow in Cargo.toml
-        run: python ./scripts/flip_ra.py
+      - name: Activate raphtory-arrow in Cargo.toml
+        run: make pull-arrow
       - name: Install bininstall
         uses: cargo-bins/cargo-binstall@main
       - name: Install nextest

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "raphtory-arrow-private"]
+	path = raphtory-arrow-private
+	url = [email protected]:Pometry/raphtory-arrow.git

Cargo.toml

@@ -30,6 +30,10 @@ inherits = "release"
 debug = true
 
 [workspace.dependencies]
+#[public-arrow]
+raphtory-arrow = { version = "0.8.1", path = "raphtory-arrow" }
+#[private-arrow]
+# raphtory-arrow = { path = "raphtory-arrow-private", package = "raphtory-arrow-private" }
 async-graphql = { version = "6.0.11", features = ["dynamic-schema"] }
 async-graphql-poem = "6.0.11"
 dynamic-graphql = "0.8.1"

Makefile

@@ -35,4 +35,13 @@ rust-test-all:
 	cargo check -p raphtory --no-default-features --features "io"
 	cargo check -p raphtory --no-default-features --features "python"
 	cargo check -p raphtory --no-default-features --features "search"
-	cargo check -p raphtory --no-default-features --features "vectors"
+	cargo check -p raphtory --no-default-features --features "vectors"
+
+activate-arrow:
+	./scripts/activate_private_arrow.py
+
+deactivate-arrow:
+	./scripts/deactivate_private_arrow.py
+
+pull-arrow: activate-arrow
+	git submodule update --init --recursive

examples/rust/src/bin/hulongbay/main.rs

@@ -2,7 +2,13 @@
 #![allow(dead_code)]
 use itertools::Itertools;
 use raphtory::{
-    algorithms::{components::weakly_connected_components, motifs::triangle_count::triangle_count},
+    algorithms::{
+        components::weakly_connected_components,
+        motifs::{
+            global_temporal_three_node_motifs::global_temporal_three_node_motif,
+            triangle_count::triangle_count,
+        },
+    },
     graph_loader::source::csv_loader::CsvLoader,
     prelude::*,
 };
@@ -207,8 +213,16 @@ fn try_main_bm() -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
+fn try_motif() -> Result<(), Box<dyn Error>> {
+    let args: Vec<String> = env::args().collect();
+    let data_dir = Path::new(args.get(1).ok_or(MissingArgumentError)?);
+    let graph = loader(data_dir)?;
+    global_temporal_three_node_motif(&graph, 3600, None);
+    Ok(())
+}
+
 fn main() {
-    if let Err(e) = try_main_bm() {
+    if let Err(e) = try_motif() {
         eprintln!("Failed: {}", e);
         std::process::exit(1)
     }

raphtory-benchmark/Cargo.toml

@@ -9,7 +9,7 @@ edition = "2021"
 criterion = { workspace = true }
 raphtory = { path = "../raphtory", features = ["io"] }
 raphtory-graphql = { path = "../raphtory-graphql", version = "0.8.1" }
-raphtory-arrow = { path = "../raphtory-arrow", version = "0.8.1" }
+raphtory-arrow.workspace = true
 sorted_vector_map = { workspace = true }
 rand = { workspace = true }
 rayon = { workspace = true }

raphtory-benchmark/benches/algobench.rs

@@ -8,30 +8,17 @@ use raphtory::{
             clustering_coefficient::clustering_coefficient,
             local_clustering_coefficient::local_clustering_coefficient,
         },
-        motifs::local_triangle_count::local_triangle_count,
+        motifs::{
+            global_temporal_three_node_motifs::global_temporal_three_node_motif,
+            local_triangle_count::local_triangle_count,
+        },
     },
     graphgen::random_attachment::random_attachment,
     prelude::*,
 };
 use rayon::prelude::*;
 
 mod common;
-
-//TODO swap to new trianglecount
-// pub fn global_triangle_count_analysis(c: &mut Criterion) {
-//     let mut group = c.benchmark_group("global_triangle_count");
-//     group.sample_size(10);
-//     bench(&mut group, "global_triangle_count", None, |b| {
-//         let g = raphtory_db::graph_loader::lotr_graph::lotr_graph(1);
-//         let windowed_graph = g.window(i64::MIN, i64::MAX);
-//         b.iter(|| {
-//             global_triangle_count(&windowed_graph).unwrap();
-//         });
-//     });
-//
-//     group.finish();
-// }
-
 pub fn local_triangle_count_analysis(c: &mut Criterion) {
     let mut group = c.benchmark_group("local_triangle_count");
     group.sample_size(10);
@@ -132,12 +119,25 @@ pub fn graphgen_large_concomp(c: &mut Criterion) {
     group.finish()
 }
 
+pub fn temporal_motifs(c: &mut Criterion) {
+    let mut group = c.benchmark_group("temporal_motifs");
+
+    bench(&mut group, "temporal_motifs", None, |b| {
+        let g: Graph = raphtory::graph_loader::example::lotr_graph::lotr_graph();
+
+        b.iter(|| global_temporal_three_node_motif(&g, 100, None))
+    });
+
+    group.finish();
+}
+
 criterion_group!(
     benches,
     local_triangle_count_analysis,
     local_clustering_coefficient_analysis,
     graphgen_large_clustering_coeff,
     graphgen_large_pagerank,
     graphgen_large_concomp,
+    temporal_motifs,
 );
 criterion_main!(benches);

raphtory-cypher/Cargo.toml

@@ -15,7 +15,7 @@ edition.workspace = true
 
 [dependencies]
 raphtory = { path = "../raphtory" }
-raphtory-arrow = { path = "../raphtory-arrow", optional = true, version = "0.8.1"}
+raphtory-arrow = { workspace = true, optional = true }
 arrow.workspace = true
 arrow-buffer.workspace = true
 arrow-schema.workspace = true

raphtory-graphql/src/model/graph/edge.rs

@@ -1,5 +1,4 @@
 use crate::model::graph::{node::Node, property::GqlProperties};
-use async_graphql::Error;
 use dynamic_graphql::{ResolvedObject, ResolvedObjectFields};
 use itertools::Itertools;
 use raphtory::{

raphtory/Cargo.toml

@@ -15,7 +15,7 @@ homepage.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-raphtory-api = { path = "../raphtory-api", version="0.8.1" }
+raphtory-api = { path = "../raphtory-api", version = "0.8.1" }
 bincode = { workspace = true }
 chrono = { workspace = true }
 itertools = { workspace = true }
@@ -74,7 +74,7 @@ tempfile = { workspace = true, optional = true }
 bytemuck = { workspace = true, optional = true }
 rpds = { workspace = true, optional = true }
 thread_local = { workspace = true, optional = true }
-raphtory-arrow = { path = "../raphtory-arrow", version="0.8.1", optional = true }
+raphtory-arrow = { workspace = true, optional = true }
 
 [dev-dependencies]
 csv = { workspace = true }

raphtory/src/algorithms/motifs/global_temporal_three_node_motifs.rs

@@ -7,7 +7,7 @@ use crate::{
         compute_state::ComputeStateVec,
     },
     db::{
-        api::view::{GraphViewOps, NodeViewOps, *},
+        api::view::*,
         graph::views::node_subgraph::NodeSubgraph,
         task::{
             context::Context,
@@ -23,15 +23,11 @@ use std::collections::HashMap;
 
 ///////////////////////////////////////////////////////
 
-pub fn star_motif_count<G>(
-    graph: &G,
-    evv: &EvalNodeView<G, ()>,
-    deltas: Vec<i64>,
-) -> Vec<[usize; 32]>
+pub fn star_motif_count<G>(evv: &EvalNodeView<G, ()>, deltas: Vec<i64>) -> Vec<[usize; 32]>
 where
     G: StaticGraphViewOps,
 {
-    let two_n_c = twonode_motif_count(graph, evv, deltas.clone());
+    let two_n_c = twonode_motif_count(evv, deltas.clone());
     let neigh_map: HashMap<u64, usize> = evv
         .neighbours()
         .into_iter()
@@ -75,20 +71,16 @@ where
 
 ///////////////////////////////////////////////////////
 
-pub fn twonode_motif_count<G>(
-    graph: &G,
-    evv: &EvalNodeView<G, ()>,
-    deltas: Vec<i64>,
-) -> Vec<[usize; 8]>
+pub fn twonode_motif_count<G>(evv: &EvalNodeView<G, ()>, deltas: Vec<i64>) -> Vec<[usize; 8]>
 where
     G: StaticGraphViewOps,
 {
     let mut results = deltas.iter().map(|_| [0; 8]).collect::<Vec<[usize; 8]>>();
 
     for nb in evv.neighbours().into_iter() {
         let nb_id = nb.id();
-        let out = graph.edge(evv.id(), nb_id);
-        let inc = graph.edge(nb_id, evv.id());
+        let out = evv.graph().edge(evv.id(), nb_id);
+        let inc = evv.graph().edge(nb_id, evv.id());
         let events: Vec<TwoNodeEvent> = out
             .iter()
             .flat_map(|e| e.explode())
@@ -181,7 +173,8 @@ where
                         .sorted()
                         .permutations(2)
                         .flat_map(|e| {
-                            g.edge(*e.first().unwrap(), *e.get(1).unwrap())
+                            u.graph()
+                                .edge(*e.first().unwrap(), *e.get(1).unwrap())
                                 .iter()
                                 .flat_map(|edge| edge.explode())
                                 .collect::<Vec<_>>()
@@ -265,8 +258,7 @@ where
     let out1 = triangle_motifs(g, deltas.clone(), threads);
 
     let step1 = ATask::new(move |evv: &mut EvalNodeView<G, _>| {
-        let g = evv.graph();
-        let star_nodes = star_motif_count(g, evv, deltas.clone());
+        let star_nodes = star_motif_count(evv, deltas.clone());
         for (i, star) in star_nodes.iter().enumerate() {
             evv.global_update(&star_mc[i], *star);
         }

raphtory/src/algorithms/motifs/local_temporal_three_node_motifs.rs

@@ -126,7 +126,6 @@ where
 ///////////////////////////////////////////////////////
 
 pub fn twonode_motif_count<'a, 'b, G, GH>(
-    graph: &'a G,
     evv: &'a EvalNodeView<'b, '_, G, MotifCounter, GH>,
     deltas: Vec<i64>,
 ) -> Vec<[usize; 8]>
@@ -144,8 +143,8 @@ where
 
     for nb in evv.neighbours().into_iter() {
         let nb_id = nb.id();
-        let out = graph.edge(evv.id(), nb_id);
-        let inc = graph.edge(nb_id, evv.id());
+        let out = evv.graph().edge(evv.id(), nb_id);
+        let inc = evv.graph().edge(nb_id, evv.id());
         let events: Vec<TwoNodeEvent> = out
             .iter()
             .flat_map(|e| e.explode())
@@ -337,8 +336,7 @@ where
     let out1 = triangle_motifs(g, deltas.clone(), motifs_counter, threads);
 
     let step1 = ATask::new(move |evv: &mut EvalNodeView<G, MotifCounter>| {
-        let g = evv.graph();
-        let two_nodes = twonode_motif_count(g, evv, deltas.clone());
+        let two_nodes = twonode_motif_count(evv, deltas.clone());
         let star_nodes = star_motif_count(evv, deltas.clone());
 
         *evv.get_mut() = MotifCounter::new(

raphtory/src/arrow/graph_impl/core_ops.rs

@@ -33,7 +33,7 @@ use crate::{
 };
 use itertools::Itertools;
 use polars_arrow::datatypes::ArrowDataType;
-use raphtory_arrow::{properties::Properties, GID};
+use raphtory_arrow::{properties::Properties, GidRef, GID};
 use rayon::prelude::*;
 impl CoreGraphOps for ArrowGraph {
     fn unfiltered_num_nodes(&self) -> usize {
@@ -97,17 +97,17 @@ impl CoreGraphOps for ArrowGraph {
 
     fn node_id(&self, v: VID) -> u64 {
         match self.inner.node_gid(v).unwrap() {
-            GID::U64(n) => n,
-            GID::I64(n) => n as u64,
-            GID::Str(s) => s.id(),
+            GidRef::U64(n) => n,
+            GidRef::I64(n) => n as u64,
+            GidRef::Str(s) => s.id(),
         }
     }
 
     fn node_name(&self, v: VID) -> String {
         match self.inner.node_gid(v).unwrap() {
-            GID::U64(n) => n.to_string(),
-            GID::I64(n) => n.to_string(),
-            GID::Str(s) => s,
+            GidRef::U64(n) => n.to_string(),
+            GidRef::I64(n) => n.to_string(),
+            GidRef::Str(s) => s.to_owned(),
         }
     }
 
@@ -200,15 +200,15 @@ impl CoreGraphOps for ArrowGraph {
     }
 
     fn core_nodes(&self) -> NodesStorage {
-        NodesStorage::Arrow(ArrowNodesOwned::new(&self.inner))
+        NodesStorage::Arrow(ArrowNodesOwned::new(self.inner.clone()))
     }
 
     fn core_node_entry(&self, vid: VID) -> NodeStorageEntry {
         NodeStorageEntry::Arrow(ArrowNode::new(&self.inner, vid))
     }
 
     fn core_node_arc(&self, vid: VID) -> NodeOwnedEntry {
-        NodeOwnedEntry::Arrow(ArrowOwnedNode::new(&self.inner, vid))
+        NodeOwnedEntry::Arrow(ArrowOwnedNode::new(self.inner.clone(), vid))
     }
 
     fn core_edge_arc(&self, eid: ELID) -> EdgeOwnedEntry {