Merge branch 'master' into feature/gql-auth

.github/workflows/_release_rust.yml

@@ -54,6 +54,12 @@ jobs:
         with:
           command: install
           args: cargo-release --force
+      - name: "Publish raphtory-api to crates.io"
+        if: ${{ !inputs.dry_run }}
+        uses: actions-rs/cargo@v1
+        with:
+          command: publish
+          args: --token ${{ secrets.CRATES_TOKEN }} --package raphtory-api --allow-dirty
       - name: "Publish raphtory-arrow to crates.io"
         if: ${{ !inputs.dry_run }}
         uses: actions-rs/cargo@v1

.github/workflows/benchmark.yml

@@ -44,10 +44,6 @@ jobs:
           toolchain: 1.77.0
           override: true
           components: rustfmt, clippy
-      - name: Cargo update
-        uses: actions-rs/cargo@v1
-        with:
-          command: update
       - name: Run benchmark (Unix)
         run: |
           set -o pipefail

.github/workflows/test_python_workflow.yml

@@ -59,8 +59,8 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
           cache: 'pip'
-      - name: Flip raphtory-arrow in Cargo.toml
-        run: python ./scripts/flip_ra.py Cargo.toml
+      - name: Activate raphtory-arrow in Cargo.toml
+        run: make pull-arrow
       - name: Run Maturin develop
         uses: PyO3/maturin-action@v1
         with:

.github/workflows/test_rust_workflow.yml

@@ -52,13 +52,19 @@ jobs:
         name: Cargo cache
         with:
           cache-all-crates: true
-      - name: Flip raphtory-arrow in Cargo.toml
-        run: python ./scripts/flip_ra.py
       - name: Install bininstall
         uses: cargo-bins/cargo-binstall@main
       - name: Install nextest
         run: cargo binstall -y --force cargo-nextest
-      - name: Run Tests (No features)
+      - name: Run all Tests (no arrow)
+        env:
+          RUSTFLAGS: -Awarnings
+          TEMPDIR: ${{ runner.temp }}
+        run: |
+          cargo nextest run --all --no-default-features
+      - name: Activate raphtory-arrow in Cargo.toml
+        run: make pull-arrow
+      - name: Run all Tests (arrow)
         env:
           RUSTFLAGS: -Awarnings
           TEMPDIR: ${{ runner.temp }}

.gitignore

@@ -22,6 +22,7 @@ docs/logs/
 docs/nx.html
 docs/graph.html
 .env
+raphtory-arrow/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "raphtory-arrow-private"]
+	path = raphtory-arrow-private
+	url = [email protected]:Pometry/raphtory-arrow.git

Cargo.toml

@@ -8,8 +8,7 @@ members = [
     "examples/netflow",
     "python",
     "js-raphtory",
-    "raphtory-graphql",
-    "raphtory-auth",
+    "raphtory-graphql", "raphtory-api",
 ]
 default-members = ["raphtory"]
 resolver = "2"
@@ -31,6 +30,10 @@ inherits = "release"
 debug = true
 
 [workspace.dependencies]
+#[public-arrow]
+raphtory-arrow = { version = "0.8.1", path = "raphtory-arrow" }
+#[private-arrow]
+# raphtory-arrow = { path = "raphtory-arrow-private", package = "raphtory-arrow-private" }
 async-graphql = { version = "7.0.5", features = ["dynamic-schema"] }
 async-graphql-poem = "7.0.5"
 dynamic-graphql = { path = "/Users/haaroony/Documents/dev/dynamic-graphql" }
@@ -59,6 +62,7 @@ flate2 = "1.0.28"
 regex = "1.10.3"
 genawaiter = "0.99.1"
 num-traits = "0.2.18"
+num-integer = "0.1"
 rand_distr = "0.4.3"
 rustc-hash = "1.1.0"
 twox-hash = "1.6.3"

Makefile

@@ -35,4 +35,13 @@ rust-test-all:
 	cargo check -p raphtory --no-default-features --features "io"
 	cargo check -p raphtory --no-default-features --features "python"
 	cargo check -p raphtory --no-default-features --features "search"
-	cargo check -p raphtory --no-default-features --features "vectors"
+	cargo check -p raphtory --no-default-features --features "vectors"
+
+activate-arrow:
+	./scripts/activate_private_arrow.py
+
+deactivate-arrow:
+	./scripts/deactivate_private_arrow.py
+
+pull-arrow: activate-arrow
+	git submodule update --init --recursive

examples/rust/src/bin/hulongbay/main.rs

@@ -2,7 +2,13 @@
 #![allow(dead_code)]
 use itertools::Itertools;
 use raphtory::{
-    algorithms::{components::weakly_connected_components, motifs::triangle_count::triangle_count},
+    algorithms::{
+        components::weakly_connected_components,
+        motifs::{
+            global_temporal_three_node_motifs::global_temporal_three_node_motif,
+            triangle_count::triangle_count,
+        },
+    },
     graph_loader::source::csv_loader::CsvLoader,
     prelude::*,
 };
@@ -207,8 +213,16 @@ fn try_main_bm() -> Result<(), Box<dyn Error>> {
     Ok(())
 }
 
+fn try_motif() -> Result<(), Box<dyn Error>> {
+    let args: Vec<String> = env::args().collect();
+    let data_dir = Path::new(args.get(1).ok_or(MissingArgumentError)?);
+    let graph = loader(data_dir)?;
+    global_temporal_three_node_motif(&graph, 3600, None);
+    Ok(())
+}
+
 fn main() {
-    if let Err(e) = try_main_bm() {
+    if let Err(e) = try_motif() {
         eprintln!("Failed: {}", e);
         std::process::exit(1)
     }

python/src/lib.rs

@@ -99,6 +99,7 @@ fn raphtory(py: Python<'_>, m: &PyModule) -> PyResult<()> {
         single_source_shortest_path,
         global_clustering_coefficient,
         temporally_reachable_nodes,
+        temporal_bipartite_graph_projection,
         local_clustering_coefficient,
         weakly_connected_components,
         strongly_connected_components,

python/tests/test_graphdb.py

@@ -2046,15 +2046,97 @@ def test_one_hop_filter_reset():
     assert len(out_out_2) == 0
 
 
-def test_node_types():
+def test_type_filter():
     g = Graph()
     g.add_node(1, 1, node_type="wallet")
     g.add_node(1, 2, node_type="timer")
     g.add_node(1, 3, node_type="timer")
     g.add_node(1, 4, node_type="wallet")
 
-    assert g.nodes.type_filter(["wallet"]).node_type.collect() == ["1", "4"]
-    assert g.subgraph_node_types(["timer"]).nodes.name.collect() == ["2", "3"]
+    assert [node.name for node in g.nodes.type_filter(["wallet"])] == ['1', '4']
+    assert g.subgraph_node_types(["timer"]).nodes.name.collect() == ['2', '3']
+
+    g = PersistentGraph()
+    g.add_node(1, 1, node_type="wallet")
+    g.add_node(2, 2, node_type="timer")
+    g.add_node(3, 3, node_type="timer")
+    g.add_node(4, 4, node_type="wallet")
+
+    assert [node.name for node in g.nodes.type_filter(["wallet"])] == ['1', '4']
+    assert g.subgraph_node_types(["timer"]).nodes.name.collect() == ['2', '3']
+
+    subgraph = g.subgraph([1, 2, 3])
+    assert [node.name for node in subgraph.nodes.type_filter(["wallet"])] == ['1']
+    assert subgraph.subgraph_node_types(["timer"]).nodes.name.collect() == ['2', '3']
+
+    w = g.window(1, 3)
+    assert [node.name for node in w.nodes.type_filter(["wallet"])] == ['1']
+    assert w.subgraph_node_types(["timer"]).nodes.name.collect() == ['2', '3']
+
+    g = Graph()
+    g.add_node(1, 1, node_type="wallet")
+    g.add_node(2, 2, node_type="timer")
+    g.add_node(3, 3, node_type="timer")
+    g.add_node(4, 4, node_type="counter")
+    g.add_edge(1, 1, 2, layer="layer1")
+    g.add_edge(2, 2, 3, layer="layer1")
+    g.add_edge(3, 2, 4, layer="layer2")
+    layer = g.layers(["layer1"])
+    assert [node.name for node in layer.nodes.type_filter(["wallet"])] == ['1']
+    assert layer.subgraph_node_types(["timer"]).nodes.name.collect() == ['2', '3']
+
+    g = Graph()
+    g.add_node(1, 1, node_type="a")
+    g.add_node(1, 2, node_type="b")
+    g.add_node(1, 3, node_type="b")
+    g.add_node(1, 4, node_type="a")
+    g.add_node(1, 5, node_type="c")
+    g.add_node(1, 6, node_type="e")
+    g.add_node(1, 7)
+    g.add_node(1, 8)
+    g.add_node(1, 9)
+    g.add_edge(2, 1, 2, layer="a")
+    g.add_edge(2, 3, 2, layer="a")
+    g.add_edge(2, 2, 4, layer="a")
+    g.add_edge(2, 4, 5, layer="a")
+    g.add_edge(2, 4, 5, layer="a")
+    g.add_edge(2, 5, 6, layer="a")
+    g.add_edge(2, 3, 6, layer="a")
+
+    assert g.nodes.type_filter([""]).name.collect() == ['7', '8', '9']
+
+    assert g.nodes.type_filter(["a"]).name.collect() == ['1', '4']
+    assert g.nodes.type_filter(["a", "c"]).name.collect() == ['1', '4', '5']
+    assert g.nodes.type_filter(["a"]).neighbours.name.collect() == [['2'], ['2', '5']]
+
+    assert g.nodes.degree().collect() == [1, 3, 2, 2, 2, 2, 0, 0, 0]
+    assert g.nodes.type_filter(['a']).degree().collect() == [1, 2]
+    assert g.nodes.type_filter(['d']).degree().collect() == []
+    assert g.nodes.type_filter([]).name.collect() == []
+
+    assert len(g.nodes) == 9
+    assert len(g.nodes.type_filter(['b'])) == 2
+    assert len(g.nodes.type_filter(['d'])) == 0
+
+    assert g.nodes.type_filter(['d']).neighbours.name.collect() == []
+    assert g.nodes.type_filter(['a']).neighbours.name.collect() == [['2'], ['2', '5']]
+    assert g.nodes.type_filter(['a', 'c']).neighbours.name.collect() == [['2'], ['2', '5'], ['4', '6']]
+
+    assert g.nodes.type_filter(['a']).neighbours.type_filter(['c']).name.collect() == [[], ['5']]
+    assert g.nodes.type_filter(['a']).neighbours.type_filter([]).name.collect() == [[], []]
+    assert g.nodes.type_filter(['a']).neighbours.type_filter(['b', 'c']).name.collect() == [['2'], ['2', '5']]
+    assert g.nodes.type_filter(['a']).neighbours.type_filter(['d']).name.collect() == [[], []]
+    assert g.nodes.type_filter(['a']).neighbours.neighbours.name.collect() == [['1', '3', '4'], ['1', '3', '4', '4', '6']]
+    assert g.nodes.type_filter(['a']).neighbours.type_filter(['c']).neighbours.name.collect() == [[], ['4', '6']]
+    assert g.nodes.type_filter(['a']).neighbours.type_filter(['d']).neighbours.name.collect() == [[], []]
+
+    assert g.node('2').neighbours.type_filter(['b']).name.collect() == ['3']
+    assert g.node('2').neighbours.type_filter(['d']).name.collect() == []
+    assert g.node('2').neighbours.type_filter([]).name.collect() == []
+    assert g.node('2').neighbours.type_filter(['c', 'a']).name.collect() == ['1', '4']
+    assert g.node('2').neighbours.type_filter(['c']).neighbours.name.collect() == []
+    assert g.node('2').neighbours.neighbours.name.collect() == ['2', '2', '6', '2', '5']
+
 
 
 def test_time_exploded_edges():
@@ -2171,6 +2253,89 @@ def test_NaN_NaT_as_properties():
     assert g.node(101).properties.temporal.get("floats") == None
 
 
+def test_unique_temporal_properties():
+    g = Graph()
+    g.add_property(1, {"name": "tarzan"})
+    g.add_property(2, {"name": "tarzan2"})
+    g.add_property(3, {"name": "tarzan2"})
+    g.add_property(2, {"salary": "1000"})
+    g.add_constant_properties({"type": "character"})
+    g.add_edge(1,1,2,properties={"status":"open"})
+    g.add_edge(2,1,2,properties={"status":"open"})
+    g.add_edge(3,1,2,properties={"status":"review"})
+    g.add_edge(4,1,2,properties={"status":"open"})
+    g.add_edge(5,1,2,properties={"status":"in-progress"})
+    g.add_edge(10,1,2,properties={"status":"in-progress"})
+    g.add_edge(6,1,2)
+    g.add_node(1, 3, {"name": "avatar1"})
+    g.add_node(2, 3, {"name": "avatar2"})
+    g.add_node(3, 3, {"name": "avatar2"})
+
+    assert g.edge(1,2).properties.temporal.get('status').ordered_dedupe(True) == [(2, "open"), (3, "review"), (4, "open"), (10, "in-progress")]
+    assert g.edge(1,2).properties.temporal.get('status').ordered_dedupe(False) == [(1, "open"), (3, "review"), (4, "open"), (5, "in-progress")]
+    assert g.properties.temporal.get('name').ordered_dedupe(True) == [(1, "tarzan"), (3, "tarzan2")]
+    assert g.properties.temporal.get('name').ordered_dedupe(False) == [(1, "tarzan"), (2, "tarzan2")]
+    assert g.node(3).properties.temporal.get('name').ordered_dedupe(True) == [(1, "avatar1"), (3, "avatar2")]
+    assert g.node(3).properties.temporal.get('name').ordered_dedupe(False) == [(1, "avatar1"), (2, "avatar2")]
+
+    g.add_node(4, 3, {"i64": 1})
+    g.add_node(5, 3, {"i64": 1})
+    g.add_node(6, 3, {"i64": 5})
+    g.add_node(7, 3, {"f64": 1.2})
+    g.add_node(8, 3, {"f64": 1.3})
+    g.add_node(9, 3, {"bool": True})
+    g.add_node(10, 3, {"bool": True})
+    g.add_node(11, 3, {"bool": False})
+    g.add_node(12, 3, {"list": [1, 2, 3]})
+    g.add_node(13, 3, {"list": [1, 2, 3]})
+    g.add_node(14, 3, {"list": [2, 3]})
+    datetime_obj = datetime.strptime("2021-01-01 12:32:00", "%Y-%m-%d %H:%M:%S")
+    datetime_obj2 = datetime.strptime("2021-01-02 12:32:00", "%Y-%m-%d %H:%M:%S")
+    g.add_node(15, 3, {"date": datetime_obj})
+    g.add_node(16, 3, {"date": datetime_obj})
+    g.add_node(17, 3, {"date": datetime_obj2})
+    g.add_node(18, 3, {"map": {"name": "bob", "value list": [1, 2, 3]}})
+    g.add_node(19, 3, {"map": {"name": "bob", "value list": [1, 2]}})
+
+    assert list(g.edge(1,2).properties.temporal.get('status')) == [(1, 'open'), (2, 'open'), (3, 'review'), (4, 'open'), (5, 'in-progress'), (10, 'in-progress')]
+    assert sorted(g.edge(1,2).properties.temporal.get('status').unique()) == ['in-progress', 'open', 'review']
+    assert list(g.properties.temporal.get('name')) == [(1, 'tarzan'), (2, 'tarzan2'), (3, 'tarzan2')]
+    assert sorted(g.properties.temporal.get('name').unique()) == ['tarzan', 'tarzan2']
+    assert list(g.node(3).properties.temporal.get('name')) == [(1, 'avatar1'), (2, 'avatar2'), (3, 'avatar2')]
+    assert sorted(g.node(3).properties.temporal.get('name').unique()) == ['avatar1', 'avatar2']
+    assert sorted(g.node(3).properties.temporal.get('i64').unique()) == [1, 5]
+    assert sorted(g.node(3).properties.temporal.get('f64').unique()) == [1.2, 1.3]
+    assert sorted(g.node(3).properties.temporal.get('bool').unique()) == [False, True]
+    assert sorted(g.node(3).properties.temporal.get('list').unique()) == [[1, 2, 3], [2, 3]]
+    assert sorted(g.node(3).properties.temporal.get('date').unique()) == [datetime_obj, datetime_obj2]
+    actual_list = g.node(3).properties.temporal.get('map').unique()
+    expected_list = [{"name": "bob", "value list": [1, 2]}, {"name": "bob", "value list": [1, 2, 3]}]
+    sorted_actual_list = sorted(actual_list, key=lambda d: (d["name"], tuple(d["value list"])))
+    sorted_expected_list = sorted(expected_list, key=lambda d: (d["name"], tuple(d["value list"])))
+    assert sorted_actual_list == sorted_expected_list
+    g1 = Graph()
+    g1.add_constant_properties({"type": "a"})
+    g1.add_node(1, "ben")
+    g.add_node(7, 3, {"graph": g1})
+    g2 = Graph()
+    g2.add_constant_properties({"type": "b"})
+    g2.add_node(1, "ben")
+    g.add_node(7, 3, {"graph": g2})
+    g3 = Graph()
+    g3.add_constant_properties({"type": "c"})
+    g3.add_node(1, "shivam")
+    g.add_node(7, 3, {"graph": g3})
+
+    actual_list = g.node(3).properties.temporal.get('graph').unique()
+    expected_list = [g1, g3]
+    sorted_actual_list = sorted(actual_list, key=lambda g: g.properties.constant.get('type'))
+    sorted_expected_list = sorted(expected_list, key=lambda g: g.properties.constant.get('type'))
+    assert sorted_actual_list == sorted_expected_list
+
+    assert g.node(3).properties.temporal.get('i64').ordered_dedupe(True) == [(5, 1), (6, 5)]
+    assert g.node(3).properties.temporal.get('i64').ordered_dedupe(False) == [(4, 1), (6, 5)]
+
+
 def test_fuzzy_search():
     g = Graph()
     g.add_node(

raphtory-api/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "raphtory-api"
+version.workspace = true
+documentation.workspace = true
+repository.workspace = true
+license.workspace = true
+readme.workspace = true
+homepage.workspace = true
+keywords.workspace = true
+authors.workspace = true
+rust-version.workspace = true
+edition.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+serde = { workspace = true, features = ["derive"] }
+chrono.workspace = true

raphtory-api/src/core/entities/edges/mod.rs

@@ -0,0 +1 @@
+pub mod edge_ref;