Refactor pre release (#1631)

* rename arrow to disk and raphtory-arrow to pometry-storage

* transition to pometry-storage naming

* rust fmt

* rename more things to pometry-storage

* get rid of the warning on Timestamps

* Renaming

* refactor bits

* rename storage types to Disk*

* Final changes

* deactivate

* deactivate 2

* use DiskGraph in python

* More refactoring

* fmt

* fixed warnings

* Fixed python tests

* typos

---------

Co-authored-by: miratepuffin <[email protected]>

.github/workflows/_release_rust.yml

@@ -60,12 +60,12 @@ jobs:
         with:
           command: publish
           args: --token ${{ secrets.CRATES_TOKEN }} --package raphtory-api --allow-dirty
-      - name: "Publish raphtory-arrow to crates.io"
+      - name: "Publish pometry-storage to crates.io"
         if: ${{ !inputs.dry_run }}
         uses: actions-rs/cargo@v1
         with:
           command: publish
-          args: --token ${{ secrets.CRATES_TOKEN }} --package raphtory-arrow --allow-dirty
+          args: --token ${{ secrets.CRATES_TOKEN }} --package pometry-storage --allow-dirty
       - name: "Publish raphtory to crates.io"
         if: ${{ !inputs.dry_run }}
         uses: actions-rs/cargo@v1

.github/workflows/benchmark.yml

@@ -34,7 +34,7 @@ jobs:
         with:
           cache-all-crates: true
       - uses: webfactory/[email protected]
-        name: Load raphtory-arrow key
+        name: Load raphtory-disk_graph key
         with:
           ssh-private-key: ${{ secrets.RA_SSH_PRIVATE_KEY }}
       - uses: actions-rs/toolchain@v1

.github/workflows/test_python_workflow.yml

@@ -47,7 +47,7 @@ jobs:
           override: true
           components: rustfmt, clippy
       - uses: webfactory/[email protected]
-        name: Load raphtory-arrow key
+        name: Load raphtory-disk_graph key
         with:
           ssh-private-key: ${{ secrets.RA_SSH_PRIVATE_KEY }}
       - uses: Swatinem/rust-cache@v2
@@ -59,15 +59,15 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
           cache: 'pip'
-      - name: Activate raphtory-arrow in Cargo.toml
-        run: make pull-arrow
+      - name: Activate pometry-storage in Cargo.toml
+        run: make pull-storage
       - name: Run Maturin develop
         uses: PyO3/maturin-action@v1
         with:
           working-directory: ./python
           command: build
           target: ${{ matrix.target }}
-          args: --release --features arrow
+          args: --release --features storage
       - name: Install Python dependencies (Unix)
         if: "contains(matrix.os, 'Ubuntu') || contains(matrix.os, 'macOS')"
         run: |

.github/workflows/test_rust_workflow.yml

@@ -43,7 +43,7 @@ jobs:
           sudo rm -rf /opt/ghc
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - uses: webfactory/[email protected]
-        name: Load raphtory-arrow key
+        name: Load pometry-storage key
         with:
           ssh-private-key: ${{ secrets.RA_SSH_PRIVATE_KEY }}
       - name: Rust version
@@ -56,20 +56,20 @@ jobs:
         uses: cargo-bins/cargo-binstall@main
       - name: Install nextest
         run: cargo binstall -y --force cargo-nextest
-      - name: Run all Tests (no arrow)
+      - name: Run all Tests (no disk_graph)
         env:
           RUSTFLAGS: -Awarnings
           TEMPDIR: ${{ runner.temp }}
         run: |
           cargo nextest run --all --no-default-features
-      - name: Activate raphtory-arrow in Cargo.toml
-        run: make pull-arrow
-      - name: Run all Tests (arrow)
+      - name: Activate pometry-storage in Cargo.toml
+        run: make pull-storage
+      - name: Run all Tests (disk_graph)
         env:
           RUSTFLAGS: -Awarnings
           TEMPDIR: ${{ runner.temp }}
         run: |
-          cargo nextest run --all --no-default-features --features "arrow"
+          cargo nextest run --all --no-default-features --features "storage"
       - name: Run Tests (features=io)
         env:
           RUSTFLAGS: -Awarnings
@@ -90,11 +90,11 @@ jobs:
           RUSTFLAGS: -Awarnings
         run: |
           cargo check -p raphtory --no-default-features --features "vectors"
-      - name: Run Tests (features=arrow)
+      - name: Run Tests (features=storage)
         env:
           RUSTFLAGS: -Awarnings
         run: |
-          cargo check -p raphtory --no-default-features --features "arrow"
+          cargo check -p raphtory --no-default-features --features "storage"
   doc-test:
     if: ${{ !inputs.skip_tests }}
     name: "Doc tests"
@@ -118,7 +118,7 @@ jobs:
         with:
           cache-all-crates: true
       - uses: webfactory/[email protected]
-        name: Load raphtory-arrow key
+        name: Load raphtory-disk_graph key
         with:
           ssh-private-key: ${{ secrets.RA_SSH_PRIVATE_KEY }}
       - name: Set up Python

.gitignore

@@ -22,7 +22,7 @@ docs/logs/
 docs/nx.html
 docs/graph.html
 .env
-raphtory-arrow/
+pometry-arrow/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "raphtory-arrow-private"]
-	path = raphtory-arrow-private
-	url = [email protected]:Pometry/raphtory-arrow.git
+[submodule "pometry-storage-private"]
+	path = pometry-storage-private
+	url = [email protected]:Pometry/pometry-storage.git

Cargo.toml

@@ -3,7 +3,7 @@ members = [
     "raphtory",
     "raphtory-cypher",
     "raphtory-benchmark",
-    "raphtory-arrow",
+    "pometry-storage",
     "examples/rust",
     "examples/netflow",
     "python",
@@ -30,10 +30,10 @@ inherits = "release"
 debug = true
 
 [workspace.dependencies]
-#[public-arrow]
-raphtory-arrow = { version = "0.8.1", path = "raphtory-arrow" }
-#[private-arrow]
-# raphtory-arrow = { path = "raphtory-arrow-private", package = "raphtory-arrow-private" }
+#[public-storage]
+pometry-storage = { version = "0.8.1", path = "pometry-storage" }
+#[private-storage]
+# pometry-storage = { path = "pometry-storage-private", package = "pometry-storage-private" }
 async-graphql = { version = "6.0.11", features = ["dynamic-schema"] }
 async-graphql-poem = "6.0.11"
 dynamic-graphql = "0.8.1"
@@ -135,7 +135,7 @@ arrow-schema = { version = "50" }
 arrow-data = { version = "50" }
 arrow-array = { version = "50" }
 
-# Make sure that transitive dependencies stick to arrow 50
+# Make sure that transitive dependencies stick to disk_graph 50
 [patch.crates-io]
 arrow = { git = "https://github.com/apache/arrow-rs.git", tag = "50.0.0" }
 arrow-buffer = { git = "https://github.com/apache/arrow-rs.git", tag = "50.0.0" }

Makefile

@@ -37,11 +37,11 @@ rust-test-all:
 	cargo check -p raphtory --no-default-features --features "search"
 	cargo check -p raphtory --no-default-features --features "vectors"
 
-activate-arrow:
-	./scripts/activate_private_arrow.py
+activate-storage:
+	./scripts/activate_private_storage.py
 
-deactivate-arrow:
-	./scripts/deactivate_private_arrow.py
+deactivate-storage:
+	./scripts/deactivate_private_storage.py
 
-pull-arrow: activate-arrow
+pull-storage: activate-storage
 	git submodule update --init --recursive

docs/source/conf.py

@@ -15,9 +15,9 @@
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 
-project = 'Raphtory'
-copyright = '2023, Pometry'
-author = 'Pometry'
+project = "Raphtory"
+copyright = "2023, Pometry"
+author = "Pometry"
 release = __version__
 git_ref = "master"
 
@@ -62,8 +62,8 @@
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 
-html_theme = 'pydata_sphinx_theme'
-html_static_path = ['_static']
+html_theme = "pydata_sphinx_theme"
+html_static_path = ["_static"]
 html_css_files = [
     "css/custom.css",
 ]
@@ -74,12 +74,10 @@
 static_assets_root = "https://raw.githubusercontent.com/Pometry/Raphtory/master"
 html_logo = "_static/logos/raphtory-logo-bright-medium.png"
 
-html_context = {
-    "default_mode": "auto"
-}
+html_context = {"default_mode": "auto"}
 
 html_theme_options = {
-    'nosidebar': True,
+    "nosidebar": True,
     "search_bar_text": "Search here...",
     "external_links": [
         {
@@ -118,7 +116,7 @@
     "show_version_warning_banner": True,
     "navbar_end": ["theme-switcher", "navbar-icon-links"],
     "check_switcher": False,
-    "show_toc_level": 3
+    "show_toc_level": 3,
 }
 
 # sphinx-favicon - Add support for custom favicons
@@ -139,4 +137,4 @@
 # sphinx view code
 viewcode_line_numbers = True
 
-autodoc_typehints = 'both'
+autodoc_typehints = "both"

pometry-storage/Cargo.toml

@@ -0,0 +1,3 @@
+[package]
+name = "pometry-storage"
+version = "0.8.1"

python/Cargo.toml

@@ -32,7 +32,7 @@ itertools = { workspace = true }
 
 [features]
 default = ["extension-module"]
-arrow = ["raphtory_core/arrow"]
+storage = ["raphtory_core/storage"]
 extension-module = ["pyo3/extension-module"]
 
 

python/python/raphtory/__init__.py

@@ -8,7 +8,7 @@
 sys.modules["raphtory.graphql"] = graphql
 
 try:
-    sys.modules["raphtory.arrow"] = arrow
+    sys.modules["raphtory.disk_graph"] = disk_graph
 except Exception as e:
     print(e)
 

python/src/lib.rs

@@ -1,11 +1,12 @@
+#![allow(non_local_definitions)]
 mod graphql;
 
 extern crate core;
 
 use graphql::*;
 use pyo3::prelude::*;
-#[cfg(feature = "arrow")]
-use raphtory_core::python::graph::arrow::{PyArrowGraph, PyGraphQuery, PyState};
+#[cfg(feature = "storage")]
+use raphtory_core::python::graph::disk_graph::{PyDiskGraph, PyGraphQuery, PyState};
 use raphtory_core::python::{
     graph::{
         algorithm_result::AlgorithmResult,
@@ -65,8 +66,8 @@ fn raphtory(py: Python<'_>, m: &PyModule) -> PyResult<()> {
         GraphIndex
     );
 
-    #[cfg(feature = "arrow")]
-    add_classes!(m, PyArrowGraph, PyGraphQuery, PyState);
+    #[cfg(feature = "storage")]
+    add_classes!(m, PyDiskGraph, PyGraphQuery, PyState);
 
     //GRAPHQL
     let graphql_module = PyModule::new(py, "graphql")?;
@@ -117,7 +118,7 @@ fn raphtory(py: Python<'_>, m: &PyModule) -> PyResult<()> {
         cohesive_fruchterman_reingold,
     );
 
-    #[cfg(feature = "arrow")]
+    #[cfg(feature = "storage")]
     add_functions!(algorithm_module, connected_components,);
 
     m.add_submodule(algorithm_module)?;

python/tests/test_arrow_graph.py → python/tests/test_disk_graph.py

@@ -1,4 +1,4 @@
-from raphtory import ArrowGraph, Query, State, PyDirection
+from raphtory import DiskGraph, Query, State, PyDirection
 import pandas as pd
 import tempfile
 
@@ -33,7 +33,7 @@
 
 
 def create_graph(edges, dir):
-    return ArrowGraph.load_from_pandas(dir, edges, "src", "dst", "time")
+    return DiskGraph.load_from_pandas(dir, edges, "src", "dst", "time")
 
 
 # in every test use with to create a temporary directory that will be deleted automatically