Parquet loader (#1666)

* rename graph/pandas to graph/io

* rename utils to panda_loaders

* refactor panda loader related functions to panda_loader from df_loaders

* init parquet loader

* impl and test process_parquet_file_to_df

* impl load edges from parquet for graph

* impl/test load nodes from parquet and fix names order issue

* impl/test load from parquet. load node/edge props from parquet

* add tests for props

* ref tests

* simplify tests

* more tests

* impl loaders for persistent graphs

* impl loader tests for persistent graphs

* move load_edges_deletions_from_pandas to panda_loaders

* impl load_edges_deletions_from_parquet

* impl edge deletions tests

* fmt

* add py dep

* add deps

* move compression to arrow dep

* restruct io

* restruct arrow

* tmpdir impl

* add feature gates

* make parquet loader compatible for rust

* move py impls to python package

* rename PretendDF to DFView, panda_loaders to pandas_loaders

* rid parent

* make test create parquet

* load parquet from dir or file

* change invalid layers error message to include valid layers

* fix issue with valid_layers

examples/rust/src/bin/bench/main.rs

@@ -1,6 +1,5 @@
 use raphtory::{
-    algorithms::centrality::pagerank::unweighted_page_rank,
-    graph_loader::source::csv_loader::CsvLoader, prelude::*,
+    algorithms::centrality::pagerank::unweighted_page_rank, io::csv_loader::CsvLoader, prelude::*,
 };
 use serde::Deserialize;
 use std::{

examples/rust/src/bin/btc/main.rs

@@ -2,7 +2,7 @@
 #![allow(dead_code)]
 
 use chrono::{DateTime, Utc};
-use raphtory::{graph_loader::source::csv_loader::CsvLoader, prelude::*};
+use raphtory::{io::csv_loader::CsvLoader, prelude::*};
 use regex::Regex;
 use serde::Deserialize;
 use std::{

examples/rust/src/bin/crypto/main.rs

@@ -5,7 +5,7 @@ use raphtory::{
         pathing::temporal_reachability::temporally_reachable_nodes,
     },
     db::api::view::*,
-    graph_loader::example::stable_coins::stable_coin_graph,
+    graph_loader::stable_coins::stable_coin_graph,
 };
 use std::{env, time::Instant};
 

examples/rust/src/bin/hulongbay/main.rs

@@ -9,7 +9,7 @@ use raphtory::{
             triangle_count::triangle_count,
         },
     },
-    graph_loader::source::csv_loader::CsvLoader,
+    io::csv_loader::CsvLoader,
     prelude::*,
 };
 use regex::Regex;

examples/rust/src/bin/lotr/main.rs

@@ -1,6 +1,6 @@
 use raphtory::{
     algorithms::pathing::temporal_reachability::temporally_reachable_nodes,
-    graph_loader::source::csv_loader::CsvLoader, prelude::*,
+    io::csv_loader::CsvLoader, prelude::*,
 };
 use serde::Deserialize;
 use std::{

examples/rust/src/bin/pokec/main.rs

@@ -3,7 +3,7 @@ use raphtory::{
         centrality::pagerank::unweighted_page_rank, components::weakly_connected_components,
     },
     db::{api::mutation::AdditionOps, graph::graph::Graph},
-    graph_loader::source::csv_loader::CsvLoader,
+    io::csv_loader::CsvLoader,
     prelude::*,
 };
 use serde::Deserialize;

python/tests/test_graphdb.py

@@ -2,6 +2,8 @@
 import math
 import sys
 import random
+import re
+
 import pandas as pd
 import pandas.core.frame
 import pytest
@@ -1435,6 +1437,18 @@ def test_layer():
     assert g.exclude_layers(["layer1", "layer2"]).count_edges() == 1
     assert g.exclude_layer("layer2").count_edges() == 4
 
+    with pytest.raises(
+            Exception,
+            match=re.escape("Invalid layer: test_layer. Valid layers: _default, layer1, layer2"),
+    ):
+        g.layers(["test_layer"])
+
+    with pytest.raises(
+            Exception,
+            match=re.escape("Invalid layer: test_layer. Valid layers: _default, layer1, layer2"),
+    ):
+        g.edge(1, 2).layers(["test_layer"])
+
 
 def test_layer_node():
     g = Graph()

python/tests/test_load_from_pandas.py

@@ -624,7 +624,7 @@ def assertions_layers_in_df(g):
         assert g.layers(["layer 3"]).edges.src.id.collect() == [3]
         with pytest.raises(
             Exception,
-            match=re.escape("Invalid layer test_layer."),
+            match=re.escape("Invalid layer: test_layer. Valid layers: _default, layer 1, layer 2, layer 3, layer 4, layer 5"),
         ):
             g.layers(["test_layer"])