diff --git a/.github/workflows/test_rust_workflow.yml b/.github/workflows/test_rust_workflow.yml index d57d2e8f57..e0547924bc 100644 --- a/.github/workflows/test_rust_workflow.yml +++ b/.github/workflows/test_rust_workflow.yml @@ -42,10 +42,6 @@ jobs: uses: arduino/setup-protoc@v3 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: webfactory/ssh-agent@v0.7.0 - name: Load pometry-storage key - with: - ssh-private-key: ${{ secrets.RA_SSH_PRIVATE_KEY }} - name: Rust version run: rustc --version --verbose - uses: Swatinem/rust-cache@v2 @@ -81,10 +77,6 @@ jobs: name: Cargo cache with: cache-all-crates: true - - uses: webfactory/ssh-agent@v0.9.0 - name: Load raphtory-disk_graph key - with: - ssh-private-key: ${{ secrets.RA_SSH_PRIVATE_KEY }} - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/Cargo.lock b/Cargo.lock index 8f443fd41f..e9dc3ea11f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aead" version = "0.5.2" @@ -70,9 +76,8 @@ checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "const-random", - "getrandom 0.2.15", + "getrandom", "once_cell", - "serde", "version_check", "zerocopy", ] @@ -136,9 +141,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.14" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" dependencies = [ "anstyle", "anstyle-parse", @@ -151,33 +156,33 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anstyle-parse" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", "windows-sys 0.52.0", @@ -204,16 +209,6 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" -[[package]] -name = "archery" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8967cd1cc9e9e1954f644e14fbd6042fe9a37da96c52a67e44a2ac18261f8561" -dependencies = [ - "static_assertions", - "triomphe", -] - [[package]] name = "array-init-cursor" version = "0.2.0" @@ -222,15 +217,15 @@ checksum = "bf7d0a018de4f6aa429b9d33d69edf69072b1c5b1cb8d3e4a5f7ef898fc3eb76" [[package]] name = "arrayref" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" +checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" @@ -518,9 +513,9 @@ checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a" [[package]] name = "async-compression" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" +checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" dependencies = [ "bzip2", "flate2", @@ -545,9 +540,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "7.0.6" +version = "7.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf338d20ba5bab309f55ce8df95d65ee19446f7737f06f4a64593ab2c6b546ad" +checksum = "2b76aba2f176af685c2229633881a3adeae51f87ae1811781e73910b7001c93e" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -579,26 +574,26 @@ dependencies = [ [[package]] name = "async-graphql-derive" -version = "7.0.6" +version = "7.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc51fd6b7102acda72bc94e8ae1543844d5688ff394a6cf7c21f2a07fe2d64e4" +checksum = "72e2e26a6b44bc61df3ca8546402cf9204c28e30c06084cc8e75cd5e34d4f150" dependencies = [ "Inflector", "async-graphql-parser", - "darling 0.20.9", + "darling", "proc-macro-crate 3.1.0", "proc-macro2", "quote", "strum", - "syn 2.0.66", + "syn 2.0.75", "thiserror", ] [[package]] name = "async-graphql-parser" -version = "7.0.6" +version = "7.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75361eefd64e39f89bead4cb45fddbaf60ddb0e7b15fb7c852b6088bcd63071f" +checksum = "f801451484b4977d6fe67b29030f81353cabdcbb754e5a064f39493582dac0cf" dependencies = [ "async-graphql-value", "pest", @@ -608,9 +603,9 @@ dependencies = [ [[package]] name = "async-graphql-poem" -version = "7.0.6" +version = "7.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "072888eb96e1dc0398d8e163015c3c24323a94e71b65757af4ddc07beb38df18" +checksum = "6fead92136d9c0c68b6da5ea16962604163b1f4d59750ec7b36155ca9ff105d3" dependencies = [ "async-graphql", "futures-util", @@ -625,9 +620,9 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "7.0.6" +version = "7.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1f665d2d52b41c4ed1f01c43f3ef27a2fe0af2452ed5c8bc7ac9b1a8719afaa" +checksum = "69117c43c01d81a69890a9f5dd6235f2f027ca8d1ec62d6d3c5e01ca0edb4f2b" dependencies = [ "bytes", "indexmap", @@ -637,18 +632,19 @@ dependencies = [ [[package]] name = "async-openai" -version = "0.17.1" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c6ce3eb96d0957f9d5af15b5b0d651e27da8b567d126ef7fefb889d21dda06" +checksum = "dc0e5ff98f9e7c605df4c88783a0439d1dc667ce86bd79e99d4164f8b0c05ccc" dependencies = [ "async-convert", "backoff", - "base64 0.21.7", + "base64 0.22.1", "bytes", "derive_builder", + "eventsource-stream", "futures", - "rand 0.8.5", - "reqwest", + "rand", + "reqwest 0.12.7", "reqwest-eventsource", "secrecy", "serde", @@ -679,18 +675,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "async-trait" -version = "0.1.80" +version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -718,10 +714,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ "futures-core", - "getrandom 0.2.15", + "getrandom", "instant", "pin-project-lite", - "rand 0.8.5", + "rand", "tokio", ] @@ -735,7 +731,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.4", "object", "rustc-demangle", ] @@ -799,9 +795,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" dependencies = [ "serde", ] @@ -826,9 +822,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.1" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" dependencies = [ "arrayref", "arrayvec", @@ -875,22 +871,22 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78834c15cb5d5efe3452d58b1e8ba890dd62d21907f867f383358198e56ebca5" +checksum = "6fd4c6dcc3b0aea2f5c0b4b82c2b15fe39ddbc76041a310848f4706edf76bb31" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee891b04274a59bd38b412188e24b849617b2e45a0fd8d057deb63e7403761b" +checksum = "0cc8b54b395f2fcfbb3d90c47b01c7f444d94d05bdeb775811dec868ac3bbc26" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -901,9 +897,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" dependencies = [ "serde", ] @@ -937,13 +933,13 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.99" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" +checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48" dependencies = [ "jobserver", "libc", - "once_cell", + "shlex", ] [[package]] @@ -960,9 +956,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cfg_aliases" -version = "0.1.1" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" @@ -976,7 +972,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -1040,9 +1036,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.7" +version = "4.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" +checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" dependencies = [ "clap_builder", "clap_derive", @@ -1050,39 +1046,39 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.7" +version = "4.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" +checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.1", + "strsim", ] [[package]] name = "clap_derive" -version = "4.5.5" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" +checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "clap_lex" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "colorchoice" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" [[package]] name = "comfy-table" @@ -1102,7 +1098,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7328b20597b53c2454f0b1919720c25c7339051c02b72b7e05409e00b14132be" dependencies = [ "async-trait", - "convert_case 0.6.0", + "convert_case", "json5", "lazy_static", "nom", @@ -1140,7 +1136,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.15", + "getrandom", "once_cell", "tiny-keccak", ] @@ -1151,12 +1147,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - [[package]] name = "convert_case" version = "0.6.0" @@ -1177,7 +1167,7 @@ dependencies = [ "hkdf", "hmac", "percent-encoding", - "rand 0.8.5", + "rand", "sha2", "subtle", "time", @@ -1196,15 +1186,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" dependencies = [ "libc", ] @@ -1316,7 +1306,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", - "rand_core 0.6.4", + "rand_core", "typenum", ] @@ -1352,81 +1342,60 @@ dependencies = [ [[package]] name = "darling" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" -dependencies = [ - "darling_core 0.14.4", - "darling_macro 0.14.4", -] - -[[package]] -name = "darling" -version = "0.20.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" -dependencies = [ - "darling_core 0.20.9", - "darling_macro 0.20.9", -] - -[[package]] -name = "darling_core" -version = "0.14.4" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 1.0.109", + "darling_core", + "darling_macro", ] [[package]] name = "darling_core" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.11.1", - "syn 2.0.66", + "strsim", + "syn 2.0.75", ] [[package]] name = "darling_macro" -version = "0.14.4" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ - "darling_core 0.14.4", + "darling_core", "quote", - "syn 1.0.109", + "syn 2.0.75", ] [[package]] -name = "darling_macro" -version = "0.20.9" +name = "dashmap" +version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ - "darling_core 0.20.9", - "quote", - "syn 2.0.66", + "cfg-if", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", ] [[package]] name = "dashmap" -version = "5.5.3" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +checksum = "804c8821570c3f8b70230c2ba75ffa5c0f9a4189b9a432b6656c536712acae28" dependencies = [ "cfg-if", + "crossbeam-utils", "hashbrown 0.14.5", "lock_api", "once_cell", @@ -1456,7 +1425,7 @@ dependencies = [ "bytes", "bzip2", "chrono", - "dashmap", + "dashmap 5.5.3", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -1479,7 +1448,7 @@ dependencies = [ "parking_lot", "parquet", "pin-project-lite", - "rand 0.8.5", + "rand", "sqlparser", "tempfile", "tokio", @@ -1518,7 +1487,7 @@ checksum = "3f9c93043081487e335399a21ebf8295626367a647ac5cb87d41d18afad7d0f7" dependencies = [ "arrow", "chrono", - "dashmap", + "dashmap 5.5.3", "datafusion-common", "datafusion-expr", "futures", @@ -1526,7 +1495,7 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.8.5", + "rand", "tempfile", "url", ] @@ -1623,7 +1592,7 @@ dependencies = [ "md-5", "paste", "petgraph", - "rand 0.8.5", + "rand", "regex", "sha2", "unicode-segmentation", @@ -1656,7 +1625,7 @@ dependencies = [ "once_cell", "parking_lot", "pin-project-lite", - "rand 0.8.5", + "rand", "tokio", "uuid", ] @@ -1696,9 +1665,20 @@ checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" [[package]] name = "deflate64" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83ace6c86376be0b6cdcf3fb41882e81d94b31587573d1cfa9d01cd06bba210d" +checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" + +[[package]] +name = "delegate" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee5df75c70b95bd3aacc8e2fd098797692fb1d54121019c4de481e42f04c8a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] [[package]] name = "deranged" @@ -1718,51 +1698,38 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "derive_builder" -version = "0.12.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" dependencies = [ "derive_builder_macro", ] [[package]] name = "derive_builder_core" -version = "0.12.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" dependencies = [ - "darling 0.14.4", + "darling", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.75", ] [[package]] name = "derive_builder_macro" -version = "0.12.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ "derive_builder_core", - "syn 1.0.109", -] - -[[package]] -name = "derive_more" -version = "0.99.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" -dependencies = [ - "convert_case 0.4.0", - "proc-macro2", - "quote", - "rustc_version", - "syn 1.0.109", + "syn 2.0.75", ] [[package]] @@ -1784,19 +1751,19 @@ dependencies = [ [[package]] name = "display-error-chain" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f77af9e75578c1ab34f5f04545a8b05be0c36fbd7a9bb3cf2d2a971e435fdbb9" +checksum = "7d305e5a3904ee14166439a70feef04853c1234226dbb27ede127b88dc5a4a9d" [[package]] name = "displaydoc" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -1850,11 +1817,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d8b5d72ddb5add27d0e071039643c894ebdfd14fac88eed61d9111af8395d27" dependencies = [ "Inflector", - "darling 0.20.9", + "darling", "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", "thiserror", ] @@ -1882,17 +1849,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.66", -] - -[[package]] -name = "env_logger" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -dependencies = [ - "log", - "regex", + "syn 2.0.75", ] [[package]] @@ -1940,10 +1897,10 @@ dependencies = [ [[package]] name = "examples" -version = "0.10.0" +version = "0.11.0" dependencies = [ "chrono", - "itertools 0.12.1", + "itertools 0.13.0", "raphtory", "regex", "serde", @@ -2000,12 +1957,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.30" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.8.0", ] [[package]] @@ -2039,12 +1996,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - [[package]] name = "futures" version = "0.3.30" @@ -2101,7 +2052,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -2140,20 +2091,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generator" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186014d53bc231d0090ef8d6f03e0920c54d85a5ed22f4f2f74315ec56cf83fb" -dependencies = [ - "cc", - "cfg-if", - "libc", - "log", - "rustversion", - "windows", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -2164,17 +2101,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.15" @@ -2184,7 +2110,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] @@ -2206,9 +2132,9 @@ checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "glam" -version = "0.25.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "151665d9be52f9bb40fc7966565d39666f2d1e69233571b71b87791c7e0528b3" +checksum = "c28091a37a5d09b555cb6628fd954da299b536433834f5b8e59eba78e0cbbf8a" [[package]] name = "glob" @@ -2237,9 +2163,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", "bytes", @@ -2338,6 +2264,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -2403,9 +2335,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http 1.1.0", @@ -2420,15 +2352,15 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "pin-project-lite", ] [[package]] name = "httparse" -version = "1.9.3" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0e7a4dd27b9476dc40cb050d3632d3bba3a70ddbff012285f7f8559a1e7e545" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" [[package]] name = "httpdate" @@ -2444,9 +2376,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.29" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", @@ -2468,22 +2400,23 @@ dependencies = [ [[package]] name = "hyper" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.5", + "h2 0.4.6", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", "pin-project-lite", "smallvec", "tokio", + "want", ] [[package]] @@ -2494,25 +2427,49 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.29", - "rustls", + "hyper 0.14.30", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.4.1", + "hyper-util", + "rustls 0.23.12", + "rustls-native-certs", + "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", + "tower-service", + "webpki-roots 0.26.3", ] [[package]] name = "hyper-util" -version = "0.1.5" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" dependencies = [ "bytes", + "futures-channel", "futures-util", "http 1.1.0", - "http-body 1.0.0", - "hyper 1.3.1", + "http-body 1.0.1", + "hyper 1.4.1", "pin-project-lite", + "socket2", "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -2526,7 +2483,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core 0.52.0", + "windows-core", ] [[package]] @@ -2538,124 +2495,6 @@ dependencies = [ "cc", ] -[[package]] -name = "icu_collections" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locid" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - -[[package]] -name = "icu_normalizer" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "utf16_iter", - "utf8_iter", - "write16", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" - -[[package]] -name = "icu_properties" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locid_transform", - "icu_properties_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" - -[[package]] -name = "icu_provider" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_provider_macros", - "stable_deref_trait", - "tinystr", - "writeable", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "ident_case" version = "1.0.1" @@ -2664,21 +2503,19 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "1.0.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ - "icu_normalizer", - "icu_properties", - "smallvec", - "utf8_iter", + "unicode-bidi", + "unicode-normalization", ] [[package]] name = "indexmap" -version = "2.2.6" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -2732,20 +2569,20 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ - "hermit-abi", + "hermit-abi 0.4.0", "libc", "windows-sys 0.52.0", ] [[package]] name = "is_terminal_polyfill" -version = "1.70.0" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" @@ -2756,6 +2593,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -2765,6 +2611,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -2773,16 +2628,16 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] [[package]] name = "js-raphtory" -version = "0.10.0" +version = "0.11.0" dependencies = [ "chrono", "console_error_panic_hook", @@ -2796,9 +2651,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -2816,13 +2671,14 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "8.3.0" +version = "9.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" +checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f" dependencies = [ "base64 0.21.7", + "js-sys", "pem", - "ring 0.16.20", + "ring", "serde", "serde_json", "simple_asn1", @@ -2841,9 +2697,9 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "levenshtein_automata" @@ -2917,9 +2773,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" [[package]] name = "libm" @@ -2939,12 +2795,6 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" -[[package]] -name = "litemap" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" - [[package]] name = "lock_api" version = "0.4.12" @@ -2964,38 +2814,24 @@ checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" - -[[package]] -name = "loom" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" -dependencies = [ - "cfg-if", - "generator", - "pin-utils", - "scoped-tls", - "tracing", - "tracing-subscriber", -] +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lru" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" +checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" dependencies = [ "hashbrown 0.14.5", ] [[package]] name = "lz4" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6eab492fe7f8651add23237ea56dbf11b3c4ff762ab83d40a47f11433421f91" +checksum = "958b4caa893816eea05507c20cfe47574a43d9a697138a7872990bba8a0ece68" dependencies = [ "libc", "lz4-sys", @@ -3003,9 +2839,9 @@ dependencies = [ [[package]] name = "lz4-sys" -version = "1.9.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9764018d143cc854c9f17f0b907de70f14393b1f502da6375dce70f00514eb3" +checksum = "109de74d5d2353660401699a4174a4ff23fcc649caf553df71933c7fb45ad868" dependencies = [ "cc", "libc", @@ -3072,9 +2908,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.3" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0d8b92cd8358e8d229c11df9358decae64d137c5be540952c5ca7b25aea768" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" @@ -3102,14 +2938,24 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" -version = "2.0.4" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" dependencies = [ "mime", "unicase", ] +[[package]] +name = "minicov" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c71e683cd655513b99affab7d317deb690528255a0d5f717f1024093c12b169" +dependencies = [ + "cc", + "walkdir", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -3118,29 +2964,39 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" -version = "0.8.11" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ + "hermit-abi 0.3.9", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.48.0", + "wasi", + "windows-sys 0.52.0", ] [[package]] name = "moka" -version = "0.12.7" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0d88686dc561d743b40de8269b26eaf0dc58781bde087b0984646602021d08" +checksum = "32cf62eb4dd975d2dde76432fb1075c49e3ee2331cf36f1f8fd4b66550d32b6f" dependencies = [ "crossbeam-channel", "crossbeam-epoch", @@ -3169,8 +3025,7 @@ dependencies = [ "httparse", "memchr", "mime", - "spin 0.9.8", - "tokio", + "spin", "version_check", ] @@ -3210,42 +3065,49 @@ checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" [[package]] name = "neo4rs" -version = "0.6.2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f1d8019cfd20115fd664e4e1ecdf144ab4fce8bf7f7ddc7b6523a8835a770a2" +checksum = "43dd99fe7dbc68f754759874d83ec2ca43a61ab7d51c10353d024094805382be" dependencies = [ "async-trait", + "backoff", "bytes", "chrono", + "chrono-tz", "deadpool", + "delegate", "futures", "log", "neo4rs-macros", + "paste", "pin-project-lite", + "rustls-native-certs", + "rustls-pemfile 2.1.3", + "serde", "thiserror", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", "url", - "webpki-roots 0.23.1", + "webpki-roots 0.26.3", ] [[package]] name = "neo4rs-macros" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf52bfa6042343585f458f38f18094a53e8d8c417221867918e9f0a6885f42a" +checksum = "53a0d57c55d2d1dc62a2b1d16a0a1079eb78d67c36bdf468d582ab4482ec7002" dependencies = [ "quote", - "syn 1.0.109", + "syn 2.0.75", ] [[package]] name = "nix" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cfg-if", "cfg_aliases", "libc", @@ -3287,9 +3149,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", @@ -3357,7 +3219,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] @@ -3369,10 +3231,10 @@ checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" dependencies = [ "base64 0.13.1", "chrono", - "getrandom 0.2.15", + "getrandom", "http 0.2.12", - "rand 0.8.5", - "reqwest", + "rand", + "reqwest 0.11.27", "serde", "serde_json", "serde_path_to_error", @@ -3383,9 +3245,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.0" +version = "0.36.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" +checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" dependencies = [ "memchr", ] @@ -3419,18 +3281,15 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oneshot" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071d1cf3298ad8e543dca18217d198cb6a3884443d204757b9624b935ef09fa0" -dependencies = [ - "loom", -] +checksum = "e296cf87e61c9cfc1a61c3c63a0f7f286ed4554e0e22be84e8a38e1d264a2a29" [[package]] name = "oorandom" -version = "11.1.3" +version = "11.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "opaque-debug" @@ -3446,25 +3305,23 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "opentelemetry" -version = "0.21.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" +checksum = "1b69a91d4893e713e06f724597ad630f1fa76057a5e1026c0ca67054a9032a76" dependencies = [ "futures-core", "futures-sink", - "indexmap", "js-sys", "once_cell", "pin-project-lite", "thiserror", - "urlencoding", ] [[package]] name = "opentelemetry-jaeger" -version = "0.20.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e617c66fd588e40e0dbbd66932fdc87393095b125d4459b1a3a10feb1712f8a1" +checksum = "501b471b67b746d9a07d4c29f8be00f952d1a2eca356922ede0098cbaddff19f" dependencies = [ "async-trait", "futures-core", @@ -3478,30 +3335,27 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.13.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" -dependencies = [ - "opentelemetry", -] +checksum = "1869fb4bb9b35c5ba8a1e40c9b128a7b4c010d07091e864a29da19e4fe2ca4d7" [[package]] name = "opentelemetry_sdk" -version = "0.21.2" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" +checksum = "ae312d58eaa90a82d2e627fd86e075cf5230b3f11794e2ed74199ebbe572d4fd" dependencies = [ "async-trait", - "crossbeam-channel", "futures-channel", "futures-executor", "futures-util", "glob", + "lazy_static", "once_cell", "opentelemetry", - "ordered-float 4.2.0", + "ordered-float 4.2.2", "percent-encoding", - "rand 0.8.5", + "rand", "thiserror", "tokio", "tokio-stream", @@ -3518,9 +3372,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.2.0" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" +checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6" dependencies = [ "num-traits", ] @@ -3557,7 +3411,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -3595,7 +3449,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -3672,11 +3526,12 @@ dependencies = [ [[package]] name = "pem" -version = "1.1.1" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" dependencies = [ - "base64 0.13.1", + "base64 0.22.1", + "serde", ] [[package]] @@ -3687,9 +3542,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" +checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" dependencies = [ "memchr", "thiserror", @@ -3698,9 +3553,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" +checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" dependencies = [ "pest", "pest_generator", @@ -3708,22 +3563,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" +checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "pest_meta" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" +checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" dependencies = [ "once_cell", "pest", @@ -3766,7 +3621,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared", - "rand 0.8.5", + "rand", ] [[package]] @@ -3778,6 +3633,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.75", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -3835,9 +3710,9 @@ dependencies = [ [[package]] name = "poem" -version = "3.0.1" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88b6912ed1e8833d7c22c9c986c517f4518d7d37e3c04566d917c789aaea591" +checksum = "f1ba1c27f8f89e1bccdda0c680f72790545a11a8d8555819472f5839d7a8ca9d" dependencies = [ "base64 0.22.1", "bytes", @@ -3847,29 +3722,24 @@ dependencies = [ "headers", "http 1.1.0", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-util", "mime", - "multer", "nix", "parking_lot", "percent-encoding", "pin-project-lite", "poem-derive", - "quick-xml", "regex", "rfc7239", "serde", "serde_json", "serde_urlencoded", - "serde_yaml", "smallvec", "sync_wrapper 1.0.1", - "tempfile", "thiserror", "time", "tokio", - "tokio-stream", "tokio-tungstenite", "tokio-util", "tracing", @@ -3878,57 +3748,14 @@ dependencies = [ [[package]] name = "poem-derive" -version = "3.0.0" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2b961d58a6c53380c20236394381d9292fda03577f902b158f1638932964dcf" +checksum = "a62fea1692d80a000126f9b28d865012a160b80000abb53ccf152b428222c155" dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.66", -] - -[[package]] -name = "poem-openapi" -version = "5.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6445b50be2e26f142d4e554d15773fc1e7510b994083c9625a65eba0d3f4287" -dependencies = [ - "base64 0.22.1", - "bytes", - "derive_more", - "futures-util", - "indexmap", - "mime", - "num-traits", - "poem", - "poem-openapi-derive", - "quick-xml", - "regex", - "serde", - "serde_json", - "serde_urlencoded", - "serde_yaml", - "thiserror", - "tokio", -] - -[[package]] -name = "poem-openapi-derive" -version = "5.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e890165626ff447a1ff3d6f2293e6ccacbf7fcbdd4c94086aa548de655735b03" -dependencies = [ - "darling 0.20.9", - "http 1.1.0", - "indexmap", - "mime", - "proc-macro-crate 3.1.0", - "proc-macro2", - "quote", - "regex", - "syn 2.0.66", - "thiserror", + "syn 2.0.75", ] [[package]] @@ -3950,7 +3777,7 @@ dependencies = [ "ethnum", "fast-float", "foreign_vec", - "getrandom 0.2.15", + "getrandom", "hashbrown 0.14.5", "itoa", "multiversion", @@ -4044,13 +3871,13 @@ dependencies = [ [[package]] name = "pometry-storage" -version = "0.10.0" +version = "0.11.0" [[package]] name = "portable-atomic" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" [[package]] name = "powerfmt" @@ -4060,9 +3887,12 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "pretty_assertions" @@ -4081,7 +3911,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -4105,9 +3935,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -4120,24 +3950,24 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", "version_check", "yansi 1.0.1", ] [[package]] name = "proptest" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" +checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.5.0", + "bitflags 2.6.0", "lazy_static", "num-traits", - "rand 0.8.5", - "rand_chacha 0.3.1", + "rand", + "rand_chacha", "rand_xorshift", "regex-syntax 0.8.4", "rusty-fork", @@ -4147,9 +3977,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.12.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +checksum = "e13db3d3fde688c61e2446b4d843bc27a7e8af269a69440c0308021dc92333cc" dependencies = [ "bytes", "prost-derive", @@ -4157,13 +3987,13 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.12.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +checksum = "5bb182580f71dd070f88d01ce3de9f4da5021db7115d2e1c3605a754153b77c1" dependencies = [ "bytes", "heck 0.5.0", - "itertools 0.12.1", + "itertools 0.13.0", "log", "multimap", "once_cell", @@ -4172,28 +4002,28 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.66", + "syn 2.0.75", "tempfile", ] [[package]] name = "prost-derive" -version = "0.12.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +checksum = "18bec9b0adc4eba778b33684b7ba3e7137789434769ee3ce3930463ef904cfca" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "prost-types" -version = "0.12.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +checksum = "cee5168b05f49d4b0ca581206eb14a7b22fafd963efe729ac48eb03266e25cc2" dependencies = [ "prost", ] @@ -4256,7 +4086,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -4269,7 +4099,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -4288,7 +4118,7 @@ dependencies = [ "libc", "once_cell", "raw-cpuid", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "web-sys", "winapi", ] @@ -4299,37 +4129,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quick-xml" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "quickcheck" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44883e74aa97ad63db83c4bf8ca490f02b2fc02f92575e720c8551e843c945f" -dependencies = [ - "env_logger 0.7.1", - "log", - "rand 0.7.3", - "rand_core 0.5.1", -] - [[package]] name = "quickcheck" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ - "env_logger 0.8.4", + "env_logger", "log", - "rand 0.8.5", + "rand", ] [[package]] @@ -4344,59 +4152,71 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.36" +name = "quinn" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" dependencies = [ - "proc-macro2", + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash 2.0.0", + "rustls 0.23.12", + "socket2", + "thiserror", + "tokio", + "tracing", ] [[package]] -name = "rand" -version = "0.4.6" +name = "quinn-proto" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" dependencies = [ - "fuchsia-cprng", - "libc", - "rand_core 0.3.1", - "rdrand", - "winapi", + "bytes", + "rand", + "ring", + "rustc-hash 2.0.0", + "rustls 0.23.12", + "slab", + "thiserror", + "tinyvec", + "tracing", ] [[package]] -name = "rand" -version = "0.7.3" +name = "quinn-udp" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" dependencies = [ - "getrandom 0.1.16", "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", ] [[package]] -name = "rand" -version = "0.8.5" +name = "quote" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", + "proc-macro2", ] [[package]] -name = "rand_chacha" -version = "0.2.2" +name = "rand" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", + "libc", + "rand_chacha", + "rand_core", ] [[package]] @@ -4406,31 +4226,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", + "rand_core", ] [[package]] @@ -4439,7 +4235,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.15", + "getrandom", ] [[package]] @@ -4449,16 +4245,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand 0.8.5", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", + "rand", ] [[package]] @@ -4467,23 +4254,20 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" dependencies = [ - "rand_core 0.6.4", + "rand_core", ] [[package]] name = "raphtory" -version = "0.10.0" +version = "0.11.0" dependencies = [ - "ahash", "async-openai", "async-trait", "bincode", - "bytemuck", - "bytes", "bzip2", "chrono", "csv", - "dashmap", + "dashmap 6.0.1", "display-error-chain", "dotenv", "either", @@ -4491,7 +4275,7 @@ dependencies = [ "flate2", "futures-util", "glam", - "itertools 0.12.1", + "itertools 0.13.0", "kdam", "lock_api", "memmap2", @@ -4500,12 +4284,11 @@ dependencies = [ "num-integer", "num-traits", "once_cell", - "ordered-float 4.2.0", + "ordered-float 4.2.2", "ouroboros", "parking_lot", "polars-arrow", "polars-parquet", - "polars-utils", "pometry-storage", "pretty_assertions", "proptest", @@ -4514,74 +4297,66 @@ dependencies = [ "prost-types", "pyo3", "quad-rand", - "quickcheck 1.0.3", + "quickcheck", "quickcheck_macros", - "rand 0.8.5", + "rand", "rand_distr", "raphtory-api", "rayon", "regex", - "reqwest", - "rpds", - "rustc-hash", + "reqwest 0.12.7", + "rustc-hash 2.0.0", "serde", "serde_json", "sorted_vector_map", "streaming-stats", "tantivy", - "tempdir", "tempfile", "thiserror", - "thread_local", "tokio", - "twox-hash", "zip", ] [[package]] name = "raphtory-api" -version = "0.10.0" +version = "0.11.0" dependencies = [ "chrono", - "dashmap", + "dashmap 6.0.1", "lock_api", "num-traits", "parking_lot", "proptest", "pyo3", - "quickcheck 1.0.3", + "quickcheck", "quickcheck_macros", - "rand 0.8.5", + "rand", "rayon", - "rustc-hash", + "rustc-hash 2.0.0", "serde", "twox-hash", ] [[package]] name = "raphtory-benchmark" -version = "0.10.0" +version = "0.11.0" dependencies = [ - "chrono", "clap", "criterion", "csv", "flate2", - "polars-arrow", "pometry-storage", - "rand 0.8.5", + "rand", "raphtory", "raphtory-api", - "raphtory-graphql", "rayon", "sorted_vector_map", "tempfile", - "tokio", ] [[package]] name = "raphtory-cypher" -version = "0.10.0" +version = "0.11.0" dependencies = [ "arrow", "arrow-array", @@ -4591,7 +4366,7 @@ dependencies = [ "clap", "datafusion", "futures", - "itertools 0.12.1", + "itertools 0.13.0", "lazy_static", "pest", "pest_derive", @@ -4599,7 +4374,6 @@ dependencies = [ "pometry-storage", "pretty_assertions", "proptest", - "rand 0.8.5", "raphtory", "rayon", "serde", @@ -4612,21 +4386,18 @@ dependencies = [ [[package]] name = "raphtory-graphql" -version = "0.10.0" +version = "0.11.0" dependencies = [ "async-graphql", "async-graphql-poem", - "async-stream", - "base64 0.21.7", + "base64 0.22.1", "base64-compat", - "bincode", "chrono", "config", "crossbeam-channel", - "dotenv", "dynamic-graphql", "futures-util", - "itertools 0.12.1", + "itertools 0.13.0", "jsonwebtoken", "moka", "oauth2", @@ -4634,21 +4405,18 @@ dependencies = [ "opentelemetry", "opentelemetry-jaeger", "opentelemetry_sdk", - "ordered-float 4.2.0", + "ordered-float 4.2.2", "parking_lot", "poem", - "poem-openapi", "pyo3", "raphtory", "raphtory-api", - "reqwest", + "reqwest 0.12.7", "serde", "serde_json", "tempfile", "thiserror", - "time", "tokio", - "toml", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -4659,25 +4427,17 @@ dependencies = [ [[package]] name = "raphtory-pymodule" -version = "0.10.0" +version = "0.11.0" dependencies = [ - "async-graphql", - "crossbeam-channel", - "dynamic-graphql", - "itertools 0.12.1", "pyo3", "pyo3-build-config", "raphtory", "raphtory-graphql", - "reqwest", - "serde", - "serde_json", - "tokio", ] [[package]] name = "raphtory_netflow" -version = "0.10.0" +version = "0.11.0" dependencies = [ "pyo3", "pyo3-build-config", @@ -4687,11 +4447,11 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.0.2" +version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e29830cbb1290e404f24c73af91c5d8d631ce7e128691e9477556b540cd01ecd" +checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] @@ -4714,29 +4474,20 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "redox_syscall" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] name = "regex" -version = "1.10.5" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ "aho-corasick", "memchr", @@ -4776,15 +4527,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] - [[package]] name = "reqwest" version = "0.11.27" @@ -4799,42 +4541,85 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", - "hyper-rustls", + "hyper 0.14.30", + "hyper-rustls 0.24.2", "ipnet", "js-sys", "log", "mime", - "mime_guess", "once_cell", "percent-encoding", "pin-project-lite", - "rustls", - "rustls-native-certs", - "rustls-pemfile", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", "system-configuration", "tokio", - "tokio-rustls", - "tokio-util", + "tokio-rustls 0.24.1", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", "web-sys", "webpki-roots 0.25.4", "winreg", ] +[[package]] +name = "reqwest" +version = "0.12.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.4.1", + "hyper-rustls 0.27.2", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "mime_guess", + "once_cell", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls 0.23.12", + "rustls-native-certs", + "rustls-pemfile 2.1.3", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 1.0.1", + "tokio", + "tokio-rustls 0.26.0", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 0.26.3", + "windows-registry", +] + [[package]] name = "reqwest-eventsource" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f03f570355882dd8d15acc3a313841e6e90eddbc76a93c748fd82cc13ba9f51" +checksum = "632c55746dbb44275691640e7b40c907c16a2dc1a5842aa98aaec90da6ec6bde" dependencies = [ "eventsource-stream", "futures-core", @@ -4842,7 +4627,7 @@ dependencies = [ "mime", "nom", "pin-project-lite", - "reqwest", + "reqwest 0.12.7", "thiserror", ] @@ -4858,22 +4643,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b106a85eeb5b0336d16d6a20eab857f92861d4fbb1eb9a239866fb98fb6a1063" dependencies = [ - "uncased", -] - -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", + "uncased", ] [[package]] @@ -4884,10 +4654,10 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.15", + "getrandom", "libc", - "spin 0.9.8", - "untrusted 0.9.0", + "spin", + "untrusted", "windows-sys 0.52.0", ] @@ -4898,21 +4668,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" dependencies = [ "base64 0.21.7", - "bitflags 2.5.0", + "bitflags 2.6.0", "serde", "serde_derive", ] -[[package]] -name = "rpds" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0e15515d3ce3313324d842629ea4905c25a13f81953eadb88f85516f59290a4" -dependencies = [ - "archery", - "serde", -] - [[package]] name = "rust-ini" version = "0.19.0" @@ -4945,6 +4705,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc_version" version = "0.4.0" @@ -4960,7 +4726,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", @@ -4974,19 +4740,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", - "ring 0.17.8", + "ring", "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki 0.102.6", + "subtle", + "zeroize", +] + [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "04182dffc9091a404e0fc069ea5cd60e5b866c3adf881eff99a32d048242dffa" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 2.1.3", + "rustls-pki-types", "schannel", "security-framework", ] @@ -5001,23 +4782,40 @@ dependencies = [ ] [[package]] -name = "rustls-webpki" -version = "0.100.3" +name = "rustls-pemfile" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6a5fc258f1c1276dfe3016516945546e2d5383911efc0fc4f1cdc5df3a4ae3" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "base64 0.22.1", + "rustls-pki-types", ] +[[package]] +name = "rustls-pki-types" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" + [[package]] name = "rustls-webpki" version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", + "ring", + "untrusted", +] + +[[package]] +name = "rustls-webpki" +version = "0.102.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -5080,8 +4878,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] @@ -5096,11 +4894,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", @@ -5109,9 +4907,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" dependencies = [ "core-foundation-sys", "libc", @@ -5131,9 +4929,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.203" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" dependencies = [ "serde_derive", ] @@ -5151,22 +4949,23 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -5183,9 +4982,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" dependencies = [ "serde", ] @@ -5202,19 +5001,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "sha1" version = "0.10.6" @@ -5246,6 +5032,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -5360,19 +5152,14 @@ dependencies = [ [[package]] name = "sorted_vector_map" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "834e4ea0bc5317590acf76b0cd074e1724668da00c82f58e653c7fde0e755d52" +checksum = "d9167648c2ababdbe45294fe7f7b0ab56555fd754990a7637a5a420774461368" dependencies = [ - "quickcheck 0.9.2", + "itertools 0.11.0", + "quickcheck", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -5397,7 +5184,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -5461,12 +5248,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - [[package]] name = "strsim" version = "0.11.1" @@ -5475,9 +5256,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ "strum_macros", ] @@ -5492,14 +5273,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "subtle" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -5514,9 +5295,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.66" +version = "2.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" dependencies = [ "proc-macro2", "quote", @@ -5538,17 +5319,6 @@ dependencies = [ "futures-core", ] -[[package]] -name = "synstructure" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "system-configuration" version = "0.5.1" @@ -5608,7 +5378,7 @@ dependencies = [ "rayon", "regex", "rust-stemmers", - "rustc-hash", + "rustc-hash 1.1.0", "serde", "serde_json", "sketches-ddsketch", @@ -5725,30 +5495,21 @@ checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" [[package]] name = "target-lexicon" -version = "0.12.14" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" - -[[package]] -name = "tempdir" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" -dependencies = [ - "rand 0.4.6", - "remove_dir_all", -] +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.10.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5763,22 +5524,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.61" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.61" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -5853,16 +5614,6 @@ dependencies = [ "crunchy", ] -[[package]] -name = "tinystr" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" -dependencies = [ - "displaydoc", - "zerovec", -] - [[package]] name = "tinytemplate" version = "1.2.1" @@ -5873,34 +5624,48 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" -version = "1.38.0" +version = "1.39.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -5909,7 +5674,18 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.12", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls 0.23.12", + "rustls-pki-types", "tokio", ] @@ -5926,9 +5702,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.21.0" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" +checksum = "c6989540ced10490aaf14e6bad2e3d33728a2813310a0c71d1574304c49631cd" dependencies = [ "futures-util", "log", @@ -5952,21 +5728,21 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.14" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.14", + "toml_edit 0.22.20", ] [[package]] name = "toml_datetime" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] @@ -5995,22 +5771,43 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.14" +version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ "indexmap", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.13", + "winnow 0.6.18", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", ] +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -6031,7 +5828,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -6057,9 +5854,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.22.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" +checksum = "f68803492bf28ab40aeccaecc7021096bd256baf7ca77c3d425d89b35a7be4e4" dependencies = [ "js-sys", "once_cell", @@ -6093,9 +5890,9 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.12" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b2cb4fbb9995eeb36ac86fadf24031ccd58f99d6b4b2d7b911db70bddb80d90" +checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" [[package]] name = "try-lock" @@ -6105,9 +5902,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.21.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" +checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" dependencies = [ "byteorder", "bytes", @@ -6115,10 +5912,9 @@ dependencies = [ "http 1.1.0", "httparse", "log", - "rand 0.8.5", + "rand", "sha1", "thiserror", - "url", "utf-8", ] @@ -6129,7 +5925,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand 0.8.5", + "rand", "static_assertions", ] @@ -6169,12 +5965,27 @@ dependencies = [ "version_check", ] +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.11.0" @@ -6203,18 +6014,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" @@ -6223,9 +6022,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.1" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", @@ -6233,36 +6032,18 @@ dependencies = [ "serde", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - [[package]] name = "utf-8" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8-ranges" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - [[package]] name = "utf8parse" version = "0.2.2" @@ -6271,11 +6052,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ - "getrandom 0.2.15", + "getrandom", "serde", ] @@ -6287,9 +6068,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wait-timeout" @@ -6319,12 +6100,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -6333,34 +6108,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" dependencies = [ "cfg-if", "js-sys", @@ -6370,9 +6146,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6380,31 +6156,32 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "wasm-bindgen-test" -version = "0.3.42" +version = "0.3.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9bf62a58e0780af3e852044583deee40983e5886da43a271dd772379987667b" +checksum = "68497a05fb21143a08a7d24fc81763384a3072ee43c44e86aad1744d6adef9d9" dependencies = [ "console_error_panic_hook", "js-sys", + "minicov", "scoped-tls", "wasm-bindgen", "wasm-bindgen-futures", @@ -6413,13 +6190,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.42" +version = "0.3.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0" +checksum = "4b8220be1fa9e4c889b30fd207d4906657e7e90b12e0e6b0c8b8d8709f5de021" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -6437,9 +6214,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", @@ -6447,9 +6224,9 @@ dependencies = [ [[package]] name = "web-time" -version = "0.2.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ "js-sys", "wasm-bindgen", @@ -6457,18 +6234,18 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.23.1" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" -dependencies = [ - "rustls-webpki 0.100.3", -] +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "webpki-roots" -version = "0.25.4" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +dependencies = [ + "rustls-pki-types", +] [[package]] name = "wildmatch" @@ -6494,11 +6271,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6507,42 +6284,43 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.54.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" -dependencies = [ - "windows-core 0.54.0", - "windows-targets 0.52.5", -] - [[package]] name = "windows-core" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] -name = "windows-core" -version = "0.54.0" +name = "windows-registry" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ "windows-result", - "windows-targets 0.52.5", + "windows-strings", + "windows-targets 0.52.6", ] [[package]] name = "windows-result" -version = "0.1.2" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ - "windows-targets 0.52.5", + "windows-result", + "windows-targets 0.52.6", ] [[package]] @@ -6560,7 +6338,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", ] [[package]] @@ -6580,18 +6367,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -6602,9 +6389,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -6614,9 +6401,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -6626,15 +6413,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -6644,9 +6431,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -6656,9 +6443,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -6668,9 +6455,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -6680,9 +6467,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" @@ -6695,9 +6482,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.13" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" dependencies = [ "memchr", ] @@ -6712,18 +6499,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - -[[package]] -name = "writeable" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" - [[package]] name = "xz2" version = "0.1.7" @@ -6754,69 +6529,25 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" -[[package]] -name = "yoke" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", - "synstructure", -] - [[package]] name = "zerocopy" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "zerofrom" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.4" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", - "synstructure", + "syn 2.0.75", ] [[package]] @@ -6836,36 +6567,14 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", -] - -[[package]] -name = "zerovec" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "zip" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775a2b471036342aa69bc5a602bc889cb0a06cda00477d0c69566757d5553d39" +checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" dependencies = [ "aes", "arbitrary", @@ -6881,7 +6590,7 @@ dependencies = [ "lzma-rs", "memchr", "pbkdf2", - "rand 0.8.5", + "rand", "sha1", "thiserror", "time", @@ -6906,27 +6615,27 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.1.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index c8b715eed7..9545cca7f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ default-members = ["raphtory"] resolver = "2" [workspace.package] -version = "0.10.0" +version = "0.11.0" documentation = "https://raphtory.readthedocs.io/en/latest/" repository = "https://github.com/Raphtory/raphtory/" license = "GPL-3.0" @@ -38,18 +38,18 @@ pometry-storage = { version = ">=0.8.1", path = "pometry-storage" } #[private-storage] # pometry-storage = { path = "pometry-storage-private", package = "pometry-storage-private" } async-graphql = { version = "7.0.5", features = ["dynamic-schema"] } +bincode = "1.3.3" async-graphql-poem = "7.0.5" dynamic-graphql = "0.9.0" -reqwest = { version = "0.11.22", default-features = false, features = ["rustls-tls"] } +reqwest = { version = "0.12.7", default-features = false, features = ["rustls-tls", "multipart", "json"] } serde = { version = "1.0.197", features = ["derive", "rc"] } serde_json = "1.0.114" pyo3 = { version = "0.20.0", features = ["multiple-pymethods", "chrono"] } -pyo3-asyncio = { version = "0.20.0", features = ["tokio-runtime"] } pyo3-build-config = "0.20.0" -itertools = "0.12.1" +itertools = "0.13.0" rand = "0.8.5" rayon = "1.8.1" -sorted_vector_map = "0.1.0" +sorted_vector_map = "0.2.0" tokio = { version = "1.36.0", features = ["full"] } once_cell = "1.19.0" parking_lot = { version = "0.12.1", features = ["serde", "arc_lock", "send_guard"] } @@ -58,30 +58,28 @@ chrono = { version = "0.4.38", features = ["serde"] } tempfile = "3.10.0" futures-util = "0.3.30" thiserror = "1.0.57" -bincode = "1.3.3" dotenv = "0.15.0" csv = "1.3.0" flate2 = "1.0.28" regex = "1.10.3" -genawaiter = "0.99.1" num-traits = "0.2.18" num-integer = "0.1" rand_distr = "0.4.3" -rustc-hash = "1.1.0" +rustc-hash = "2.0.0" twox-hash = "1.6.3" lock_api = { version = "0.4.11", features = ["arc_lock", "serde"] } -dashmap = { version = "5.5.3", features = ["serde"] } +dashmap = { version = "6.0.1", features = ["serde"] } enum_dispatch = "0.3.12" -glam = "0.25.0" +glam = "0.29.0" quad-rand = "0.2.1" zip = "2.1.3" -neo4rs = "0.6.2" +neo4rs = "0.8.0" bzip2 = "0.4.4" tantivy = "0.22" async-trait = "0.1.77" -async-openai = "0.17.1" +async-openai = "0.23.4" oauth2 = "4.0" -jsonwebtoken = "8.0" +jsonwebtoken = "9.3.0" num = "0.4.1" display-error-chain = "0.2.0" polars-arrow = "0.39.2" @@ -91,28 +89,23 @@ kdam = { version = "0.5.1" } pretty_assertions = "1.4.0" quickcheck = "1.0.3" quickcheck_macros = "1.0.0" -tempdir = "0.3.7" streaming-stats = "0.2.3" proptest = "1.4.0" criterion = "0.5.1" crossbeam-channel = "0.5.11" -base64 = "0.21.7" -poem = "3.0.1" -poem-openapi = "5.0.2" -async-stream = "0.3.5" -opentelemetry = "0.21.0" -opentelemetry_sdk = { version = "0.21.0", features = ["rt-tokio"] } -opentelemetry-jaeger = { version = "0.20.0", features = ["rt-tokio"] } +base64 = "0.22.1" +poem = { version = "3.0.1", features = ["cookie"] } +opentelemetry = "0.23.0" +opentelemetry_sdk = { version = "0.23.0", features = ["rt-tokio"] } +opentelemetry-jaeger = { version = "0.22.0", features = ["rt-tokio"] } tracing = "0.1.37" -tracing-opentelemetry = "0.22.0" +tracing-opentelemetry = "0.24.0" tracing-subscriber = { version = "0.3.16", features = ["std", "env-filter"] } walkdir = "2" uuid = { version = "1.0", features = ["v4"] } config = "0.14.0" either = "=1.11.0" -toml = "0.8.10" clap = { version = "4.3.11", features = ["derive"] } -tar = "0.4.38" wasm-bindgen = "0.2.91" serde-wasm-bindgen = "0.6.4" js-sys = "0.3.68" @@ -120,20 +113,15 @@ console_error_panic_hook = "0.1.7" wasm-bindgen-test = "0.3.41" memmap2 = { version = "0.9.4" } ahash = { version = "0.8.3", features = ["serde"] } -xxhash-rust = "0.8.7" strum = { version = "0.26.1", features = ["derive"] } bytemuck = { version = "1.15.0" } -rpds = { version = "1.1.0", features = ["serde"] } -thread_local = "1.1.8" ouroboros = "0.18.3" url = "2.2" base64-compat = { package = "base64-compat", version = "1.0.0" } -time = "0.3.36" -prost = "0.12" -prost-types = "0.12" -bytes = "1.6.0" -prost-build = "0.12" +prost = "0.13.1" +prost-types = "0.13.1" +prost-build = "0.13.1" lazy_static = "1.4.0" pest = "2.7.8" @@ -145,7 +133,6 @@ futures = "0.3" arrow = { version = "50" } arrow-buffer = { version = "50" } arrow-schema = { version = "50" } -arrow-data = { version = "50" } arrow-array = { version = "50" } moka = { version = "0.12.7", features = ["sync"] } diff --git a/README.md b/README.md index af7ae05a36..89776e145b 100644 --- a/README.md +++ b/README.md @@ -151,28 +151,35 @@ Below is a small example creating a graph, running a server hosting this data, a ```python from raphtory import Graph -from raphtory.graphql import RaphtoryServer +from raphtory.graphql import GraphServer import pandas as pd +import os # URL for lord of the rings data from our main tutorial url = "https://raw.githubusercontent.com/Raphtory/Data/main/lotr-with-header.csv" df = pd.read_csv(url) # Load the lord of the rings graph from the dataframe -graph = Graph.load_from_pandas(df,"src_id","dst_id","time") +graph = Graph() +graph.load_edges_from_pandas(df,"time","src_id","dst_id") -#Create a dictionary of queryable graphs and start the graphql server with it. This returns a client we can query with -client = RaphtoryServer({"lotr_graph":graph}).start() +#Create a working_dir for your server and save your graph into it +#You can save any number of graphs here or create them via the server ones its running +os.makedirs("graphs/", exist_ok=True) +graph.save_to_file("graphs/lotr_graph") -#Wait until the server has started up -client.wait_for_online() +# Launch the server and get a client to it. +server = GraphServer(work_dir="graphs/").start() +client = server.get_client() #Run a basic query to get the names of the characters + their degree results = client.query("""{ - graph(name: "lotr_graph") { + graph(path: "lotr_graph") { nodes { - name - degree + list{ + name + degree + } } } }""") diff --git a/docs/source/reference/graphql/server.rst b/docs/source/reference/graphql/server.rst index f0014dfc94..ade4f26be1 100644 --- a/docs/source/reference/graphql/server.rst +++ b/docs/source/reference/graphql/server.rst @@ -1,7 +1,7 @@ Server ====== -.. autoclass:: raphtory.graphql.RaphtoryServer +.. autoclass:: raphtory.graphql.GraphServer :autosummary: :members: :undoc-members: diff --git a/examples/netflow/Cargo.toml b/examples/netflow/Cargo.toml index 6000f876d7..cd9b9dddb1 100644 --- a/examples/netflow/Cargo.toml +++ b/examples/netflow/Cargo.toml @@ -6,8 +6,8 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -raphtory_core = { path = "../../raphtory", version = "0.10.0", features = ["python", "search", "vectors"], package = "raphtory" } -raphtory-graphql = { path = "../../raphtory-graphql", version = "0.10.0",features = ["python"] } +raphtory_core = { path = "../../raphtory", version = "0.11.0", features = ["python", "search", "vectors"], package = "raphtory" } +raphtory-graphql = { path = "../../raphtory-graphql", version = "0.11.0",features = ["python"] } pyo3 = { workspace = true } [lib] diff --git a/examples/python/socio-patterns/example.ipynb b/examples/python/socio-patterns/example.ipynb index 833be8e6c8..b727f28446 100644 --- a/examples/python/socio-patterns/example.ipynb +++ b/examples/python/socio-patterns/example.ipynb @@ -99,13 +99,13 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "16de15732c834eafb018c88c0b052c00", + "model_id": "ec2b2e5c92b54503940f06b3c30f184a", "version_major": 2, "version_minor": 0 }, @@ -120,7 +120,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Graph(number_of_nodes=22, number_of_edges=290, number_of_temporal_edges=3196, earliest_time=1560419400000, latest_time=1562756700000)\n" + "Graph(number_of_nodes=22, number_of_edges=290, number_of_temporal_edges=6392, earliest_time=1560419400000, latest_time=1562756700000)\n" ] } ], @@ -130,7 +130,7 @@ " src=\"Actor\",\n", " dst=\"Recipient\",\n", " time=\"DateTime\",\n", - " layer=\"Behavior\",\n", + " layer_col=\"Behavior\",\n", " properties=[\"Weight\"],\n", ")\n", "print(g)" @@ -149,7 +149,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -159,8 +159,8 @@ "Stats on the graph structure:\n", "Number of nodes (Baboons): 22\n", "Number of unique edges (src,dst,layer): 290\n", - "Total interactions (edge updates): 3196\n", - "Unique layers: ['_default', 'Grooming', 'Resting', 'Presenting', 'Playing with', 'Grunting-Lipsmacking', 'Supplanting', 'Threatening', 'Submission', 'Touching', 'Avoiding', 'Attacking', 'Carrying', 'Embracing', 'Mounting', 'Copulating', 'Chasing'] \n", + "Total interactions (edge updates): 6392\n", + "Unique layers: ['_default', 'Behavior', 'Grooming', 'Resting', 'Presenting', 'Playing with', 'Grunting-Lipsmacking', 'Supplanting', 'Threatening', 'Submission', 'Touching', 'Avoiding', 'Attacking', 'Carrying', 'Embracing', 'Mounting', 'Copulating', 'Chasing'] \n", "\n", "Stats on the graphs time range:\n", "Earliest datetime: 2019-06-13 09:50:00+00:00\n", @@ -213,7 +213,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -226,11 +226,11 @@ "\n", "Getting individual nodes and edges:\n", "Node(name=LOME, earliest_time=1560419520000, latest_time=1562756100000)\n", - "Edge(source=LOME, target=NEKKE, earliest_time=1560421080000, latest_time=1562755980000, properties={Weight: 1}) \n", + "Edge(source=LOME, target=NEKKE, earliest_time=1560421080000, latest_time=1562755980000, properties={Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1}) \n", "\n", "Getting iterators over all nodes and edges:\n", "[Node(name=ANGELE, earliest_time=1560419400000, latest_time=1562754600000), Node(name=FELIPE, earliest_time=1560419400000, latest_time=1562756700000), Node(name=LIPS, earliest_time=1560419460000, latest_time=1562756700000), Node(name=NEKKE, earliest_time=1560419520000, latest_time=1562756700000), Node(name=LOME, earliest_time=1560419520000, latest_time=1562756100000)]\n", - "[Edge(source=ANGELE, target=FELIPE, earliest_time=1560419400000, latest_time=1562753640000, properties={Weight: 1}), Edge(source=LOME, target=FEYA, earliest_time=1560421260000, latest_time=1562328420000, properties={Weight: 1}), Edge(source=VIOLETTE, target=LIPS, earliest_time=1560423600000, latest_time=1560423600000, properties={Weight: -1})]\n" + "[Edge(source=ANGELE, target=FELIPE, earliest_time=1560419400000, latest_time=1562753640000, properties={Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1}), Edge(source=FELIPE, target=ANGELE, earliest_time=1560419460000, latest_time=1562754600000, properties={Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1}), Edge(source=FELIPE, target=LIPS, earliest_time=1560419460000, latest_time=1562251080000, properties={Weight: 1, Weight: 1, Weight: 1})]\n" ] } ], @@ -263,7 +263,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -307,7 +307,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "metadata": { "scrolled": true }, @@ -318,7 +318,7 @@ "text": [ "FELIPE has 17 incoming interactions and 18 outgoing interactions.\n", "\n", - "[Edge(source=ANGELE, target=FELIPE, earliest_time=1560419400000, latest_time=1562753640000, properties={Weight: 1}), Edge(source=LIPS, target=FELIPE, earliest_time=1560423600000, latest_time=1562756700000, properties={Weight: 1}), Edge(source=NEKKE, target=FELIPE, earliest_time=1560443040000, latest_time=1562596380000, properties={Weight: 1})]\n", + "[Edge(source=ANGELE, target=FELIPE, earliest_time=1560419400000, latest_time=1562753640000, properties={Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1}), Edge(source=LIPS, target=FELIPE, earliest_time=1560423600000, latest_time=1562756700000, properties={Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1}), Edge(source=NEKKE, target=FELIPE, earliest_time=1560443040000, latest_time=1562596380000, properties={Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1, Weight: 1})]\n", "[Node(name=ANGELE, earliest_time=1560419400000, latest_time=1562754600000), Node(name=LIPS, earliest_time=1560419460000, latest_time=1562756700000), Node(name=NEKKE, earliest_time=1560419520000, latest_time=1562756700000)] \n", "\n", "FELIPE interacted with the following baboons ['ANGELE', 'LIPS', 'NEKKE', 'LOME', 'BOBO', 'ATMOSPHERE', 'FEYA', 'FANA', 'PIPO', 'MUSE', 'MAKO', 'MALI', 'PETOULETTE', 'ARIELLE', 'HARLEM', 'VIOLETTE', 'EWINE', 'SELF']\n" @@ -361,7 +361,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "metadata": { "scrolled": true }, @@ -371,18 +371,18 @@ "output_type": "stream", "text": [ "Update history per layer:\n", + "FELIPE interacted with MAKO with the following behaviour 'Behavior' at this times: [1560437400000, 1560437640000, 1560935460000, 1561043280000, 1561043280000, 1561043340000, 1561117620000, 1561373880000, 1561373880000, 1561373940000, 1561373940000, 1561373940000, 1561373940000, 1561373940000, 1561390860000, 1561390860000, 1561390860000, 1561390920000, 1561643580000, 1561717080000, 1561717140000, 1561970760000, 1562148960000, 1562148960000, 1562149020000, 1562149020000, 1562149080000, 1562671020000]\n", "FELIPE interacted with MAKO with the following behaviour 'Grooming' at this times: [1561043280000, 1561043340000]\n", "FELIPE interacted with MAKO with the following behaviour 'Resting' at this times: [1560437400000, 1560437640000, 1560935460000, 1561117620000, 1561373880000, 1561390860000, 1561390860000, 1561390860000, 1561643580000, 1561970760000, 1562149020000, 1562671020000]\n", "FELIPE interacted with MAKO with the following behaviour 'Playing with' at this times: [1561373880000, 1561373940000, 1561373940000, 1561390920000, 1562148960000, 1562148960000, 1562149080000]\n", "FELIPE interacted with MAKO with the following behaviour 'Grunting-Lipsmacking' at this times: [1561373940000, 1561717080000, 1561717140000]\n", - "FELIPE interacted with MAKO with the following behaviour 'Touching' at this times: [1562149020000]\n", "\n", "Individual updates as edges:\n", + "At 2019-06-13 14:50:00+00:00 FELIPE interacted with MAKO in the following manner: 'Behavior'\n", "At 2019-06-13 14:50:00+00:00 FELIPE interacted with MAKO in the following manner: 'Resting'\n", + "At 2019-06-13 14:54:00+00:00 FELIPE interacted with MAKO in the following manner: 'Behavior'\n", "At 2019-06-13 14:54:00+00:00 FELIPE interacted with MAKO in the following manner: 'Resting'\n", - "At 2019-06-19 09:11:00+00:00 FELIPE interacted with MAKO in the following manner: 'Resting'\n", - "At 2019-06-20 15:08:00+00:00 FELIPE interacted with MAKO in the following manner: 'Carrying'\n", - "At 2019-06-20 15:08:00+00:00 FELIPE interacted with MAKO in the following manner: 'Grooming'\n", + "At 2019-06-19 09:11:00+00:00 FELIPE interacted with MAKO in the following manner: 'Behavior'\n", "...\n", "\n", "Individual updates for 'Touching' and 'Carrying:\n", @@ -435,7 +435,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 12, "metadata": { "scrolled": true }, @@ -485,15 +485,15 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Felipe's favourite baboons in descending order are [('NEKKE', 41), ('ANGELE', 31), ('MAKO', 26), ('LOME', 23), ('LIPS', 11), ('HARLEM', 10), ('FANA', 8), ('MALI', 6), ('FEYA', 5), ('ARIELLE', 5), ('EWINE', 5), ('PIPO', 3), ('SELF', 2), ('BOBO', 1), ('ATMOSPHERE', 1), ('PETOULETTE', 1), ('VIOLETTE', 1), ('MUSE', -1)]\n", - "EXTERNE is the most annoying monkey with an average score of -2.0\n" + "Felipe's favourite baboons in descending order are [('NEKKE', 82), ('ANGELE', 62), ('MAKO', 52), ('LOME', 46), ('LIPS', 22), ('HARLEM', 20), ('FANA', 16), ('MALI', 12), ('FEYA', 10), ('ARIELLE', 10), ('EWINE', 10), ('PIPO', 6), ('SELF', 4), ('BOBO', 2), ('ATMOSPHERE', 2), ('PETOULETTE', 2), ('VIOLETTE', 2), ('MUSE', -2)]\n", + "EXTERNE is the most annoying monkey with an average score of -4.0\n" ] } ], @@ -547,16 +547,16 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Across the full dataset LOME interacted with NEKKE 41 times\n", - "Between None and 2019-06-13 12:17:19+00:00, LOME interacted with NEKKE 8 times\n", - "Window start: 2019-06-13 00:00:00+00:00, First update: 2019-06-13 10:18:00+00:00, Last update: 2019-06-13 15:05:00+00:00, Window End: 2019-06-14 00:00:00+00:00\n" + "Across the full dataset LOME interacted with NEKKE 82 times\n", + "Between None and 2019-06-13 12:17:19+00:00, LOME interacted with NEKKE 16 times\n", + "Window start: 2019-06-12 23:00:00+00:00, First update: 2019-06-13 10:18:00+00:00, Last update: 2019-06-13 15:05:00+00:00, Window End: 2019-06-13 23:00:00+00:00\n" ] } ], @@ -601,7 +601,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -646,14 +646,14 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Total weight across all edges is 2948.\n", + "Total weight across all edges is 5896.\n", "Total weight across Grooming and Resting is 1685.\n", "Total weight across Grooming and Resting between 2019-06-13 00:00:00 and 2019-06-20 00:00:00 is 403.\n" ] @@ -701,7 +701,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -759,7 +759,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 18, "metadata": {}, "outputs": [ { @@ -801,7 +801,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -848,7 +848,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 20, "metadata": {}, "outputs": [ { diff --git a/examples/rust/src/bin/bench/main.rs b/examples/rust/src/bin/bench/main.rs index 783c6fb065..d6cb8ee065 100644 --- a/examples/rust/src/bin/bench/main.rs +++ b/examples/rust/src/bin/bench/main.rs @@ -36,7 +36,7 @@ fn main() { println!("Loading data"); let graph = if encoded_data_dir.exists() { let now = Instant::now(); - let g = Graph::load_from_file(encoded_data_dir.as_path(), false) + let g = Graph::decode(encoded_data_dir.as_path()) .expect("Failed to load graph from encoded data files"); println!( @@ -74,8 +74,7 @@ fn main() { now.elapsed().as_secs() ); - g.save_to_file(encoded_data_dir) - .expect("Failed to save graph"); + g.encode(encoded_data_dir).expect("Failed to save graph"); g }; diff --git a/examples/rust/src/bin/btc/main.rs b/examples/rust/src/bin/btc/main.rs index 994e336ff9..1364085c42 100644 --- a/examples/rust/src/bin/btc/main.rs +++ b/examples/rust/src/bin/btc/main.rs @@ -62,7 +62,7 @@ fn main() { let graph = if encoded_data_dir.exists() { let now = Instant::now(); - let g = Graph::load_from_file(encoded_data_dir.as_path(), false) + let g = Graph::decode(encoded_data_dir.as_path()) .expect("Failed to load graph from encoded data files"); println!( @@ -109,8 +109,7 @@ fn main() { now.elapsed().as_secs() ); - g.save_to_file(encoded_data_dir) - .expect("Failed to save graph"); + g.encode(encoded_data_dir).expect("Failed to save graph"); g }; diff --git a/examples/rust/src/bin/hulongbay/main.rs b/examples/rust/src/bin/hulongbay/main.rs index 1b44ecffb1..677bcb2073 100644 --- a/examples/rust/src/bin/hulongbay/main.rs +++ b/examples/rust/src/bin/hulongbay/main.rs @@ -63,7 +63,7 @@ pub fn loader(data_dir: &Path) -> Result> { let encoded_data_dir = data_dir.join("graphdb.bincode"); if encoded_data_dir.exists() { let now = Instant::now(); - let g = Graph::load_from_file(encoded_data_dir.as_path(), false)?; + let g = Graph::decode(encoded_data_dir.as_path())?; println!( "Loaded graph from path {} with {} nodes, {} edges, took {} seconds", @@ -104,7 +104,7 @@ pub fn loader(data_dir: &Path) -> Result> { now.elapsed().as_secs() ); - g.save_to_file(encoded_data_dir)?; + g.encode(encoded_data_dir)?; Ok(g) } } diff --git a/examples/rust/src/bin/lotr/main.rs b/examples/rust/src/bin/lotr/main.rs index d96fe32443..acaab631f3 100644 --- a/examples/rust/src/bin/lotr/main.rs +++ b/examples/rust/src/bin/lotr/main.rs @@ -37,7 +37,7 @@ fn main() { let graph = if encoded_data_dir.exists() { let now = Instant::now(); - let g = Graph::load_from_file(encoded_data_dir.as_path(), false) + let g = Graph::decode(encoded_data_dir.as_path()) .expect("Failed to load graph from encoded data files"); println!( @@ -90,8 +90,7 @@ fn main() { now.elapsed().as_secs() ); - g.save_to_file(encoded_data_dir) - .expect("Failed to save graph"); + g.encode(encoded_data_dir).expect("Failed to save graph"); g }; diff --git a/examples/rust/src/bin/pokec/main.rs b/examples/rust/src/bin/pokec/main.rs index 2be1a7737d..00edba5e11 100644 --- a/examples/rust/src/bin/pokec/main.rs +++ b/examples/rust/src/bin/pokec/main.rs @@ -22,7 +22,7 @@ fn main() { let data_dir = Path::new(args.get(1).expect("No data directory provided")); let g = if std::path::Path::new("/tmp/pokec").exists() { - Graph::load_from_file("/tmp/pokec", false).unwrap() + Graph::decode("/tmp/pokec").unwrap() } else { let g = Graph::new(); CsvLoader::new(data_dir) @@ -34,7 +34,7 @@ fn main() { }) .expect("Failed to load graph from encoded data files"); - g.save_to_file("/tmp/pokec") + g.encode("/tmp/pokec") .expect("Failed to save graph to file"); g }; diff --git a/pometry-storage-private b/pometry-storage-private index 84e0e1d484..942230d446 160000 --- a/pometry-storage-private +++ b/pometry-storage-private @@ -1 +1 @@ -Subproject commit 84e0e1d48482588b14626b491a2e366479963a58 +Subproject commit 942230d44640241016f5780c348ee795bc633568 diff --git a/python/Cargo.toml b/python/Cargo.toml index 80db325dd1..1db0d0e81f 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -19,16 +19,8 @@ crate-type = ["cdylib"] [dependencies] pyo3 = { workspace = true } -raphtory_core = { path = "../raphtory", version = "0.10.0", features = ["python", "search", "vectors", "proto"], package = "raphtory" } -raphtory-graphql = { path = "../raphtory-graphql", version = "0.10.0",features = ["python"] } -serde_json = { workspace = true } -reqwest = { workspace = true, features = ["multipart"] } -tokio = { workspace = true } -crossbeam-channel = { workspace = true } -serde = { workspace = true } -async-graphql = { workspace = true } -dynamic-graphql = { workspace = true } -itertools = { workspace = true } +raphtory_core = { path = "../raphtory", version = "0.11.0", features = ["python", "search", "vectors", "proto"], package = "raphtory" } +raphtory-graphql = { path = "../raphtory-graphql", version = "0.11.0",features = ["python"] } [features] default = ["extension-module"] diff --git a/python/python/raphtory/__init__.pyi b/python/python/raphtory/__init__.pyi index e3384bcdf6..283bcb4315 100644 --- a/python/python/raphtory/__init__.pyi +++ b/python/python/raphtory/__init__.pyi @@ -8,10 +8,8 @@ ############################################################################### class AlgorithmResult: - def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def get(self, key): """ Returns the value corresponding to the provided key @@ -19,7 +17,6 @@ class AlgorithmResult: Arguments: key: The key of type `H` for which the value is to be retrieved. """ - def get_all(self): """ Returns a Dict containing all the nodes (as keys) and their corresponding values (values) or none. @@ -27,10 +24,8 @@ class AlgorithmResult: Returns: A dict of nodes and their values """ - def get_all_values(self): """Returns a a list of all values""" - def get_all_with_names(self): """ Returns a dict with node names and values @@ -38,7 +33,6 @@ class AlgorithmResult: Returns: a dict with node names and values """ - def group_by(self): """ Groups the `AlgorithmResult` by its values. @@ -47,16 +41,12 @@ class AlgorithmResult: A `HashMap` where keys are unique values from the `AlgorithmResult` and values are vectors containing keys of type `H` that share the same value. """ - def max(self): """Returns a tuple of the max result with its key""" - def median(self): """Returns a tuple of the median result with its key""" - def min(self): """Returns a tuple of the min result with its key""" - def sort_by_node(self, reverse=True): """ Sorts by node id in ascending or descending order. @@ -67,7 +57,6 @@ class AlgorithmResult: Returns: A sorted list of tuples containing node names and values. """ - def sort_by_node_name(self, reverse=True): """ The function `sort_by_node_name` sorts a vector of tuples containing a node and an optional @@ -81,7 +70,6 @@ class AlgorithmResult: Returns: The function sort_by_node_name returns a vector of tuples. Each tuple contains a Node and value """ - def sort_by_value(self, reverse=True): """ Sorts the `AlgorithmResult` by its values in ascending or descending order. @@ -92,7 +80,6 @@ class AlgorithmResult: Returns: A sorted vector of tuples containing keys of type `H` and values of type `Y`. """ - def to_df(self): """ Creates a dataframe from the result @@ -100,10 +87,8 @@ class AlgorithmResult: Returns: A `pandas.DataFrame` containing the result """ - def to_string(self): """Returns a formatted string representation of the algorithm.""" - def top_k(self, k, percentage=False, reverse=True): """ Retrieves the top-k elements from the `AlgorithmResult` based on its values. @@ -125,14 +110,12 @@ class ConstProperties: def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def as_dict(self): """ as_dict() -> dict[str, Any] convert the properties view to a python dict """ - def get(self, key): """ get(key: str) -> Any | None @@ -142,21 +125,18 @@ class ConstProperties: get property value by key (returns `None` if key does not exist) """ - def items(self): """ items() -> list[tuple[str, Any]] lists the property keys together with the corresponding value """ - def keys(self): """ keys() -> list[str] lists the available property keys """ - def values(self): """ values() -> list[Any] @@ -164,6 +144,34 @@ class ConstProperties: lists the property values """ +class DiskGraphStorage: + def __init__(self): + """Initialize self. See help(type(self)) for accurate signature.""" + def graph_dir(self): ... + @staticmethod + def load_from_dir(graph_dir): ... + @staticmethod + def load_from_pandas(graph_dir, edge_df, time_col, src_col, dst_col): ... + @staticmethod + def load_from_parquets( + graph_dir, + layer_parquet_cols, + node_properties, + chunk_size, + t_props_chunk_size, + read_chunk_size, + concurrent_files, + num_threads, + node_type_col, + ): ... + def merge_by_sorted_gids(self, other, graph_dir): + """ + Merge this graph with another `DiskGraph`. Note that both graphs should have nodes that are + sorted by their global ids or the resulting graph will be nonsense! + """ + def to_events(self): ... + def to_persistent(self): ... + class Edge: """ PyEdge is a Python class that represents an edge in the graph. @@ -172,7 +180,6 @@ class Edge: def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def after(self, start): """ Create a view of the Edge including all events after `start` (exclusive). @@ -183,7 +190,6 @@ class Edge: Returns: A Edge object. """ - def at(self, time): """ Create a view of the Edge including all events at `time`. @@ -194,7 +200,6 @@ class Edge: Returns: A Edge object. """ - def before(self, end): """ Create a view of the Edge including all events before `end` (exclusive). @@ -205,7 +210,6 @@ class Edge: Returns: A Edge object. """ - @property def date_time(self): """ @@ -214,14 +218,12 @@ class Edge: Returns: (datetime) the datetime of an exploded edge """ - def default_layer(self): """ Return a view of Edge containing only the default edge layer Returns: Edge: The layered view """ - def deletions(self): """ Returns a list of timestamps of when an edge is deleted @@ -229,7 +231,6 @@ class Edge: Returns: A list of unix timestamps """ - def deletions_data_time(self): """ Returns a list of timestamps of when an edge is deleted @@ -237,11 +238,9 @@ class Edge: Returns: A list of DateTime objects """ - @property def dst(self): """Returns the destination node of the edge.""" - @property def earliest_date_time(self): """ @@ -250,7 +249,6 @@ class Edge: Returns: the earliest datetime of an edge """ - @property def earliest_time(self): """ @@ -259,7 +257,6 @@ class Edge: Returns: (int) The earliest time of an edge """ - @property def end(self): """ @@ -268,7 +265,6 @@ class Edge: Returns: The latest time that this Edge is valid or None if the Edge is valid for all times. """ - @property def end_date_time(self): """ @@ -277,7 +273,6 @@ class Edge: Returns: The latest datetime that this Edge is valid or None if the Edge is valid for all times. """ - def exclude_layer(self, name): """ Return a view of Edge containing all layers except the excluded `name` @@ -289,7 +284,6 @@ class Edge: Returns: Edge: The layered view """ - def exclude_layers(self, names): """ Return a view of Edge containing all layers except the excluded `names` @@ -301,7 +295,6 @@ class Edge: Returns: Edge: The layered view """ - def exclude_valid_layer(self, name): """ Return a view of Edge containing all layers except the excluded `name` @@ -311,7 +304,6 @@ class Edge: Returns: Edge: The layered view """ - def exclude_valid_layers(self, names): """ Return a view of Edge containing all layers except the excluded `names` @@ -321,7 +313,6 @@ class Edge: Returns: Edge: The layered view """ - def expanding(self, step): """ Creates a `WindowSet` with the given `step` size using an expanding window. @@ -334,16 +325,11 @@ class Edge: Returns: A `WindowSet` object. """ - def explode(self): """Explodes an edge and returns all instances it had been updated as seperate edges""" - - def explode_layers(self): - ... - + def explode_layers(self): ... def has_layer(self, name): - """ Check if Edge has the layer `"name"`""" - + """Check if Edge has the layer `"name"`""" def history(self): """ Returns a list of timestamps of when an edge is added or change to an edge is made. @@ -352,7 +338,6 @@ class Edge: A list of unix timestamps. """ - def history_date_time(self): """ Returns a list of timestamps of when an edge is added or change to an edge is made. @@ -361,20 +346,15 @@ class Edge: A list of timestamps. """ - @property def id(self): """The id of the edge.""" - def is_deleted(self): """Check if the edge is currently deleted""" - def is_self_loop(self): """Check if the edge is on the same node""" - def is_valid(self): """Check if the edge is currently valid (i.e., not deleted)""" - @property def latest_date_time(self): """ @@ -383,7 +363,6 @@ class Edge: Returns: (datetime) the latest datetime of an edge """ - @property def latest_time(self): """ @@ -392,7 +371,6 @@ class Edge: Returns: (int) The latest time of an edge """ - def layer(self, name): """ Return a view of Edge containing the layer `"name"` @@ -401,7 +379,6 @@ class Edge: Returns: Edge: The layered view """ - @property def layer_name(self): """ @@ -410,7 +387,6 @@ class Edge: Returns: (List) The name of the layer """ - @property def layer_names(self): """ @@ -419,7 +395,6 @@ class Edge: Returns: (List) The name of the layer """ - def layers(self, names): """ Return a view of Edge containing all layers `names` @@ -431,11 +406,9 @@ class Edge: Returns: Edge: The layered view """ - @property def nbr(self): """Returns the node at the other end of the edge (same as `dst()` for out-edges and `src()` for in-edges)""" - @property def properties(self): """ @@ -444,7 +417,6 @@ class Edge: Returns: Properties on the Edge. """ - def rolling(self, window, step=None): """ Creates a `WindowSet` with the given `window` size and optional `step` using a rolling window. @@ -458,7 +430,6 @@ class Edge: Returns: A `WindowSet` object. """ - def shrink_end(self, end): """ Set the end of the window to the smaller of `end` and `self.end()` @@ -468,7 +439,6 @@ class Edge: Returns: A Edge object. """ - def shrink_start(self, start): """ Set the start of the window to the larger of `start` and `self.start()` @@ -479,7 +449,6 @@ class Edge: Returns: A Edge object. """ - def shrink_window(self, start, end): """ Shrink both the start and end of the window (same as calling `shrink_start` followed by `shrink_end` but more efficient) @@ -487,11 +456,9 @@ class Edge: Arguments: """ - @property def src(self): """Returns the source node of the edge.""" - @property def start(self): """ @@ -500,7 +467,6 @@ class Edge: Returns: The earliest time that this Edge is valid or None if the Edge is valid for all times. """ - @property def start_date_time(self): """ @@ -509,7 +475,6 @@ class Edge: Returns: The earliest datetime that this Edge is valid or None if the Edge is valid for all times. """ - @property def time(self): """ @@ -518,7 +483,6 @@ class Edge: Returns: (int) The time of an exploded edge """ - def valid_layers(self, names): """ Return a view of Edge containing all layers `names` @@ -530,7 +494,6 @@ class Edge: Returns: Edge: The layered view """ - def window(self, start, end): """ Create a view of the Edge including all events between `start` (inclusive) and `end` (exclusive) @@ -542,17 +505,15 @@ class Edge: Returns: r A Edge object. """ - @property def window_size(self): - """ Get the window size (difference between start and end) for this Edge""" + """Get the window size (difference between start and end) for this Edge""" class Edges: """A list of edges that can be iterated over.""" def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def after(self, start): """ Create a view of the Edges including all events after `start` (exclusive). @@ -563,7 +524,6 @@ class Edges: Returns: A Edges object. """ - def at(self, time): """ Create a view of the Edges including all events at `time`. @@ -574,7 +534,6 @@ class Edges: Returns: A Edges object. """ - def before(self, end): """ Create a view of the Edges including all events before `end` (exclusive). @@ -585,7 +544,6 @@ class Edges: Returns: A Edges object. """ - def collect(self): """ Collect all edges into a list @@ -593,10 +551,8 @@ class Edges: Returns: list[Edge]: the list of edges """ - def count(self): """Returns the number of edges""" - @property def date_time(self): """ @@ -605,14 +561,12 @@ class Edges: Returns: A list of date times. """ - def default_layer(self): """ Return a view of Edges containing only the default edge layer Returns: Edges: The layered view """ - def deletions(self): """ Returns all timestamps of edges where an edge is deleted @@ -620,7 +574,6 @@ class Edges: Returns: A list of lists of unix timestamps """ - def deletions_date_time(self): """ Returns all timestamps of edges where an edge is deleted @@ -628,11 +581,9 @@ class Edges: Returns: A list of lists of DateTime objects """ - @property def dst(self): """Returns the destination node of the edge.""" - @property def earliest_date_time(self): """ @@ -641,7 +592,6 @@ class Edges: Returns: Earliest date time of the edges. """ - @property def earliest_time(self): """ @@ -650,7 +600,6 @@ class Edges: Returns: Earliest time of the edges. """ - @property def end(self): """ @@ -659,7 +608,6 @@ class Edges: Returns: The latest time that this Edges is valid or None if the Edges is valid for all times. """ - @property def end_date_time(self): """ @@ -668,7 +616,6 @@ class Edges: Returns: The latest datetime that this Edges is valid or None if the Edges is valid for all times. """ - def exclude_layer(self, name): """ Return a view of Edges containing all layers except the excluded `name` @@ -680,7 +627,6 @@ class Edges: Returns: Edges: The layered view """ - def exclude_layers(self, names): """ Return a view of Edges containing all layers except the excluded `names` @@ -692,7 +638,6 @@ class Edges: Returns: Edges: The layered view """ - def exclude_valid_layer(self, name): """ Return a view of Edges containing all layers except the excluded `name` @@ -702,7 +647,6 @@ class Edges: Returns: Edges: The layered view """ - def exclude_valid_layers(self, names): """ Return a view of Edges containing all layers except the excluded `names` @@ -712,7 +656,6 @@ class Edges: Returns: Edges: The layered view """ - def expanding(self, step): """ Creates a `WindowSet` with the given `step` size using an expanding window. @@ -725,16 +668,11 @@ class Edges: Returns: A `WindowSet` object. """ - def explode(self): """Explodes an edge and returns all instances it had been updated as seperate edges""" - - def explode_layers(self): - ... - + def explode_layers(self): ... def has_layer(self, name): - """ Check if Edges has the layer `"name"`""" - + """Check if Edges has the layer `"name"`""" def history(self): """ Returns all timestamps of edges, when an edge is added or change to an edge is made. @@ -743,7 +681,6 @@ class Edges: A list of lists unix timestamps. """ - def history_date_time(self): """ Returns all timestamps of edges, when an edge is added or change to an edge is made. @@ -752,20 +689,15 @@ class Edges: A list of lists of timestamps. """ - @property def id(self): """Returns all ids of the edges.""" - def is_deleted(self): """Check if the edges are deleted""" - def is_self_loop(self): """Check if the edges are on the same node""" - def is_valid(self): """Check if the edges are valid (i.e. not deleted)""" - @property def latest_date_time(self): """ @@ -774,7 +706,6 @@ class Edges: Returns: Latest date time of the edges. """ - @property def latest_time(self): """ @@ -783,7 +714,6 @@ class Edges: Returns: Latest time of the edges. """ - def layer(self, name): """ Return a view of Edges containing the layer `"name"` @@ -792,7 +722,6 @@ class Edges: Returns: Edges: The layered view """ - @property def layer_name(self): """ @@ -801,7 +730,6 @@ class Edges: Returns: The name of the layer """ - @property def layer_names(self): """ @@ -810,7 +738,6 @@ class Edges: Returns: A list of layer names """ - def layers(self, names): """ Return a view of Edges containing all layers `names` @@ -822,15 +749,12 @@ class Edges: Returns: Edges: The layered view """ - @property def nbr(self): """Returns the node at the other end of the edge (same as `dst()` for out-edges and `src()` for in-edges)""" - @property def properties(self): """Returns all properties of the edges""" - def rolling(self, window, step=None): """ Creates a `WindowSet` with the given `window` size and optional `step` using a rolling window. @@ -844,7 +768,6 @@ class Edges: Returns: A `WindowSet` object. """ - def shrink_end(self, end): """ Set the end of the window to the smaller of `end` and `self.end()` @@ -854,7 +777,6 @@ class Edges: Returns: A Edges object. """ - def shrink_start(self, start): """ Set the start of the window to the larger of `start` and `self.start()` @@ -865,7 +787,6 @@ class Edges: Returns: A Edges object. """ - def shrink_window(self, start, end): """ Shrink both the start and end of the window (same as calling `shrink_start` followed by `shrink_end` but more efficient) @@ -873,11 +794,9 @@ class Edges: Arguments: """ - @property def src(self): """Returns the source node of the edge.""" - @property def start(self): """ @@ -886,7 +805,6 @@ class Edges: Returns: The earliest time that this Edges is valid or None if the Edges is valid for all times. """ - @property def start_date_time(self): """ @@ -895,7 +813,6 @@ class Edges: Returns: The earliest datetime that this Edges is valid or None if the Edges is valid for all times. """ - @property def time(self): """ @@ -904,8 +821,9 @@ class Edges: Returns: Time of edge """ - - def to_df(self, include_property_history=True, convert_datetime=False, explode=False): + def to_df( + self, include_property_history=True, convert_datetime=False, explode=False + ): """ Converts the graph's edges into a Pandas DataFrame. @@ -924,7 +842,6 @@ class Edges: Returns: If successful, this PyObject will be a Pandas DataFrame. """ - def valid_layers(self, names): """ Return a view of Edges containing all layers `names` @@ -936,7 +853,6 @@ class Edges: Returns: Edges: The layered view """ - def window(self, start, end): """ Create a view of the Edges including all events between `start` (inclusive) and `end` (exclusive) @@ -948,17 +864,15 @@ class Edges: Returns: r A Edges object. """ - @property def window_size(self): - """ Get the window size (difference between start and end) for this Edges""" + """Get the window size (difference between start and end) for this Edges""" class Graph: """A temporal graph.""" def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def add_constant_properties(self, properties): """ Adds static properties to the graph. @@ -969,7 +883,6 @@ class Graph: Returns: None """ - def add_edge(self, timestamp, src, dst, properties=None, layer=None): """ Adds a new edge with the given source and destination nodes and properties to the graph. @@ -984,7 +897,6 @@ class Graph: Returns: None """ - def add_node(self, timestamp, id, properties=None, node_type=None): """ Adds a new node with the given id and properties to the graph. @@ -997,7 +909,6 @@ class Graph: Returns: None """ - def add_property(self, timestamp, properties): """ Adds properties to the graph. @@ -1009,7 +920,6 @@ class Graph: Returns: None """ - def after(self, start): """ Create a view of the GraphView including all events after `start` (exclusive). @@ -1020,7 +930,6 @@ class Graph: Returns: A GraphView object. """ - def at(self, time): """ Create a view of the GraphView including all events at `time`. @@ -1031,7 +940,6 @@ class Graph: Returns: A GraphView object. """ - def before(self, end): """ Create a view of the GraphView including all events before `end` (exclusive). @@ -1042,10 +950,16 @@ class Graph: Returns: A GraphView object. """ + def cache(self, path): + """ + Write Graph to cache file and initialise the cache. - def bincode(self): - """Get bincode encoded graph""" + Future updates are tracked. Use `write_updates` to persist them to the + cache file. If the file already exists its contents are overwritten. + Arguments: + path (str): The path to the cache file + """ def count_edges(self): """ Number of edges in the graph @@ -1053,7 +967,6 @@ class Graph: Returns: the number of edges in the graph """ - def count_nodes(self): """ Number of nodes in the graph @@ -1061,7 +974,6 @@ class Graph: Returns: the number of nodes in the graph """ - def count_temporal_edges(self): """ Number of edges in the graph @@ -1069,14 +981,23 @@ class Graph: Returns: the number of temporal edges in the graph """ - def default_layer(self): """ Return a view of GraphView containing only the default edge layer Returns: GraphView: The layered view """ + @staticmethod + def deserialise(bytes): + """ + Load Graph from serialised bytes. + + Arguments: + bytes (Bytes): The serialised bytes to decode + Returns: + Graph + """ @property def earliest_date_time(self): """ @@ -1085,7 +1006,6 @@ class Graph: Returns: the datetime of the earliest activity in the graph """ - @property def earliest_time(self): """ @@ -1094,7 +1014,6 @@ class Graph: Returns: the timestamp of the earliest activity in the graph """ - def edge(self, src, dst): """ Gets the edge with the specified source and destination nodes @@ -1106,7 +1025,6 @@ class Graph: Returns: the edge with the specified source and destination nodes, or None if the edge does not exist """ - @property def edges(self): """ @@ -1115,7 +1033,6 @@ class Graph: Returns: the edges in the graph """ - @property def end(self): """ @@ -1124,7 +1041,6 @@ class Graph: Returns: The latest time that this GraphView is valid or None if the GraphView is valid for all times. """ - @property def end_date_time(self): """ @@ -1133,7 +1049,6 @@ class Graph: Returns: The latest datetime that this GraphView is valid or None if the GraphView is valid for all times. """ - def exclude_layer(self, name): """ Return a view of GraphView containing all layers except the excluded `name` @@ -1145,7 +1060,6 @@ class Graph: Returns: GraphView: The layered view """ - def exclude_layers(self, names): """ Return a view of GraphView containing all layers except the excluded `names` @@ -1157,7 +1071,6 @@ class Graph: Returns: GraphView: The layered view """ - def exclude_nodes(self, nodes): """ Returns a subgraph given a set of nodes that are excluded from the subgraph @@ -1168,7 +1081,6 @@ class Graph: Returns: GraphView - Returns the subgraph """ - def exclude_valid_layer(self, name): """ Return a view of GraphView containing all layers except the excluded `name` @@ -1178,7 +1090,6 @@ class Graph: Returns: GraphView: The layered view """ - def exclude_valid_layers(self, names): """ Return a view of GraphView containing all layers except the excluded `names` @@ -1188,7 +1099,6 @@ class Graph: Returns: GraphView: The layered view """ - def expanding(self, step): """ Creates a `WindowSet` with the given `step` size using an expanding window. @@ -1201,7 +1111,6 @@ class Graph: Returns: A `WindowSet` object. """ - def find_edges(self, properties_dict): """ Get the edges that match the properties name and value @@ -1210,7 +1119,6 @@ class Graph: Returns: the edges that match the properties name and value """ - def find_nodes(self, properties_dict): """ Get the nodes that match the properties name and value @@ -1219,7 +1127,6 @@ class Graph: Returns: the nodes that match the properties name and value """ - def get_all_node_types(self): """ Returns all the node types in the graph. @@ -1227,7 +1134,6 @@ class Graph: Returns: A list of node types """ - def has_edge(self, src, dst): """ Returns true if the graph contains the specified edge @@ -1239,10 +1145,8 @@ class Graph: Returns: true if the graph contains the specified edge, false otherwise """ - def has_layer(self, name): - """ Check if GraphView has the layer `"name"`""" - + """Check if GraphView has the layer `"name"`""" def has_node(self, id): """ Returns true if the graph contains the specified node @@ -1253,7 +1157,6 @@ class Graph: Returns: true if the graph contains the specified node, false otherwise """ - def import_edge(self, edge, force=False): """ Import a single edge into the graph. @@ -1269,7 +1172,6 @@ class Graph: Returns: Result, GraphError> - A Result object which is Ok if the edge was successfully imported, and Err otherwise. """ - def import_edges(self, edges, force=False): """ Import multiple edges into the graph. @@ -1282,10 +1184,7 @@ class Graph: edges (List(edges)) - A vector of PyEdge objects representing the edges to be imported. force (boolean) - An optional boolean flag indicating whether to force the import of the edges. - Returns: - Result), GraphError> - A Result object which is Ok if the edges were successfully imported, and Err otherwise. """ - def import_node(self, node, force=False): """ Import a single node into the graph. @@ -1300,7 +1199,6 @@ class Graph: Returns: Result, GraphError> - A Result object which is Ok if the node was successfully imported, and Err otherwise. """ - def import_nodes(self, nodes, force=False): """ Import multiple nodes into the graph. @@ -1313,10 +1211,7 @@ class Graph: nodes (List(Node))- A vector of PyNode objects representing the nodes to be imported. force (boolean) - An optional boolean flag indicating whether to force the import of the nodes. - Returns: - Result), GraphError> - A Result object which is Ok if the nodes were successfully imported, and Err otherwise. """ - def index(self): """ Indexes all node and edge properties. @@ -1326,7 +1221,6 @@ class Graph: Returns: GraphIndex - Returns a GraphIndex """ - def largest_connected_component(self): """ Gives the large connected component of a graph. @@ -1338,7 +1232,6 @@ class Graph: A raphtory graph, which essentially is a sub-graph of the graph `g` """ - @property def latest_date_time(self): """ @@ -1347,7 +1240,6 @@ class Graph: Returns: the datetime of the latest activity in the graph """ - @property def latest_time(self): """ @@ -1356,7 +1248,6 @@ class Graph: Returns: the timestamp of the latest activity in the graph """ - def layer(self, name): """ Return a view of GraphView containing the layer `"name"` @@ -1365,7 +1256,6 @@ class Graph: Returns: GraphView: The layered view """ - def layers(self, names): """ Return a view of GraphView containing all layers `names` @@ -1377,8 +1267,30 @@ class Graph: Returns: GraphView: The layered view """ + @staticmethod + def load_cached(path): + """ + Load Graph from a file and initialise it as a cache file. + + Future updates are tracked. Use `write_updates` to persist them to the + cache file. + + Arguments: + path (str): The path to the cache file - def load_edge_props_from_pandas(self, df, src, dst, const_properties=None, shared_const_properties=None, layer=None, layer_in_df=True): + Returns: + Graph + """ + def load_edge_props_from_pandas( + self, + df, + src, + dst, + constant_properties=None, + shared_constant_properties=None, + layer=None, + layer_col=None, + ): """ Load edge properties from a Pandas DataFrame. @@ -1386,16 +1298,27 @@ class Graph: df (Dataframe): The Pandas DataFrame containing edge information. src (str): The column name for the source node. dst (str): The column name for the destination node. - const_properties (List): List of constant edge property column names. Defaults to None. (optional) - shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - layer (str): Layer name. Defaults to None. (optional) - layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the data frame or if it should be used directly as the layer for all edges (optional) defaults to True. + constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + layer (str): The edge layer name (optional) Defaults to None. + layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. Returns: - Result<(), GraphError>: Result of the operation. - """ + None: If the operation is successful. - def load_edge_props_from_parquet(self, parquet_path, src, dst, const_properties=None, shared_const_properties=None, layer=None, layer_in_df=True): + Raises: + GraphError: If the operation fails. + """ + def load_edge_props_from_parquet( + self, + parquet_path, + src, + dst, + constant_properties=None, + shared_constant_properties=None, + layer=None, + layer_col=None, + ): """ Load edge properties from parquet file @@ -1403,183 +1326,200 @@ class Graph: parquet_path (str): Parquet file or directory of Parquet files path containing edge information. src (str): The column name for the source node. dst (str): The column name for the destination node. - const_properties (List): List of constant edge property column names. Defaults to None. (optional) - shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - layer (str): Layer name. Defaults to None. (optional) - layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the data frame or if it should be used directly as the layer for all edges (optional) defaults to True. + constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + layer (str): The edge layer name (optional) Defaults to None. + layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. Returns: - Result<(), GraphError>: Result of the operation. - """ + None: If the operation is successful. - def load_edges_from_pandas(self, df, src, dst, time, properties=None, const_properties=None, shared_const_properties=None, layer=None, layer_in_df=True): + Raises: + GraphError: If the operation fails. + """ + def load_edges_from_pandas( + self, + df, + time, + src, + dst, + properties=None, + constant_properties=None, + shared_constant_properties=None, + layer=None, + layer_col=None, + ): """ Load edges from a Pandas DataFrame into the graph. Arguments: df (Dataframe): The Pandas DataFrame containing the edges. + time (str): The column name for the update timestamps. src (str): The column name for the source node ids. dst (str): The column name for the destination node ids. - time (str): The column name for the update timestamps. - properties (List): List of edge property column names. Defaults to None. (optional) - const_properties (List): List of constant edge property column names. Defaults to None. (optional) - shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - layer (str): The edge layer name (optional) Defaults to None. - layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the dateframe or if it should be used directly as the layer for all edges (optional) defaults to True. - - Returns: - Result<(), GraphError>: Result of the operation. - """ - - def load_edges_from_parquet(self, parquet_path, src, dst, time, properties=None, const_properties=None, shared_const_properties=None, layer=None, layer_in_df=True): + properties (List[str]): List of edge property column names. Defaults to None. (optional) + constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + layer (str): A constant value to use as the layer for all edges (optional) Defaults to None. (cannot be used in combination with layer_col) + layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. (cannot be used in combination with layer) + Returns: + None: If the operation is successful. + + Raises: + GraphError: If the operation fails. + """ + def load_edges_from_parquet( + self, + parquet_path, + time, + src, + dst, + properties=None, + constant_properties=None, + shared_constant_properties=None, + layer=None, + layer_col=None, + ): """ Load edges from a Parquet file into the graph. Arguments: parquet_path (str): Parquet file or directory of Parquet files path containing edges + time (str): The column name for the update timestamps. src (str): The column name for the source node ids. dst (str): The column name for the destination node ids. - time (str): The column name for the update timestamps. - properties (List): List of edge property column names. Defaults to None. (optional) - const_properties (List): List of constant edge property column names. Defaults to None. (optional) - shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - layer (str): The edge layer name (optional) Defaults to None. - layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the dataframe or if it should be used directly as the layer for all edges (optional) defaults to True. - + properties (List[str]): List of edge property column names. Defaults to None. (optional) + constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + layer (str): A constant value to use as the layer for all edges (optional) Defaults to None. (cannot be used in combination with layer_col) + layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. (cannot be used in combination with layer) Returns: - Result<(), GraphError>: Result of the operation. - """ + None: If the operation is successful. + Raises: + GraphError: If the operation fails. + """ @staticmethod - def load_from_file(path, force=False): + def load_from_file(path): """ - Loads a graph from the given path. + Load Graph from a file. Arguments: - path (str): The path to the graph. + path (str): The path to the file. Returns: - Graph: The loaded graph. - """ - - @staticmethod - def load_from_pandas(edge_df, edge_src, edge_dst, edge_time, edge_properties=None, edge_const_properties=None, edge_shared_const_properties=None, edge_layer=None, layer_in_df=True, node_df=None, node_id=None, node_time=None, node_properties=None, node_const_properties=None, node_shared_const_properties=None, node_type=None, node_type_in_df=True): - """ - Load a graph from a Pandas DataFrame. - - Args: - edge_df (pandas.DataFrame): The DataFrame containing the edges. - edge_src (str): The column name for the source node ids. - edge_dst (str): The column name for the destination node ids. - edge_time (str): The column name for the timestamps. - edge_properties (list): The column names for the temporal properties (optional) Defaults to None. - edge_const_properties (list): The column names for the constant properties (optional) Defaults to None. - edge_shared_const_properties (dict): A dictionary of constant properties that will be added to every edge (optional) Defaults to None. - edge_layer (str): The edge layer name (optional) Defaults to None. - layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the edge_df or if it should be used directly as the layer for all edges (optional) defaults to True. - node_df (pandas.DataFrame): The DataFrame containing the nodes (optional) Defaults to None. - node_id (str): The column name for the node ids (optional) Defaults to None. - node_time (str): The column name for the node timestamps (optional) Defaults to None. - node_properties (list): The column names for the node temporal properties (optional) Defaults to None. - node_const_properties (list): The column names for the node constant properties (optional) Defaults to None. - node_shared_const_properties (dict): A dictionary of constant properties that will be added to every node (optional) Defaults to None. - node_type (str): the column name for the node type - node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - - Returns: - Graph: The loaded Graph object. - """ - - @staticmethod - def load_from_parquet(edge_parquet_path, edge_src, edge_dst, edge_time, edge_properties=None, edge_const_properties=None, edge_shared_const_properties=None, edge_layer=None, layer_in_df=True, node_parquet_path=None, node_id=None, node_time=None, node_properties=None, node_const_properties=None, node_shared_const_properties=None, node_type=None, node_type_in_df=True): + Graph """ - Load a graph from Parquet file. - - Args: - edge_parquet_path (str): Parquet file or directory of Parquet files containing the edges. - edge_src (str): The column name for the source node ids. - edge_dst (str): The column name for the destination node ids. - edge_time (str): The column name for the timestamps. - edge_properties (list): The column names for the temporal properties (optional) Defaults to None. - edge_const_properties (list): The column names for the constant properties (optional) Defaults to None. - edge_shared_const_properties (dict): A dictionary of constant properties that will be added to every edge (optional) Defaults to None. - edge_layer (str): The edge layer name (optional) Defaults to None. - layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the edge_df or if it should be used directly as the layer for all edges (optional) defaults to True. - node_parquet_path (str): Parquet file or directory of Parquet files containing the nodes (optional) Defaults to None. - node_id (str): The column name for the node ids (optional) Defaults to None. - node_time (str): The column name for the node timestamps (optional) Defaults to None. - node_properties (list): The column names for the node temporal properties (optional) Defaults to None. - node_const_properties (list): The column names for the node constant properties (optional) Defaults to None. - node_shared_const_properties (dict): A dictionary of constant properties that will be added to every node (optional) Defaults to None. - node_type (str): the column name for the node type - node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - - Returns: - Graph: The loaded Graph object. - """ - - def load_node_props_from_pandas(self, df, id, const_properties=None, shared_const_properties=None): + def load_node_props_from_pandas( + self, + df, + id, + node_type=None, + node_type_col=None, + constant_properties=None, + shared_constant_properties=None, + ): """ Load node properties from a Pandas DataFrame. Arguments: df (Dataframe): The Pandas DataFrame containing node information. id(str): The column name for the node IDs. - const_properties (List): List of constant node property column names. Defaults to None. (optional) - shared_const_properties (>): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) Returns: - Result<(), GraphError>: Result of the operation. - """ + None: If the operation is successful. - def load_node_props_from_parquet(self, parquet_path, id, const_properties=None, shared_const_properties=None): + Raises: + GraphError: If the operation fails. + """ + def load_node_props_from_parquet( + self, + parquet_path, + id, + node_type=None, + node_type_col=None, + constant_properties=None, + shared_constant_properties=None, + ): """ Load node properties from a parquet file. Arguments: parquet_path (str): Parquet file or directory of Parquet files path containing node information. id(str): The column name for the node IDs. - const_properties (List): List of constant node property column names. Defaults to None. (optional) - shared_const_properties (>): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) Returns: - Result<(), GraphError>: Result of the operation. - """ + None: If the operation is successful. - def load_nodes_from_pandas(self, df, id, time, node_type=None, node_type_in_df=True, properties=None, const_properties=None, shared_const_properties=None): + Raises: + GraphError: If the operation fails. + """ + def load_nodes_from_pandas( + self, + df, + time, + id, + node_type=None, + node_type_col=None, + properties=None, + constant_properties=None, + shared_constant_properties=None, + ): """ Load nodes from a Pandas DataFrame into the graph. Arguments: df (pandas.DataFrame): The Pandas DataFrame containing the nodes. - id (str): The column name for the node IDs. time (str): The column name for the timestamps. - node_type (str): the column name for the node type - node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - properties (List): List of node property column names. Defaults to None. (optional) - const_properties (List): List of constant node property column names. Defaults to None. (optional) - shared_const_properties (Dictionary/Hashmap of properties): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) - Returns: - Result<(), GraphError>: Result of the operation. - """ - - def load_nodes_from_parquet(self, parquet_path, id, time, node_type=None, node_type_in_df=True, properties=None, const_properties=None, shared_const_properties=None): + id (str): The column name for the node IDs. + node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + properties (List[str]): List of node property column names. Defaults to None. (optional) + constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + Returns: + None: If the operation is successful. + + Raises: + GraphError: If the operation fails. + """ + def load_nodes_from_parquet( + self, + parquet_path, + time, + id, + node_type=None, + node_type_col=None, + properties=None, + constant_properties=None, + shared_constant_properties=None, + ): """ Load nodes from a Parquet file into the graph. Arguments: parquet_path (str): Parquet file or directory of Parquet files containing the nodes - id (str): The column name for the node IDs. time (str): The column name for the timestamps. - node_type (str): the column name for the node type - node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - properties (List): List of node property column names. Defaults to None. (optional) - const_properties (List): List of constant node property column names. Defaults to None. (optional) - shared_const_properties (Dictionary/Hashmap of properties): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + id (str): The column name for the node IDs. + node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + properties (List[str]): List of node property column names. Defaults to None. (optional) + constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) Returns: - Result<(), GraphError>: Result of the operation. - """ + None: If the operation is successful. + Raises: + GraphError: If the operation fails. + """ def materialize(self): """ Returns a 'materialized' clone of the graph view - i.e. a new graph with a copy of the data seen within the view instead of just a mask over the original graph @@ -1587,7 +1527,6 @@ class Graph: Returns: GraphView - Returns a graph clone """ - def node(self, id): """ Gets the node with the specified id @@ -1598,7 +1537,6 @@ class Graph: Returns: the node with the specified id, or None if the node does not exist """ - @property def nodes(self): """ @@ -1607,10 +1545,10 @@ class Graph: Returns: the nodes in the graph """ - + def persist_as_disk_graph(self, graph_dir): + """save graph in disk_graph format and memory map the result""" def persistent_graph(self): """Get persistent graph""" - @property def properties(self): """ @@ -1620,7 +1558,6 @@ class Graph: Returns: HashMap - Properties paired with their names """ - def rolling(self, window, step=None): """ Creates a `WindowSet` with the given `window` size and optional `step` using a rolling window. @@ -1634,18 +1571,20 @@ class Graph: Returns: A `WindowSet` object. """ - def save_to_file(self, path): """ - Saves the graph to the given path. + Saves the Graph to the given path. Arguments: - path (str): The path to the graph. + path (str): The path to the file. + """ + def serialise(self): + """ + Serialise Graph to bytes. Returns: - None + Bytes """ - def shrink_end(self, end): """ Set the end of the window to the smaller of `end` and `self.end()` @@ -1655,7 +1594,6 @@ class Graph: Returns: A GraphView object. """ - def shrink_start(self, start): """ Set the start of the window to the larger of `start` and `self.start()` @@ -1666,7 +1604,6 @@ class Graph: Returns: A GraphView object. """ - def shrink_window(self, start, end): """ Shrink both the start and end of the window (same as calling `shrink_start` followed by `shrink_end` but more efficient) @@ -1674,7 +1611,6 @@ class Graph: Arguments: """ - @property def start(self): """ @@ -1683,7 +1619,6 @@ class Graph: Returns: The earliest time that this GraphView is valid or None if the GraphView is valid for all times. """ - @property def start_date_time(self): """ @@ -1692,7 +1627,6 @@ class Graph: Returns: The earliest datetime that this GraphView is valid or None if the GraphView is valid for all times. """ - def subgraph(self, nodes): """ Returns a subgraph given a set of nodes @@ -1703,7 +1637,6 @@ class Graph: Returns: GraphView - Returns the subgraph """ - def subgraph_node_types(self, node_types): """ Returns a subgraph filtered by node types given a set of node types @@ -1714,8 +1647,15 @@ class Graph: Returns: GraphView - Returns the subgraph """ - - def to_networkx(self, explode_edges=False, include_node_properties=True, include_edge_properties=True, include_update_history=True, include_property_history=True): + def to_disk_graph(self, graph_dir): ... + def to_networkx( + self, + explode_edges=False, + include_node_properties=True, + include_edge_properties=True, + include_update_history=True, + include_property_history=True, + ): """ Returns a graph with NetworkX. @@ -1733,8 +1673,18 @@ class Graph: Returns: A Networkx MultiDiGraph. """ - - def to_pyvis(self, explode_edges=False, edge_color="#000000", shape=None, node_image=None, edge_weight=None, edge_label=None, colour_nodes_by_type=False, notebook=False, **kwargs): + def to_pyvis( + self, + explode_edges=False, + edge_color="#000000", + shape=None, + node_image=None, + edge_weight=None, + edge_label=None, + colour_nodes_by_type=False, + notebook=False, + **kwargs, + ): """ Draw a graph with PyVis. Pyvis is a required dependency. If you intend to use this function make sure that you install Pyvis @@ -1758,11 +1708,9 @@ class Graph: Returns: A pyvis network """ - @property def unique_layers(self): """Return all the layer ids in the graph""" - def update_constant_properties(self, properties): """ Updates static properties to the graph. @@ -1773,7 +1721,6 @@ class Graph: Returns: None """ - def valid_layers(self, names): """ Return a view of GraphView containing all layers `names` @@ -1785,8 +1732,16 @@ class Graph: Returns: GraphView: The layered view """ - - def vectorise(self, embedding, cache=None, overwrite_cache=False, graph_document=None, node_document=None, edge_document=None, verbose=False): + def vectorise( + self, + embedding, + cache=None, + overwrite_cache=False, + graph_document=None, + node_document=None, + edge_document=None, + verbose=False, + ): """ Create a VectorisedGraph from the current graph @@ -1801,7 +1756,6 @@ class Graph: Returns: A VectorisedGraph with all the documents/embeddings computed and with an initial empty selection """ - def window(self, start, end): """ Create a view of the GraphView including all events between `start` (inclusive) and `end` (exclusive) @@ -1813,10 +1767,11 @@ class Graph: Returns: r A GraphView object. """ - @property def window_size(self): - """ Get the window size (difference between start and end) for this GraphView""" + """Get the window size (difference between start and end) for this GraphView""" + def write_updates(self): + """Persist the new updates by appending them to the cache file.""" class GraphIndex: """ @@ -1827,8 +1782,9 @@ class GraphIndex: def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - - def fuzzy_search_edges(self, query, limit=25, offset=0, prefix=False, levenshtein_distance=0): + def fuzzy_search_edges( + self, query, limit=25, offset=0, prefix=False, levenshtein_distance=0 + ): """ Searches for edges which match the given query. This uses Tantivy's fuzzy search. @@ -1842,8 +1798,9 @@ class GraphIndex: Returns: A list of edges which match the query. The list will be empty if no edges match the query. """ - - def fuzzy_search_nodes(self, query, limit=25, offset=0, prefix=False, levenshtein_distance=0): + def fuzzy_search_nodes( + self, query, limit=25, offset=0, prefix=False, levenshtein_distance=0 + ): """ Searches for nodes which match the given query. This uses Tantivy's fuzzy search. If you would like to better understand the query syntax, please visit our documentation at https://docs.raphtory.com @@ -1858,7 +1815,6 @@ class GraphIndex: Returns: A list of nodes which match the query. The list will be empty if no nodes match. """ - def search_edges(self, query, limit=25, offset=0): """ Searches for edges which match the given query. This uses Tantivy's exact search. @@ -1871,7 +1827,6 @@ class GraphIndex: Returns: A list of edges which match the query. The list will be empty if no edges match the query. """ - def search_nodes(self, query, limit=25, offset=0): """ Searches for nodes which match the given query. This uses Tantivy's exact search. @@ -1886,10 +1841,8 @@ class GraphIndex: """ class MutableEdge: - def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def add_constant_properties(self, properties, layer=None): """ Add constant properties to an edge in the graph. @@ -1905,7 +1858,6 @@ class MutableEdge: Returns: Result: A result object indicating success or failure. On failure, it contains a GraphError. """ - def add_updates(self, t, properties=None, layer=None): """ Add updates to an edge in the graph at a specified time. @@ -1920,7 +1872,6 @@ class MutableEdge: Returns: Result: A result object indicating success or failure. On failure, it contains a GraphError. """ - def after(self, start): """ Create a view of the Edge including all events after `start` (exclusive). @@ -1931,7 +1882,6 @@ class MutableEdge: Returns: A Edge object. """ - def at(self, time): """ Create a view of the Edge including all events at `time`. @@ -1942,7 +1892,6 @@ class MutableEdge: Returns: A Edge object. """ - def before(self, end): """ Create a view of the Edge including all events before `end` (exclusive). @@ -1953,7 +1902,6 @@ class MutableEdge: Returns: A Edge object. """ - @property def date_time(self): """ @@ -1962,14 +1910,12 @@ class MutableEdge: Returns: (datetime) the datetime of an exploded edge """ - def default_layer(self): """ Return a view of Edge containing only the default edge layer Returns: Edge: The layered view """ - def deletions(self): """ Returns a list of timestamps of when an edge is deleted @@ -1977,7 +1923,6 @@ class MutableEdge: Returns: A list of unix timestamps """ - def deletions_data_time(self): """ Returns a list of timestamps of when an edge is deleted @@ -1985,11 +1930,9 @@ class MutableEdge: Returns: A list of DateTime objects """ - @property def dst(self): """Returns the destination node of the edge.""" - @property def earliest_date_time(self): """ @@ -1998,7 +1941,6 @@ class MutableEdge: Returns: the earliest datetime of an edge """ - @property def earliest_time(self): """ @@ -2007,7 +1949,6 @@ class MutableEdge: Returns: (int) The earliest time of an edge """ - @property def end(self): """ @@ -2016,7 +1957,6 @@ class MutableEdge: Returns: The latest time that this Edge is valid or None if the Edge is valid for all times. """ - @property def end_date_time(self): """ @@ -2025,7 +1965,6 @@ class MutableEdge: Returns: The latest datetime that this Edge is valid or None if the Edge is valid for all times. """ - def exclude_layer(self, name): """ Return a view of Edge containing all layers except the excluded `name` @@ -2037,7 +1976,6 @@ class MutableEdge: Returns: Edge: The layered view """ - def exclude_layers(self, names): """ Return a view of Edge containing all layers except the excluded `names` @@ -2049,7 +1987,6 @@ class MutableEdge: Returns: Edge: The layered view """ - def exclude_valid_layer(self, name): """ Return a view of Edge containing all layers except the excluded `name` @@ -2059,7 +1996,6 @@ class MutableEdge: Returns: Edge: The layered view """ - def exclude_valid_layers(self, names): """ Return a view of Edge containing all layers except the excluded `names` @@ -2069,7 +2005,6 @@ class MutableEdge: Returns: Edge: The layered view """ - def expanding(self, step): """ Creates a `WindowSet` with the given `step` size using an expanding window. @@ -2082,16 +2017,11 @@ class MutableEdge: Returns: A `WindowSet` object. """ - def explode(self): """Explodes an edge and returns all instances it had been updated as seperate edges""" - - def explode_layers(self): - ... - + def explode_layers(self): ... def has_layer(self, name): - """ Check if Edge has the layer `"name"`""" - + """Check if Edge has the layer `"name"`""" def history(self): """ Returns a list of timestamps of when an edge is added or change to an edge is made. @@ -2100,7 +2030,6 @@ class MutableEdge: A list of unix timestamps. """ - def history_date_time(self): """ Returns a list of timestamps of when an edge is added or change to an edge is made. @@ -2109,20 +2038,15 @@ class MutableEdge: A list of timestamps. """ - @property def id(self): """The id of the edge.""" - def is_deleted(self): """Check if the edge is currently deleted""" - def is_self_loop(self): """Check if the edge is on the same node""" - def is_valid(self): """Check if the edge is currently valid (i.e., not deleted)""" - @property def latest_date_time(self): """ @@ -2131,7 +2055,6 @@ class MutableEdge: Returns: (datetime) the latest datetime of an edge """ - @property def latest_time(self): """ @@ -2140,7 +2063,6 @@ class MutableEdge: Returns: (int) The latest time of an edge """ - def layer(self, name): """ Return a view of Edge containing the layer `"name"` @@ -2149,7 +2071,6 @@ class MutableEdge: Returns: Edge: The layered view """ - @property def layer_name(self): """ @@ -2158,7 +2079,6 @@ class MutableEdge: Returns: (List) The name of the layer """ - @property def layer_names(self): """ @@ -2167,7 +2087,6 @@ class MutableEdge: Returns: (List) The name of the layer """ - def layers(self, names): """ Return a view of Edge containing all layers `names` @@ -2179,11 +2098,9 @@ class MutableEdge: Returns: Edge: The layered view """ - @property def nbr(self): """Returns the node at the other end of the edge (same as `dst()` for out-edges and `src()` for in-edges)""" - @property def properties(self): """ @@ -2192,7 +2109,6 @@ class MutableEdge: Returns: Properties on the Edge. """ - def rolling(self, window, step=None): """ Creates a `WindowSet` with the given `window` size and optional `step` using a rolling window. @@ -2206,7 +2122,6 @@ class MutableEdge: Returns: A `WindowSet` object. """ - def shrink_end(self, end): """ Set the end of the window to the smaller of `end` and `self.end()` @@ -2216,7 +2131,6 @@ class MutableEdge: Returns: A Edge object. """ - def shrink_start(self, start): """ Set the start of the window to the larger of `start` and `self.start()` @@ -2227,7 +2141,6 @@ class MutableEdge: Returns: A Edge object. """ - def shrink_window(self, start, end): """ Shrink both the start and end of the window (same as calling `shrink_start` followed by `shrink_end` but more efficient) @@ -2235,11 +2148,9 @@ class MutableEdge: Arguments: """ - @property def src(self): """Returns the source node of the edge.""" - @property def start(self): """ @@ -2248,7 +2159,6 @@ class MutableEdge: Returns: The earliest time that this Edge is valid or None if the Edge is valid for all times. """ - @property def start_date_time(self): """ @@ -2257,7 +2167,6 @@ class MutableEdge: Returns: The earliest datetime that this Edge is valid or None if the Edge is valid for all times. """ - @property def time(self): """ @@ -2266,7 +2175,6 @@ class MutableEdge: Returns: (int) The time of an exploded edge """ - def update_constant_properties(self, properties, layer=None): """ Update constant properties of an edge in the graph overwriting existing values. @@ -2282,7 +2190,6 @@ class MutableEdge: Returns: Result: A result object indicating success or failure. On failure, it contains a GraphError. """ - def valid_layers(self, names): """ Return a view of Edge containing all layers `names` @@ -2294,7 +2201,6 @@ class MutableEdge: Returns: Edge: The layered view """ - def window(self, start, end): """ Create a view of the Edge including all events between `start` (inclusive) and `end` (exclusive) @@ -2306,16 +2212,13 @@ class MutableEdge: Returns: r A Edge object. """ - @property def window_size(self): - """ Get the window size (difference between start and end) for this Edge""" + """Get the window size (difference between start and end) for this Edge""" class MutableNode: - def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def add_constant_properties(self, properties): """ Add constant properties to a node in the graph. @@ -2330,7 +2233,6 @@ class MutableNode: Returns: Result: A result object indicating success or failure. On failure, it contains a GraphError. """ - def add_updates(self, t, properties=None): """ Add updates to a node in the graph at a specified time. @@ -2345,7 +2247,6 @@ class MutableNode: Returns: Result: A result object indicating success or failure. On failure, it contains a GraphError. """ - def after(self, start): """ Create a view of the Node including all events after `start` (exclusive). @@ -2356,7 +2257,6 @@ class MutableNode: Returns: A Node object. """ - def at(self, time): """ Create a view of the Node including all events at `time`. @@ -2367,7 +2267,6 @@ class MutableNode: Returns: A Node object. """ - def before(self, end): """ Create a view of the Node including all events before `end` (exclusive). @@ -2378,14 +2277,12 @@ class MutableNode: Returns: A Node object. """ - def default_layer(self): """ Return a view of Node containing only the default edge layer Returns: Node: The layered view """ - def degree(self): """ Get the degree of this node (i.e., the number of edges that are incident to it). @@ -2393,7 +2290,6 @@ class MutableNode: Returns The degree of this node. """ - @property def earliest_date_time(self): """ @@ -2402,7 +2298,6 @@ class MutableNode: Returns: The earliest datetime that the node exists as an integer. """ - @property def earliest_time(self): """ @@ -2411,7 +2306,6 @@ class MutableNode: Returns: The earliest time that the node exists as an integer. """ - @property def edges(self): """ @@ -2421,7 +2315,6 @@ class MutableNode: An iterator over the edges that are incident to this node. """ - @property def end(self): """ @@ -2430,7 +2323,6 @@ class MutableNode: Returns: The latest time that this Node is valid or None if the Node is valid for all times. """ - @property def end_date_time(self): """ @@ -2439,7 +2331,6 @@ class MutableNode: Returns: The latest datetime that this Node is valid or None if the Node is valid for all times. """ - def exclude_layer(self, name): """ Return a view of Node containing all layers except the excluded `name` @@ -2451,7 +2342,6 @@ class MutableNode: Returns: Node: The layered view """ - def exclude_layers(self, names): """ Return a view of Node containing all layers except the excluded `names` @@ -2463,7 +2353,6 @@ class MutableNode: Returns: Node: The layered view """ - def exclude_valid_layer(self, name): """ Return a view of Node containing all layers except the excluded `name` @@ -2473,7 +2362,6 @@ class MutableNode: Returns: Node: The layered view """ - def exclude_valid_layers(self, names): """ Return a view of Node containing all layers except the excluded `names` @@ -2483,7 +2371,6 @@ class MutableNode: Returns: Node: The layered view """ - def expanding(self, step): """ Creates a `WindowSet` with the given `step` size using an expanding window. @@ -2496,10 +2383,8 @@ class MutableNode: Returns: A `WindowSet` object. """ - def has_layer(self, name): - """ Check if Node has the layer `"name"`""" - + """Check if Node has the layer `"name"`""" def history(self): """ Returns the history of a node, including node additions and changes made to node. @@ -2507,7 +2392,6 @@ class MutableNode: Returns: A list of unix timestamps of the event history of node. """ - def history_date_time(self): """ Returns the history of a node, including node additions and changes made to node. @@ -2516,7 +2400,6 @@ class MutableNode: A list of timestamps of the event history of node. """ - @property def id(self): """ @@ -2526,7 +2409,6 @@ class MutableNode: Returns: The id of the node as an integer. """ - def in_degree(self): """ Get the in-degree of this node (i.e., the number of edges that are incident to it from other nodes). @@ -2534,7 +2416,6 @@ class MutableNode: Returns: The in-degree of this node. """ - @property def in_edges(self): """ @@ -2544,7 +2425,6 @@ class MutableNode: An iterator over the edges that point into this node. """ - @property def in_neighbours(self): """ @@ -2554,7 +2434,6 @@ class MutableNode: An iterator over the neighbours of this node that point into this node. """ - @property def latest_date_time(self): """ @@ -2566,7 +2445,6 @@ class MutableNode: Returns: The latest datetime that the node exists as an integer. """ - @property def latest_time(self): """ @@ -2575,7 +2453,6 @@ class MutableNode: Returns: The latest time that the node exists as an integer. """ - def layer(self, name): """ Return a view of Node containing the layer `"name"` @@ -2584,7 +2461,6 @@ class MutableNode: Returns: Node: The layered view """ - def layers(self, names): """ Return a view of Node containing all layers `names` @@ -2596,7 +2472,6 @@ class MutableNode: Returns: Node: The layered view """ - @property def name(self): """ @@ -2605,7 +2480,6 @@ class MutableNode: Returns: The name of the node as a string. """ - @property def neighbours(self): """ @@ -2615,11 +2489,9 @@ class MutableNode: An iterator over the neighbours of this node. """ - @property def node_type(self): """Returns the type of node""" - def out_degree(self): """ Get the out-degree of this node (i.e., the number of edges that are incident to it from this node). @@ -2627,7 +2499,6 @@ class MutableNode: Returns: The out-degree of this node. """ - @property def out_edges(self): """ @@ -2637,7 +2508,6 @@ class MutableNode: An iterator over the edges that point out of this node. """ - @property def out_neighbours(self): """ @@ -2647,7 +2517,6 @@ class MutableNode: An iterator over the neighbours of this node that point out of this node. """ - @property def properties(self): """ @@ -2656,7 +2525,6 @@ class MutableNode: Returns: A list of properties. """ - def rolling(self, window, step=None): """ Creates a `WindowSet` with the given `window` size and optional `step` using a rolling window. @@ -2670,7 +2538,6 @@ class MutableNode: Returns: A `WindowSet` object. """ - def set_node_type(self, new_type): """ Set the type on the node. This only works if the type has not been previously set, otherwise will @@ -2682,7 +2549,6 @@ class MutableNode: Returns: Result: A result object indicating success or failure. On failure, it contains a GraphError. """ - def shrink_end(self, end): """ Set the end of the window to the smaller of `end` and `self.end()` @@ -2692,7 +2558,6 @@ class MutableNode: Returns: A Node object. """ - def shrink_start(self, start): """ Set the start of the window to the larger of `start` and `self.start()` @@ -2703,7 +2568,6 @@ class MutableNode: Returns: A Node object. """ - def shrink_window(self, start, end): """ Shrink both the start and end of the window (same as calling `shrink_start` followed by `shrink_end` but more efficient) @@ -2711,7 +2575,6 @@ class MutableNode: Arguments: """ - @property def start(self): """ @@ -2720,7 +2583,6 @@ class MutableNode: Returns: The earliest time that this Node is valid or None if the Node is valid for all times. """ - @property def start_date_time(self): """ @@ -2729,7 +2591,6 @@ class MutableNode: Returns: The earliest datetime that this Node is valid or None if the Node is valid for all times. """ - def update_constant_properties(self, properties): """ Update constant properties of a node in the graph overwriting existing values. @@ -2744,7 +2605,6 @@ class MutableNode: Returns: Result: A result object indicating success or failure. On failure, it contains a GraphError. """ - def valid_layers(self, names): """ Return a view of Node containing all layers `names` @@ -2756,7 +2616,6 @@ class MutableNode: Returns: Node: The layered view """ - def window(self, start, end): """ Create a view of the Node including all events between `start` (inclusive) and `end` (exclusive) @@ -2768,17 +2627,15 @@ class MutableNode: Returns: r A Node object. """ - @property def window_size(self): - """ Get the window size (difference between start and end) for this Node""" + """Get the window size (difference between start and end) for this Node""" class Node: """A node (or node) in the graph.""" def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def after(self, start): """ Create a view of the Node including all events after `start` (exclusive). @@ -2789,7 +2646,6 @@ class Node: Returns: A Node object. """ - def at(self, time): """ Create a view of the Node including all events at `time`. @@ -2800,7 +2656,6 @@ class Node: Returns: A Node object. """ - def before(self, end): """ Create a view of the Node including all events before `end` (exclusive). @@ -2811,14 +2666,12 @@ class Node: Returns: A Node object. """ - def default_layer(self): """ Return a view of Node containing only the default edge layer Returns: Node: The layered view """ - def degree(self): """ Get the degree of this node (i.e., the number of edges that are incident to it). @@ -2826,7 +2679,6 @@ class Node: Returns The degree of this node. """ - @property def earliest_date_time(self): """ @@ -2835,7 +2687,6 @@ class Node: Returns: The earliest datetime that the node exists as an integer. """ - @property def earliest_time(self): """ @@ -2844,7 +2695,6 @@ class Node: Returns: The earliest time that the node exists as an integer. """ - @property def edges(self): """ @@ -2854,7 +2704,6 @@ class Node: An iterator over the edges that are incident to this node. """ - @property def end(self): """ @@ -2863,7 +2712,6 @@ class Node: Returns: The latest time that this Node is valid or None if the Node is valid for all times. """ - @property def end_date_time(self): """ @@ -2872,7 +2720,6 @@ class Node: Returns: The latest datetime that this Node is valid or None if the Node is valid for all times. """ - def exclude_layer(self, name): """ Return a view of Node containing all layers except the excluded `name` @@ -2884,7 +2731,6 @@ class Node: Returns: Node: The layered view """ - def exclude_layers(self, names): """ Return a view of Node containing all layers except the excluded `names` @@ -2896,7 +2742,6 @@ class Node: Returns: Node: The layered view """ - def exclude_valid_layer(self, name): """ Return a view of Node containing all layers except the excluded `name` @@ -2906,7 +2751,6 @@ class Node: Returns: Node: The layered view """ - def exclude_valid_layers(self, names): """ Return a view of Node containing all layers except the excluded `names` @@ -2916,7 +2760,6 @@ class Node: Returns: Node: The layered view """ - def expanding(self, step): """ Creates a `WindowSet` with the given `step` size using an expanding window. @@ -2929,10 +2772,8 @@ class Node: Returns: A `WindowSet` object. """ - def has_layer(self, name): - """ Check if Node has the layer `"name"`""" - + """Check if Node has the layer `"name"`""" def history(self): """ Returns the history of a node, including node additions and changes made to node. @@ -2940,7 +2781,6 @@ class Node: Returns: A list of unix timestamps of the event history of node. """ - def history_date_time(self): """ Returns the history of a node, including node additions and changes made to node. @@ -2949,7 +2789,6 @@ class Node: A list of timestamps of the event history of node. """ - @property def id(self): """ @@ -2959,7 +2798,6 @@ class Node: Returns: The id of the node as an integer. """ - def in_degree(self): """ Get the in-degree of this node (i.e., the number of edges that are incident to it from other nodes). @@ -2967,7 +2805,6 @@ class Node: Returns: The in-degree of this node. """ - @property def in_edges(self): """ @@ -2977,7 +2814,6 @@ class Node: An iterator over the edges that point into this node. """ - @property def in_neighbours(self): """ @@ -2987,7 +2823,6 @@ class Node: An iterator over the neighbours of this node that point into this node. """ - @property def latest_date_time(self): """ @@ -2999,7 +2834,6 @@ class Node: Returns: The latest datetime that the node exists as an integer. """ - @property def latest_time(self): """ @@ -3008,7 +2842,6 @@ class Node: Returns: The latest time that the node exists as an integer. """ - def layer(self, name): """ Return a view of Node containing the layer `"name"` @@ -3017,7 +2850,6 @@ class Node: Returns: Node: The layered view """ - def layers(self, names): """ Return a view of Node containing all layers `names` @@ -3029,7 +2861,6 @@ class Node: Returns: Node: The layered view """ - @property def name(self): """ @@ -3038,7 +2869,6 @@ class Node: Returns: The name of the node as a string. """ - @property def neighbours(self): """ @@ -3048,11 +2878,9 @@ class Node: An iterator over the neighbours of this node. """ - @property def node_type(self): """Returns the type of node""" - def out_degree(self): """ Get the out-degree of this node (i.e., the number of edges that are incident to it from this node). @@ -3060,7 +2888,6 @@ class Node: Returns: The out-degree of this node. """ - @property def out_edges(self): """ @@ -3070,7 +2897,6 @@ class Node: An iterator over the edges that point out of this node. """ - @property def out_neighbours(self): """ @@ -3080,7 +2906,6 @@ class Node: An iterator over the neighbours of this node that point out of this node. """ - @property def properties(self): """ @@ -3089,7 +2914,6 @@ class Node: Returns: A list of properties. """ - def rolling(self, window, step=None): """ Creates a `WindowSet` with the given `window` size and optional `step` using a rolling window. @@ -3103,7 +2927,6 @@ class Node: Returns: A `WindowSet` object. """ - def shrink_end(self, end): """ Set the end of the window to the smaller of `end` and `self.end()` @@ -3113,7 +2936,6 @@ class Node: Returns: A Node object. """ - def shrink_start(self, start): """ Set the start of the window to the larger of `start` and `self.start()` @@ -3124,7 +2946,6 @@ class Node: Returns: A Node object. """ - def shrink_window(self, start, end): """ Shrink both the start and end of the window (same as calling `shrink_start` followed by `shrink_end` but more efficient) @@ -3132,7 +2953,6 @@ class Node: Arguments: """ - @property def start(self): """ @@ -3141,7 +2961,6 @@ class Node: Returns: The earliest time that this Node is valid or None if the Node is valid for all times. """ - @property def start_date_time(self): """ @@ -3150,7 +2969,6 @@ class Node: Returns: The earliest datetime that this Node is valid or None if the Node is valid for all times. """ - def valid_layers(self, names): """ Return a view of Node containing all layers `names` @@ -3162,7 +2980,6 @@ class Node: Returns: Node: The layered view """ - def window(self, start, end): """ Create a view of the Node including all events between `start` (inclusive) and `end` (exclusive) @@ -3174,17 +2991,15 @@ class Node: Returns: r A Node object. """ - @property def window_size(self): - """ Get the window size (difference between start and end) for this Node""" + """Get the window size (difference between start and end) for this Node""" class Nodes: """A list of nodes that can be iterated over.""" def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def after(self, start): """ Create a view of the Nodes including all events after `start` (exclusive). @@ -3195,7 +3010,6 @@ class Nodes: Returns: A Nodes object. """ - def at(self, time): """ Create a view of the Nodes including all events at `time`. @@ -3206,7 +3020,6 @@ class Nodes: Returns: A Nodes object. """ - def before(self, end): """ Create a view of the Nodes including all events before `end` (exclusive). @@ -3217,7 +3030,6 @@ class Nodes: Returns: A Nodes object. """ - def collect(self): """ Collect all nodes into a list @@ -3225,14 +3037,12 @@ class Nodes: Returns: list[Node]: the list of nodes """ - def default_layer(self): """ Return a view of Nodes containing only the default edge layer Returns: Nodes: The layered view """ - def degree(self): """ Returns the number of edges of the nodes @@ -3240,7 +3050,6 @@ class Nodes: Returns: An iterator of the number of edges of the nodes """ - @property def earliest_date_time(self): """ @@ -3249,11 +3058,9 @@ class Nodes: Returns: Earliest time of the nodes. """ - @property def earliest_time(self): """Returns an iterator over the nodes earliest time""" - @property def edges(self): """ @@ -3263,7 +3070,6 @@ class Nodes: An iterator over the edges that are incident to this node. """ - @property def end(self): """ @@ -3272,7 +3078,6 @@ class Nodes: Returns: The latest time that this Nodes is valid or None if the Nodes is valid for all times. """ - @property def end_date_time(self): """ @@ -3281,7 +3086,6 @@ class Nodes: Returns: The latest datetime that this Nodes is valid or None if the Nodes is valid for all times. """ - def exclude_layer(self, name): """ Return a view of Nodes containing all layers except the excluded `name` @@ -3293,7 +3097,6 @@ class Nodes: Returns: Nodes: The layered view """ - def exclude_layers(self, names): """ Return a view of Nodes containing all layers except the excluded `names` @@ -3305,7 +3108,6 @@ class Nodes: Returns: Nodes: The layered view """ - def exclude_valid_layer(self, name): """ Return a view of Nodes containing all layers except the excluded `name` @@ -3315,7 +3117,6 @@ class Nodes: Returns: Nodes: The layered view """ - def exclude_valid_layers(self, names): """ Return a view of Nodes containing all layers except the excluded `names` @@ -3325,7 +3126,6 @@ class Nodes: Returns: Nodes: The layered view """ - def expanding(self, step): """ Creates a `WindowSet` with the given `step` size using an expanding window. @@ -3338,10 +3138,8 @@ class Nodes: Returns: A `WindowSet` object. """ - def has_layer(self, name): - """ Check if Nodes has the layer `"name"`""" - + """Check if Nodes has the layer `"name"`""" def history(self): """ Returns all timestamps of nodes, when an node is added or change to an node is made. @@ -3350,7 +3148,6 @@ class Nodes: A list of unix timestamps. """ - def history_date_time(self): """ Returns all timestamps of nodes, when an node is added or change to an node is made. @@ -3359,11 +3156,9 @@ class Nodes: An list of timestamps. """ - @property def id(self): """Returns an iterator over the nodes ids""" - def in_degree(self): """ Returns the number of in edges of the nodes @@ -3371,7 +3166,6 @@ class Nodes: Returns: An iterator of the number of in edges of the nodes """ - @property def in_edges(self): """ @@ -3381,7 +3175,6 @@ class Nodes: An iterator over the edges that point into this node. """ - @property def in_neighbours(self): """ @@ -3391,7 +3184,6 @@ class Nodes: An iterator over the neighbours of this node that point into this node. """ - @property def latest_date_time(self): """ @@ -3400,11 +3192,9 @@ class Nodes: Returns: Latest date time of the nodes. """ - @property def latest_time(self): """Returns an iterator over the nodes latest time""" - def layer(self, name): """ Return a view of Nodes containing the layer `"name"` @@ -3413,7 +3203,6 @@ class Nodes: Returns: Nodes: The layered view """ - def layers(self, names): """ Return a view of Nodes containing all layers `names` @@ -3425,11 +3214,9 @@ class Nodes: Returns: Nodes: The layered view """ - @property def name(self): """Returns an iterator over the nodes name""" - @property def neighbours(self): """ @@ -3439,11 +3226,9 @@ class Nodes: An iterator over the neighbours of this node. """ - @property def node_type(self): """Returns the type of node""" - def out_degree(self): """ Returns the number of out edges of the nodes @@ -3451,7 +3236,6 @@ class Nodes: Returns: An iterator of the number of out edges of the nodes """ - @property def out_edges(self): """ @@ -3461,7 +3245,6 @@ class Nodes: An iterator over the edges that point out of this node. """ - @property def out_neighbours(self): """ @@ -3471,7 +3254,6 @@ class Nodes: An iterator over the neighbours of this node that point out of this node. """ - @property def properties(self): """ @@ -3480,7 +3262,6 @@ class Nodes: Returns: A List of properties """ - def rolling(self, window, step=None): """ Creates a `WindowSet` with the given `window` size and optional `step` using a rolling window. @@ -3494,7 +3275,6 @@ class Nodes: Returns: A `WindowSet` object. """ - def shrink_end(self, end): """ Set the end of the window to the smaller of `end` and `self.end()` @@ -3504,7 +3284,6 @@ class Nodes: Returns: A Nodes object. """ - def shrink_start(self, start): """ Set the start of the window to the larger of `start` and `self.start()` @@ -3515,7 +3294,6 @@ class Nodes: Returns: A Nodes object. """ - def shrink_window(self, start, end): """ Shrink both the start and end of the window (same as calling `shrink_start` followed by `shrink_end` but more efficient) @@ -3523,7 +3301,6 @@ class Nodes: Arguments: """ - @property def start(self): """ @@ -3532,7 +3309,6 @@ class Nodes: Returns: The earliest time that this Nodes is valid or None if the Nodes is valid for all times. """ - @property def start_date_time(self): """ @@ -3541,7 +3317,6 @@ class Nodes: Returns: The earliest datetime that this Nodes is valid or None if the Nodes is valid for all times. """ - def to_df(self, include_property_history=False, convert_datetime=False): """ Converts the graph's nodes into a Pandas DataFrame. @@ -3558,10 +3333,7 @@ class Nodes: Returns: If successful, this PyObject will be a Pandas DataFrame. """ - - def type_filter(self, node_types): - ... - + def type_filter(self, node_types): ... def valid_layers(self, names): """ Return a view of Nodes containing all layers `names` @@ -3573,7 +3345,6 @@ class Nodes: Returns: Nodes: The layered view """ - def window(self, start, end): """ Create a view of the Nodes including all events between `start` (inclusive) and `end` (exclusive) @@ -3585,17 +3356,15 @@ class Nodes: Returns: r A Nodes object. """ - @property def window_size(self): - """ Get the window size (difference between start and end) for this Nodes""" + """Get the window size (difference between start and end) for this Nodes""" class PersistentGraph: """A temporal graph that allows edges and nodes to be deleted.""" def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def add_constant_properties(self, properties): """ Adds static properties to the graph. @@ -3606,7 +3375,6 @@ class PersistentGraph: Returns: None """ - def add_edge(self, timestamp, src, dst, properties=None, layer=None): """ Adds a new edge with the given source and destination nodes and properties to the graph. @@ -3621,7 +3389,6 @@ class PersistentGraph: Returns: None """ - def add_node(self, timestamp, id, properties=None, node_type=None): """ Adds a new node with the given id and properties to the graph. @@ -3635,7 +3402,6 @@ class PersistentGraph: Returns: None """ - def add_property(self, timestamp, properties): """ Adds properties to the graph. @@ -3647,7 +3413,6 @@ class PersistentGraph: Returns: None """ - def after(self, start): """ Create a view of the GraphView including all events after `start` (exclusive). @@ -3658,7 +3423,6 @@ class PersistentGraph: Returns: A GraphView object. """ - def at(self, time): """ Create a view of the GraphView including all events at `time`. @@ -3669,7 +3433,6 @@ class PersistentGraph: Returns: A GraphView object. """ - def before(self, end): """ Create a view of the GraphView including all events before `end` (exclusive). @@ -3680,10 +3443,16 @@ class PersistentGraph: Returns: A GraphView object. """ + def cache(self, path): + """ + Write PersistentGraph to cache file and initialise the cache. - def bincode(self): - """Get bincode encoded graph""" + Future updates are tracked. Use `write_updates` to persist them to the + cache file. If the file already exists its contents are overwritten. + Arguments: + path (str): The path to the cache file + """ def count_edges(self): """ Number of edges in the graph @@ -3691,7 +3460,6 @@ class PersistentGraph: Returns: the number of edges in the graph """ - def count_nodes(self): """ Number of nodes in the graph @@ -3699,7 +3467,6 @@ class PersistentGraph: Returns: the number of nodes in the graph """ - def count_temporal_edges(self): """ Number of edges in the graph @@ -3707,14 +3474,12 @@ class PersistentGraph: Returns: the number of temporal edges in the graph """ - def default_layer(self): """ Return a view of GraphView containing only the default edge layer Returns: GraphView: The layered view """ - def delete_edge(self, timestamp, src, dst, layer=None): """ Deletes an edge given the timestamp, src and dst nodes and layer (optional) @@ -3726,9 +3491,19 @@ class PersistentGraph: layer (str): The layer of the edge. (optional) Returns: - None or a GraphError if the edge could not be deleted + The deleted edge + """ + @staticmethod + def deserialise(bytes): """ + Load PersistentGraph from serialised bytes. + Arguments: + bytes (Bytes): The serialised bytes to decode + + Returns: + PersistentGraph + """ @property def earliest_date_time(self): """ @@ -3737,7 +3512,6 @@ class PersistentGraph: Returns: the datetime of the earliest activity in the graph """ - @property def earliest_time(self): """ @@ -3746,7 +3520,6 @@ class PersistentGraph: Returns: the timestamp of the earliest activity in the graph """ - def edge(self, src, dst): """ Gets the edge with the specified source and destination nodes @@ -3758,7 +3531,6 @@ class PersistentGraph: Returns: the edge with the specified source and destination nodes, or None if the edge does not exist """ - @property def edges(self): """ @@ -3767,7 +3539,6 @@ class PersistentGraph: Returns: the edges in the graph """ - @property def end(self): """ @@ -3776,7 +3547,6 @@ class PersistentGraph: Returns: The latest time that this GraphView is valid or None if the GraphView is valid for all times. """ - @property def end_date_time(self): """ @@ -3785,10 +3555,8 @@ class PersistentGraph: Returns: The latest datetime that this GraphView is valid or None if the GraphView is valid for all times. """ - def event_graph(self): """Get event graph""" - def exclude_layer(self, name): """ Return a view of GraphView containing all layers except the excluded `name` @@ -3800,7 +3568,6 @@ class PersistentGraph: Returns: GraphView: The layered view """ - def exclude_layers(self, names): """ Return a view of GraphView containing all layers except the excluded `names` @@ -3812,7 +3579,6 @@ class PersistentGraph: Returns: GraphView: The layered view """ - def exclude_nodes(self, nodes): """ Returns a subgraph given a set of nodes that are excluded from the subgraph @@ -3823,7 +3589,6 @@ class PersistentGraph: Returns: GraphView - Returns the subgraph """ - def exclude_valid_layer(self, name): """ Return a view of GraphView containing all layers except the excluded `name` @@ -3833,7 +3598,6 @@ class PersistentGraph: Returns: GraphView: The layered view """ - def exclude_valid_layers(self, names): """ Return a view of GraphView containing all layers except the excluded `names` @@ -3843,7 +3607,6 @@ class PersistentGraph: Returns: GraphView: The layered view """ - def expanding(self, step): """ Creates a `WindowSet` with the given `step` size using an expanding window. @@ -3856,7 +3619,6 @@ class PersistentGraph: Returns: A `WindowSet` object. """ - def find_edges(self, properties_dict): """ Get the edges that match the properties name and value @@ -3865,7 +3627,6 @@ class PersistentGraph: Returns: the edges that match the properties name and value """ - def find_nodes(self, properties_dict): """ Get the nodes that match the properties name and value @@ -3874,7 +3635,6 @@ class PersistentGraph: Returns: the nodes that match the properties name and value """ - def get_all_node_types(self): """ Returns all the node types in the graph. @@ -3882,7 +3642,6 @@ class PersistentGraph: Returns: A list of node types """ - def has_edge(self, src, dst): """ Returns true if the graph contains the specified edge @@ -3894,10 +3653,8 @@ class PersistentGraph: Returns: true if the graph contains the specified edge, false otherwise """ - def has_layer(self, name): - """ Check if GraphView has the layer `"name"`""" - + """Check if GraphView has the layer `"name"`""" def has_node(self, id): """ Returns true if the graph contains the specified node @@ -3908,7 +3665,6 @@ class PersistentGraph: Returns: true if the graph contains the specified node, false otherwise """ - def import_edge(self, edge, force=False): """ Import a single edge into the graph. @@ -3924,7 +3680,6 @@ class PersistentGraph: Returns: Result, GraphError> - A Result object which is Ok if the edge was successfully imported, and Err otherwise. """ - def import_edges(self, edges, force=False): """ Import multiple edges into the graph. @@ -3937,10 +3692,7 @@ class PersistentGraph: edges (List(edges)) - A vector of PyEdge objects representing the edges to be imported. force (boolean) - An optional boolean flag indicating whether to force the import of the edges. - Returns: - Result), GraphError> - A Result object which is Ok if the edges were successfully imported, and Err otherwise. """ - def import_node(self, node, force=False): """ Import a single node into the graph. @@ -3955,7 +3707,6 @@ class PersistentGraph: Returns: Result, GraphError> - A Result object which is Ok if the node was successfully imported, and Err otherwise. """ - def import_nodes(self, nodes, force=False): """ Import multiple nodes into the graph. @@ -3968,10 +3719,7 @@ class PersistentGraph: nodes (List(Node))- A vector of PyNode objects representing the nodes to be imported. force (boolean) - An optional boolean flag indicating whether to force the import of the nodes. - Returns: - Result), GraphError> - A Result object which is Ok if the nodes were successfully imported, and Err otherwise. """ - def index(self): """ Indexes all node and edge properties. @@ -3981,7 +3729,6 @@ class PersistentGraph: Returns: GraphIndex - Returns a GraphIndex """ - @property def latest_date_time(self): """ @@ -3990,7 +3737,6 @@ class PersistentGraph: Returns: the datetime of the latest activity in the graph """ - @property def latest_time(self): """ @@ -3999,7 +3745,6 @@ class PersistentGraph: Returns: the timestamp of the latest activity in the graph """ - def layer(self, name): """ Return a view of GraphView containing the layer `"name"` @@ -4008,7 +3753,6 @@ class PersistentGraph: Returns: GraphView: The layered view """ - def layers(self, names): """ Return a view of GraphView containing all layers `names` @@ -4020,58 +3764,42 @@ class PersistentGraph: Returns: GraphView: The layered view """ - - def load_edge_props_from_pandas(self, df, src, dst, const_properties=None, shared_const_properties=None, layer=None, layer_in_df=True): - """ - Load edge properties from a Pandas DataFrame. - - Arguments: - df (Dataframe): The Pandas DataFrame containing edge information. - src (str): The column name for the source node. - dst (str): The column name for the destination node. - const_properties (List): List of constant edge property column names. Defaults to None. (optional) - shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - layer (str): Layer name. Defaults to None. (optional) - layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the data frame or if it should be used directly as the layer for all edges (optional) defaults to True. - - Returns: - Result<(), GraphError>: Result of the operation. + @staticmethod + def load_cached(path): """ + Load PersistentGraph from a file and initialise it as a cache file. - def load_edge_props_from_parquet(self, parquet_path, src, dst, const_properties=None, shared_const_properties=None, layer=None, layer_in_df=True): - """ - Load edge properties from parquet file + Future updates are tracked. Use `write_updates` to persist them to the + cache file. Arguments: - parquet_path (str): Parquet file or directory of Parquet files path containing edge information. - src (str): The column name for the source node. - dst (str): The column name for the destination node. - const_properties (List): List of constant edge property column names. Defaults to None. (optional) - shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - layer (str): Layer name. Defaults to None. (optional) - layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the data frame or if it should be used directly as the layer for all edges (optional) defaults to True. + path (str): The path to the cache file Returns: - Result<(), GraphError>: Result of the operation. + PersistentGraph """ - - def load_edges_deletions_from_pandas(self, df, src, dst, time, layer=None, layer_in_df=True): + def load_edge_deletions_from_pandas( + self, df, time, src, dst, layer=None, layer_col=None + ): """ Load edges deletions from a Pandas DataFrame into the graph. Arguments: df (Dataframe): The Pandas DataFrame containing the edges. + time (str): The column name for the update timestamps. src (str): The column name for the source node ids. dst (str): The column name for the destination node ids. - time (str): The column name for the update timestamps. - layer (str): The edge layer name (optional) Defaults to None. - layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the dataframe or if it should be used directly as the layer for all edges (optional) defaults to True. - + layer (str): A constant value to use as the layer for all edges (optional) Defaults to None. (cannot be used in combination with layer_col) + layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. (cannot be used in combination with layer) Returns: - Result<(), GraphError>: Result of the operation. - """ + None: If the operation is successful. - def load_edges_deletions_from_parquet(self, parquet_path, src, dst, time, layer=None, layer_in_df=True): + Raises: + GraphError: If the operation fails. + """ + def load_edge_deletions_from_parquet( + self, parquet_path, time, src, dst, layer=None, layer_col=None + ): """ Load edges deletions from a Parquet file into the graph. @@ -4080,181 +3808,253 @@ class PersistentGraph: src (str): The column name for the source node ids. dst (str): The column name for the destination node ids. time (str): The column name for the update timestamps. + layer (str): A constant value to use as the layer for all edges (optional) Defaults to None. (cannot be used in combination with layer_col) + layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. (cannot be used in combination with layer) + Returns: + None: If the operation is successful. + + Raises: + GraphError: If the operation fails. + """ + def load_edge_props_from_pandas( + self, + df, + src, + dst, + constant_properties=None, + shared_constant_properties=None, + layer=None, + layer_col=None, + ): + """ + Load edge properties from a Pandas DataFrame. + + Arguments: + df (Dataframe): The Pandas DataFrame containing edge information. + src (str): The column name for the source node. + dst (str): The column name for the destination node. + constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) layer (str): The edge layer name (optional) Defaults to None. - layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the dataframe or if it should be used directly as the layer for all edges (optional) defaults to True. + layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. Returns: - Result<(), GraphError>: Result of the operation. - """ + None: If the operation is successful. - def load_edges_from_pandas(self, df, src, dst, time, properties=None, const_properties=None, shared_const_properties=None, layer=None, layer_in_df=True): + Raises: + GraphError: If the operation fails. """ - Load edges from a Pandas DataFrame into the graph. + def load_edge_props_from_parquet( + self, + parquet_path, + src, + dst, + constant_properties=None, + shared_constant_properties=None, + layer=None, + layer_col=None, + ): + """ + Load edge properties from parquet file Arguments: - df (Dataframe): The Pandas DataFrame containing the edges. - src (str): The column name for the source node ids. - dst (str): The column name for the destination node ids. - time (str): The column name for the update timestamps. - properties (List): List of edge property column names. Defaults to None. (optional) - const_properties (List): List of constant edge property column names. Defaults to None. (optional) - shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + parquet_path (str): Parquet file or directory of Parquet files path containing edge information. + src (str): The column name for the source node. + dst (str): The column name for the destination node. + constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) layer (str): The edge layer name (optional) Defaults to None. - layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the dataframe or if it should be used directly as the layer for all edges (optional) defaults to True. + layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. Returns: - Result<(), GraphError>: Result of the operation. + None: If the operation is successful. + + Raises: + GraphError: If the operation fails. """ + def load_edges_from_pandas( + self, + df, + time, + src, + dst, + properties=None, + constant_properties=None, + shared_constant_properties=None, + layer=None, + layer_col=None, + ): + """ + Load edges from a Pandas DataFrame into the graph. - def load_edges_from_parquet(self, parquet_path, src, dst, time, properties=None, const_properties=None, shared_const_properties=None, layer=None, layer_in_df=True): + Arguments: + df (Dataframe): The Pandas DataFrame containing the edges. + time (str): The column name for the update timestamps. + src (str): The column name for the source node ids. + dst (str): The column name for the destination node ids. + properties (List[str]): List of edge property column names. Defaults to None. (optional) + constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + layer (str): A constant value to use as the layer for all edges (optional) Defaults to None. (cannot be used in combination with layer_col) + layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. (cannot be used in combination with layer) + Returns: + None: If the operation is successful. + + Raises: + GraphError: If the operation fails. + """ + def load_edges_from_parquet( + self, + parquet_path, + time, + src, + dst, + properties=None, + constant_properties=None, + shared_constant_properties=None, + layer=None, + layer_col=None, + ): """ Load edges from a Parquet file into the graph. Arguments: parquet_path (str): Parquet file or directory of Parquet files path containing edges + time (str): The column name for the update timestamps. src (str): The column name for the source node ids. dst (str): The column name for the destination node ids. - time (str): The column name for the update timestamps. - properties (List): List of edge property column names. Defaults to None. (optional) - const_properties (List): List of constant edge property column names. Defaults to None. (optional) - shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - layer (str): The edge layer name (optional) Defaults to None. - layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the dataframe or if it should be used directly as the layer for all edges (optional) defaults to True. - + properties (List[str]): List of edge property column names. Defaults to None. (optional) + constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + layer (str): A constant value to use as the layer for all edges (optional) Defaults to None. (cannot be used in combination with layer_col) + layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. (cannot be used in combination with layer) Returns: - Result<(), GraphError>: Result of the operation. - """ + None: If the operation is successful. + Raises: + GraphError: If the operation fails. + """ @staticmethod - def load_from_file(path, force=False): + def load_from_file(path): """ - Loads a graph from the given path. + Load PersistentGraph from a file. Arguments: - path (str): The path to the graph. + path (str): The path to the file. Returns: - Graph: The loaded graph. + PersistentGraph """ - - @staticmethod - def load_from_pandas(edge_df, edge_src, edge_dst, edge_time, edge_properties=None, edge_const_properties=None, edge_shared_const_properties=None, edge_layer=None, layer_in_df=True, node_df=None, node_id=None, node_time=None, node_properties=None, node_const_properties=None, node_shared_const_properties=None, node_type=None, node_type_in_df=True): - """ - Load a graph from a Pandas DataFrame. - - Args: - edge_df (pandas.DataFrame): The DataFrame containing the edges. - edge_src (str): The column name for the source node ids. - edge_dst (str): The column name for the destination node ids. - edge_time (str): The column name for the timestamps. - edge_properties (list): The column names for the temporal properties (optional) Defaults to None. - edge_const_properties (list): The column names for the constant properties (optional) Defaults to None. - edge_shared_const_properties (dict): A dictionary of constant properties that will be added to every edge (optional) Defaults to None. - edge_layer (str): The edge layer name (optional) Defaults to None. - layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the edge_df or if it should be used directly as the layer for all edges (optional) defaults to True. - node_df (pandas.DataFrame): The DataFrame containing the nodes (optional) Defaults to None. - node_id (str): The column name for the node ids (optional) Defaults to None. - node_time (str): The column name for the node timestamps (optional) Defaults to None. - node_properties (list): The column names for the node temporal properties (optional) Defaults to None. - node_const_properties (list): The column names for the node constant properties (optional) Defaults to None. - node_shared_const_properties (dict): A dictionary of constant properties that will be added to every node (optional) Defaults to None. - node_type (str): the column name for the node type - node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - - Returns: - Graph: The loaded Graph object. - """ - - @staticmethod - def load_from_parquet(edge_parquet_path, edge_src, edge_dst, edge_time, edge_properties=None, edge_const_properties=None, edge_shared_const_properties=None, edge_layer=None, layer_in_df=True, node_parquet_path=None, node_id=None, node_time=None, node_properties=None, node_const_properties=None, node_shared_const_properties=None, node_type=None, node_type_in_df=True): - """ - Load a graph from Parquet file. - - Args: - edge_parquet_path (str): Parquet file or directory of Parquet files containing the edges. - edge_src (str): The column name for the source node ids. - edge_dst (str): The column name for the destination node ids. - edge_time (str): The column name for the timestamps. - edge_properties (list): The column names for the temporal properties (optional) Defaults to None. - edge_const_properties (list): The column names for the constant properties (optional) Defaults to None. - edge_shared_const_properties (dict): A dictionary of constant properties that will be added to every edge (optional) Defaults to None. - edge_layer (str): The edge layer name (optional) Defaults to None. - layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the edge_df or if it should be used directly as the layer for all edges (optional) defaults to True. - node_parquet_path (str): Parquet file or directory of Parquet files containing the nodes (optional) Defaults to None. - node_id (str): The column name for the node ids (optional) Defaults to None. - node_time (str): The column name for the node timestamps (optional) Defaults to None. - node_properties (list): The column names for the node temporal properties (optional) Defaults to None. - node_const_properties (list): The column names for the node constant properties (optional) Defaults to None. - node_shared_const_properties (dict): A dictionary of constant properties that will be added to every node (optional) Defaults to None. - node_type (str): the column name for the node type - node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - - Returns: - Graph: The loaded Graph object. - """ - - def load_node_props_from_pandas(self, df, id, const_properties=None, shared_const_properties=None): + def load_node_props_from_pandas( + self, + df, + id, + node_type=None, + node_type_col=None, + constant_properties=None, + shared_constant_properties=None, + ): """ Load node properties from a Pandas DataFrame. Arguments: df (Dataframe): The Pandas DataFrame containing node information. id(str): The column name for the node IDs. - const_properties (List): List of constant node property column names. Defaults to None. (optional) - shared_const_properties (>): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) Returns: - Result<(), GraphError>: Result of the operation. - """ + None: If the operation is successful. - def load_node_props_from_parquet(self, parquet_path, id, const_properties=None, shared_const_properties=None): + Raises: + GraphError: If the operation fails. + """ + def load_node_props_from_parquet( + self, + parquet_path, + id, + node_type=None, + node_type_col=None, + constant_properties=None, + shared_constant_properties=None, + ): """ Load node properties from a parquet file. Arguments: parquet_path (str): Parquet file or directory of Parquet files path containing node information. id(str): The column name for the node IDs. - const_properties (List): List of constant node property column names. Defaults to None. (optional) - shared_const_properties (>): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) Returns: - Result<(), GraphError>: Result of the operation. - """ + None: If the operation is successful. - def load_nodes_from_pandas(self, df, id, time, node_type=None, node_type_in_df=True, properties=None, const_properties=None, shared_const_properties=None): + Raises: + GraphError: If the operation fails. + """ + def load_nodes_from_pandas( + self, + df, + time, + id, + node_type=None, + node_type_col=None, + properties=None, + constant_properties=None, + shared_constant_properties=None, + ): """ Load nodes from a Pandas DataFrame into the graph. Arguments: df (pandas.DataFrame): The Pandas DataFrame containing the nodes. - id (str): The column name for the node IDs. time (str): The column name for the timestamps. - node_type (str): the column name for the node type - node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - properties (List): List of node property column names. Defaults to None. (optional) - const_properties (List): List of constant node property column names. Defaults to None. (optional) - shared_const_properties (Dictionary/Hashmap of properties): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) - Returns: - Result<(), GraphError>: Result of the operation. - """ - - def load_nodes_from_parquet(self, parquet_path, id, time, node_type=None, node_type_in_df=True, properties=None, const_properties=None, shared_const_properties=None): + id (str): The column name for the node IDs. + node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + properties (List[str]): List of node property column names. Defaults to None. (optional) + constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + Returns: + None: If the operation is successful. + + Raises: + GraphError: If the operation fails. + """ + def load_nodes_from_parquet( + self, + parquet_path, + time, + id, + node_type=None, + node_type_col=None, + properties=None, + constant_properties=None, + shared_constant_properties=None, + ): """ Load nodes from a Parquet file into the graph. Arguments: parquet_path (str): Parquet file or directory of Parquet files containing the nodes - id (str): The column name for the node IDs. time (str): The column name for the timestamps. - node_type (str): the column name for the node type - node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - properties (List): List of node property column names. Defaults to None. (optional) - const_properties (List): List of constant node property column names. Defaults to None. (optional) - shared_const_properties (Dictionary/Hashmap of properties): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + id (str): The column name for the node IDs. + node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + properties (List[str]): List of node property column names. Defaults to None. (optional) + constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) Returns: - Result<(), GraphError>: Result of the operation. - """ + None: If the operation is successful. + Raises: + GraphError: If the operation fails. + """ def materialize(self): """ Returns a 'materialized' clone of the graph view - i.e. a new graph with a copy of the data seen within the view instead of just a mask over the original graph @@ -4262,7 +4062,6 @@ class PersistentGraph: Returns: GraphView - Returns a graph clone """ - def node(self, id): """ Gets the node with the specified id @@ -4273,7 +4072,6 @@ class PersistentGraph: Returns: the node with the specified id, or None if the node does not exist """ - @property def nodes(self): """ @@ -4282,7 +4080,6 @@ class PersistentGraph: Returns: the nodes in the graph """ - @property def properties(self): """ @@ -4292,7 +4089,6 @@ class PersistentGraph: Returns: HashMap - Properties paired with their names """ - def rolling(self, window, step=None): """ Creates a `WindowSet` with the given `window` size and optional `step` using a rolling window. @@ -4306,18 +4102,20 @@ class PersistentGraph: Returns: A `WindowSet` object. """ - def save_to_file(self, path): """ - Saves the graph to the given path. + Saves the PersistentGraph to the given path. Arguments: - path (str): The path to the graph. + path (str): The path to the file. + """ + def serialise(self): + """ + Serialise PersistentGraph to bytes. Returns: - None + Bytes """ - def shrink_end(self, end): """ Set the end of the window to the smaller of `end` and `self.end()` @@ -4327,7 +4125,6 @@ class PersistentGraph: Returns: A GraphView object. """ - def shrink_start(self, start): """ Set the start of the window to the larger of `start` and `self.start()` @@ -4338,7 +4135,6 @@ class PersistentGraph: Returns: A GraphView object. """ - def shrink_window(self, start, end): """ Shrink both the start and end of the window (same as calling `shrink_start` followed by `shrink_end` but more efficient) @@ -4346,7 +4142,6 @@ class PersistentGraph: Arguments: """ - @property def start(self): """ @@ -4355,7 +4150,6 @@ class PersistentGraph: Returns: The earliest time that this GraphView is valid or None if the GraphView is valid for all times. """ - @property def start_date_time(self): """ @@ -4364,7 +4158,6 @@ class PersistentGraph: Returns: The earliest datetime that this GraphView is valid or None if the GraphView is valid for all times. """ - def subgraph(self, nodes): """ Returns a subgraph given a set of nodes @@ -4375,7 +4168,6 @@ class PersistentGraph: Returns: GraphView - Returns the subgraph """ - def subgraph_node_types(self, node_types): """ Returns a subgraph filtered by node types given a set of node types @@ -4386,8 +4178,14 @@ class PersistentGraph: Returns: GraphView - Returns the subgraph """ - - def to_networkx(self, explode_edges=False, include_node_properties=True, include_edge_properties=True, include_update_history=True, include_property_history=True): + def to_networkx( + self, + explode_edges=False, + include_node_properties=True, + include_edge_properties=True, + include_update_history=True, + include_property_history=True, + ): """ Returns a graph with NetworkX. @@ -4405,8 +4203,18 @@ class PersistentGraph: Returns: A Networkx MultiDiGraph. """ - - def to_pyvis(self, explode_edges=False, edge_color="#000000", shape=None, node_image=None, edge_weight=None, edge_label=None, colour_nodes_by_type=False, notebook=False, **kwargs): + def to_pyvis( + self, + explode_edges=False, + edge_color="#000000", + shape=None, + node_image=None, + edge_weight=None, + edge_label=None, + colour_nodes_by_type=False, + notebook=False, + **kwargs, + ): """ Draw a graph with PyVis. Pyvis is a required dependency. If you intend to use this function make sure that you install Pyvis @@ -4430,11 +4238,9 @@ class PersistentGraph: Returns: A pyvis network """ - @property def unique_layers(self): """Return all the layer ids in the graph""" - def update_constant_properties(self, properties): """ Updates static properties to the graph. @@ -4445,7 +4251,6 @@ class PersistentGraph: Returns: None """ - def valid_layers(self, names): """ Return a view of GraphView containing all layers `names` @@ -4457,8 +4262,16 @@ class PersistentGraph: Returns: GraphView: The layered view """ - - def vectorise(self, embedding, cache=None, overwrite_cache=False, graph_document=None, node_document=None, edge_document=None, verbose=False): + def vectorise( + self, + embedding, + cache=None, + overwrite_cache=False, + graph_document=None, + node_document=None, + edge_document=None, + verbose=False, + ): """ Create a VectorisedGraph from the current graph @@ -4473,7 +4286,6 @@ class PersistentGraph: Returns: A VectorisedGraph with all the documents/embeddings computed and with an initial empty selection """ - def window(self, start, end): """ Create a view of the GraphView including all events between `start` (inclusive) and `end` (exclusive) @@ -4485,24 +4297,22 @@ class PersistentGraph: Returns: r A GraphView object. """ - @property def window_size(self): - """ Get the window size (difference between start and end) for this GraphView""" + """Get the window size (difference between start and end) for this GraphView""" + def write_updates(self): + """Persist the new updates by appending them to the cache file.""" class Properties: """A view of the properties of an entity""" def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def as_dict(self): """Convert properties view to a dict""" - @property def constant(self): """Get a view of the constant properties (meta-data) only.""" - def get(self, key): """ Get property value. @@ -4510,17 +4320,13 @@ class Properties: First searches temporal properties and returns latest value if it exists. If not, it falls back to static properties. """ - def items(self): """Get a list of key-value pairs""" - def keys(self): """Get the names for all properties (includes temporal and static properties)""" - @property def temporal(self): """Get a view of the temporal properties only.""" - def values(self): """ Get the values of the properties @@ -4534,12 +4340,9 @@ class PyDirection: def __init__(self, direction): """Initialize self. See help(type(self)) for accurate signature.""" - - def as_str(self): - ... + def as_str(self): ... class PyGraphEncoder: - def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" @@ -4548,10 +4351,8 @@ class TemporalProp: def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def at(self, t): """Get the value of the property at time `t`""" - def average(self): """ Compute the average of all property values. Alias for mean(). @@ -4559,7 +4360,6 @@ class TemporalProp: Returns: Prop: The average of each property values, or None if count is zero. """ - def count(self): """ Count the number of properties. @@ -4567,19 +4367,14 @@ class TemporalProp: Returns: int: The number of properties. """ - def history(self): """Get the timestamps at which the property was updated""" - def history_date_time(self): """Get the timestamps at which the property was updated""" - def items(self): """List update timestamps and corresponding property values""" - def items_date_time(self): """List update timestamps and corresponding property values""" - def max(self): """ Find the maximum property value and its associated time. @@ -4587,7 +4382,6 @@ class TemporalProp: Returns: (i64, Prop): A tuple containing the time and the maximum property value. """ - def mean(self): """ Compute the mean of all property values. Alias for mean(). @@ -4595,7 +4389,6 @@ class TemporalProp: Returns: Prop: The mean of each property values, or None if count is zero. """ - def median(self): """ Compute the median of all property values. @@ -4603,7 +4396,6 @@ class TemporalProp: Returns: (i64, Prop): A tuple containing the time and the median property value, or None if empty """ - def min(self): """ Find the minimum property value and its associated time. @@ -4611,10 +4403,7 @@ class TemporalProp: Returns: (i64, Prop): A tuple containing the time and the minimum property value. """ - - def ordered_dedupe(self, latest_time): - ... - + def ordered_dedupe(self, latest_time): ... def sum(self): """ Compute the sum of all property values. @@ -4622,13 +4411,9 @@ class TemporalProp: Returns: Prop: The sum of all property values. """ - - def unique(self): - ... - + def unique(self): ... def value(self): """Get the latest value of the property""" - def values(self): """Get the property values for each update""" @@ -4637,7 +4422,6 @@ class TemporalProperties: def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def get(self, key): """ get(key: str) -> Optional[TemporalProp] @@ -4647,7 +4431,6 @@ class TemporalProperties: Returns: the property view if it exists, otherwise `None` """ - def histories(self): """ Get the histories of all properties @@ -4655,7 +4438,6 @@ class TemporalProperties: Returns: dict[str, list[(int, Any)]]: the mapping of property keys to histories """ - def histories_date_time(self): """ Get the histories of all properties @@ -4663,13 +4445,10 @@ class TemporalProperties: Returns: dict[str, list[(datetime, Any)]]: the mapping of property keys to histories """ - def items(self): """List the property keys together with the corresponding values""" - def keys(self): """List the available property keys""" - def latest(self): """ Get the latest value of all properties @@ -4677,7 +4456,6 @@ class TemporalProperties: Returns: dict[str, Any]: the mapping of property keys to latest values """ - def values(self): """ List the values of the properties diff --git a/python/python/raphtory/algorithms/__init__.pyi b/python/python/raphtory/algorithms/__init__.pyi index 0466b74dde..2f58811862 100644 --- a/python/python/raphtory/algorithms/__init__.pyi +++ b/python/python/raphtory/algorithms/__init__.pyi @@ -68,9 +68,12 @@ def betweenness_centrality(g, k=None, normalized=True): AlgorithmResult[float]: Returns an `AlgorithmResult` containing the betweenness centrality of each node. """ -def cohesive_fruchterman_reingold(graph, iterations=100, scale=1.0, node_start_size=1.0, cooloff_factor=0.95, dt=0.1): +def cohesive_fruchterman_reingold( + graph, iterations=100, scale=1.0, node_start_size=1.0, cooloff_factor=0.95, dt=0.1 +): """Cohesive version of `fruchterman_reingold` that adds virtual edges between isolated nodes""" +def connected_components(g): ... def degree_centrality(g, threads=None): """ Computes the degree centrality of all nodes in the graph. The values are normalized @@ -85,7 +88,9 @@ def degree_centrality(g, threads=None): AlgorithmResult>: A result containing a mapping of node names to the computed sum of their associated degree centrality. """ -def dijkstra_single_source_shortest_paths(g, source, targets, direction=..., weight=...): +def dijkstra_single_source_shortest_paths( + g, source, targets, direction=..., weight=... +): """ Finds the shortest paths from a single source to multiple targets in a graph. @@ -115,7 +120,9 @@ def directed_graph_density(g): float : Directed graph density of G. """ -def fruchterman_reingold(graph, iterations=100, scale=1.0, node_start_size=1.0, cooloff_factor=0.95, dt=0.1): +def fruchterman_reingold( + graph, iterations=100, scale=1.0, node_start_size=1.0, cooloff_factor=0.95, dt=0.1 +): """ Fruchterman Reingold layout algorithm @@ -446,7 +453,15 @@ def strongly_connected_components(g): Vec> : List of strongly connected nodes identified by ids """ -def temporal_SEIR(graph, seeds, infection_prob, initial_infection, recovery_rate=None, incubation_rate=None, rng_seed=None): +def temporal_SEIR( + graph, + seeds, + infection_prob, + initial_infection, + recovery_rate=None, + incubation_rate=None, + rng_seed=None, +): """ Simulate an SEIR dynamic on the network @@ -470,7 +485,7 @@ def temporal_SEIR(graph, seeds, infection_prob, initial_infection, recovery_rate Returns: AlgorithmResult[Infected]: Returns an `Infected` object for each infected node with attributes - + `infected`: the time stamp of the infection event `active`: the time stamp at which the node actively starts spreading the infection (i.e., the end of the incubation period) diff --git a/python/python/raphtory/graph_loader/__init__.pyi b/python/python/raphtory/graph_loader/__init__.pyi index 63b6858373..9232d66a23 100644 --- a/python/python/raphtory/graph_loader/__init__.pyi +++ b/python/python/raphtory/graph_loader/__init__.pyi @@ -54,9 +54,7 @@ def lotr_graph(): A Graph containing the LOTR dataset """ -def neo4j_movie_graph(uri, username, password, database=...): - ... - +def neo4j_movie_graph(uri, username, password, database=...): ... def reddit_hyperlink_graph(timeout_seconds=600): """ Load (a subset of) Reddit hyperlinks dataset into a graph. @@ -96,8 +94,5 @@ def reddit_hyperlink_graph(timeout_seconds=600): A Graph containing the Reddit hyperlinks dataset """ -def reddit_hyperlink_graph_local(file_path): - ... - -def stable_coin_graph(path=None, subset=None): - ... +def reddit_hyperlink_graph_local(file_path): ... +def stable_coin_graph(path=None, subset=None): ... diff --git a/python/python/raphtory/graphql/__init__.pyi b/python/python/raphtory/graphql/__init__.pyi index d795c6ed6b..817444700c 100644 --- a/python/python/raphtory/graphql/__init__.pyi +++ b/python/python/raphtory/graphql/__init__.pyi @@ -7,106 +7,33 @@ # # ############################################################################### -class GraphqlGraphs: - """ - A class for accessing graphs hosted in a Raphtory GraphQL server and running global search for - graph documents - """ - - def __init__(self): - """Initialize self. See help(type(self)) for accurate signature.""" - - def get(self, name): - """Return the `VectorisedGraph` with name `name` or `None` if it doesn't exist""" - - def search_graph_documents(self, query, limit, window): - """ - Return the top documents with the smallest cosine distance to `query` - - # Arguments - * query - the text or the embedding to score against - * limit - the maximum number of documents to return - * window - the window where documents need to belong to in order to be considered - - # Returns - A list of documents - """ - - def search_graph_documents_with_scores(self, query, limit, window): - """Same as `search_graph_documents` but it also returns the scores alongside the documents""" - -class RaphtoryClient: - """A client for handling GraphQL operations in the context of Raphtory.""" - - def __init__(self, url): - """Initialize self. See help(type(self)) for accurate signature.""" - - def load_graphs_from_path(self, path, overwrite=False): - """ - Set the server to load all the graphs from its path `path`. - - Arguments: - * `path`: the path to load the graphs from. - * `overwrite`: whether or not to overwrite existing graphs (defaults to False) - - Returns: - The `data` field from the graphQL response after executing the mutation. - """ - - def query(self, query, variables=None): - """ - Make a graphQL query against the server. - - Arguments: - * `query`: the query to make. - * `variables`: a dict of variables present on the query and their values. - - Returns: - The `data` field from the graphQL response. - """ - - def send_graph(self, name, graph): - """ - Send a graph to the server. - - Arguments: - * `name`: the name of the graph sent. - * `graph`: the graph to send. - - Returns: - The `data` field from the graphQL response after executing the mutation. - """ - - def wait_for_online(self, millis=None): - """ - Wait for the server to be online. - - Arguments: - * `millis`: the minimum number of milliseconds to wait (default 5000). - """ - -class RaphtoryServer: +class GraphServer: """A class for defining and running a Raphtory GraphQL server""" - def __init__(self, graphs=None, graph_dir=None): + def __init__( + self, + work_dir, + cache_capacity=None, + cache_tti_seconds=None, + log_level=None, + config_path=None, + ): """Initialize self. See help(type(self)) for accurate signature.""" - - def run(self, port=1736, log_level=..., enable_tracing=False, enable_auth=False): + def run(self, port=1736, timeout_ms=...): """ Run the server until completion. Arguments: * `port`: the port to use (defaults to 1736). """ - - def start(self, port=1736, log_level=..., enable_tracing=False, enable_auth=False): + def start(self, port=1736, timeout_ms=None): """ Start the server and return a handle to it. Arguments: * `port`: the port to use (defaults to 1736). + * `timeout_ms`: wait for server to be online (defaults to 5000). The server is stopped if not online within timeout_ms but manages to come online as soon as timeout_ms finishes! """ - def with_document_search_function(self, name, input, function): """ Register a function in the GraphQL schema for document search over a graph. @@ -124,7 +51,6 @@ class RaphtoryServer: Returns: A new server object containing the vectorised graphs. """ - def with_global_search_function(self, name, input, function): """ Register a function in the GraphQL schema for document search among all the graphs. @@ -142,8 +68,15 @@ class RaphtoryServer: Returns: A new server object containing the vectorised graphs. """ - - def with_vectorised(self, cache, graph_names=None, embedding=None, graph_document=None, node_document=None, edge_document=None): + def with_vectorised( + self, + cache, + graph_names=None, + embedding=None, + graph_document=None, + node_document=None, + edge_document=None, + ): """ Vectorise a subset of the graphs of the server. @@ -163,24 +96,75 @@ class RaphtoryServer: A new server object containing the vectorised graphs. """ -class RunningRaphtoryServer: - """A Raphtory server handler that also enables querying the server""" +class GraphqlGraphs: + """ + A class for accessing graphs hosted in a Raphtory GraphQL server and running global search for + graph documents + """ def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" + def get(self, name): + """Return the `VectorisedGraph` with name `name` or `None` if it doesn't exist""" + def search_graph_documents(self, query, limit, window): + """ + Return the top documents with the smallest cosine distance to `query` - def load_graphs_from_path(self, path, overwrite=False): + # Arguments + * query - the text or the embedding to score against + * limit - the maximum number of documents to return + * window - the window where documents need to belong to in order to be considered + + # Returns + A list of documents + """ + def search_graph_documents_with_scores(self, query, limit, window): + """Same as `search_graph_documents` but it also returns the scores alongside the documents""" + +class RaphtoryClient: + """A client for handling GraphQL operations in the context of Raphtory.""" + + def __init__(self, url): + """Initialize self. See help(type(self)) for accurate signature.""" + def copy_graph(self, path, new_path): """ - Set the server to load all the graphs from its path `path`. + Copy graph from a path `path` on the server to a `new_path` on the server Arguments: - * `path`: the path to load the graphs from. - * `overwrite`: whether or not to overwrite existing graphs (defaults to False) + * `path`: the path of the graph to be copied + * `new_path`: the new path of the copied graph Returns: - The `data` field from the graphQL response after executing the mutation. + Copy status as boolean """ + def delete_graph(self, path): + """ + Delete graph from a path `path` on the server + + Arguments: + * `path`: the path of the graph to be deleted + Returns: + Delete status as boolean + """ + def is_server_online(self): + """ + Check if the server is online. + + Returns: + Returns true if server is online otherwise false. + """ + def move_graph(self, path, new_path): + """ + Move graph from a path `path` on the server to a `new_path` on the server + + Arguments: + * `path`: the path of the graph to be moved + * `new_path`: the new path of the moved graph + + Returns: + Move status as boolean + """ def query(self, query, variables=None): """ Make a graphQL query against the server. @@ -192,29 +176,46 @@ class RunningRaphtoryServer: Returns: The `data` field from the graphQL response. """ - - def send_graph(self, name, graph): + def receive_graph(self, path): """ - Send a graph to the server. + Receive graph from a path `path` on the server Arguments: - * `name`: the name of the graph sent. - * `graph`: the graph to send. + * `path`: the path of the graph to be received Returns: - The `data` field from the graphQL response after executing the mutation. + Graph as string """ + def send_graph(self, path, graph, overwrite=False): + """ + Send a graph to the server - def stop(self): - """Stop the server.""" - - def wait(self): - """Wait until server completion.""" + Arguments: + * `path`: the path of the graph + * `graph`: the graph to send + * `overwrite`: overwrite existing graph (defaults to False) - def wait_for_online(self, timeout_millis=None): + Returns: + The `data` field from the graphQL response after executing the mutation. """ - Wait for the server to be online. + def upload_graph(self, path, file_path, overwrite=False): + """ + Upload graph file from a path `file_path` on the client Arguments: - * `timeout_millis`: the timeout in milliseconds (default 5000). + * `path`: the name of the graph + * `file_path`: the path of the graph on the client + * `overwrite`: overwrite existing graph (defaults to False) + + Returns: + The `data` field from the graphQL response after executing the mutation. """ + +class RunningGraphServer: + """A Raphtory server handler that also enables querying the server""" + + def __init__(self): + """Initialize self. See help(type(self)) for accurate signature.""" + def get_client(self): ... + def stop(self): + """Stop the server and wait for it to finish""" diff --git a/python/python/raphtory/vectors/__init__.pyi b/python/python/raphtory/vectors/__init__.pyi index 6f42e39246..9e09acbe87 100644 --- a/python/python/raphtory/vectors/__init__.pyi +++ b/python/python/raphtory/vectors/__init__.pyi @@ -8,27 +8,18 @@ ############################################################################### class Document: - def __init__(self, content, life=None): """Initialize self. See help(type(self)) for accurate signature.""" - @property - def content(self): - ... - + def content(self): ... @property - def entity(self): - ... - + def entity(self): ... @property - def life(self): - ... + def life(self): ... class VectorisedGraph: - def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def append(self, nodes, edges): """ Add all the documents from `nodes` and `edges` to the current selection @@ -42,7 +33,6 @@ class VectorisedGraph: Returns: A new vectorised graph containing the updated selection """ - def append_by_similarity(self, query, limit, window=None): """ Add the top `limit` documents to the current selection using `query` @@ -55,7 +45,6 @@ class VectorisedGraph: Returns: A new vectorised graph containing the updated selection """ - def append_edges(self, edges): """ Add all the documents from `edges` to the current selection @@ -68,7 +57,6 @@ class VectorisedGraph: Returns: A new vectorised graph containing the updated selection """ - def append_edges_by_similarity(self, query, limit, window=None): """ Add the top `limit` edge documents to the current selection using `query` @@ -81,7 +69,6 @@ class VectorisedGraph: Returns: A new vectorised graph containing the updated selection """ - def append_nodes(self, nodes): """ Add all the documents from `nodes` to the current selection @@ -94,7 +81,6 @@ class VectorisedGraph: Returns: A new vectorised graph containing the updated selection """ - def append_nodes_by_similarity(self, query, limit, window=None): """ Add the top `limit` node documents to the current selection using `query` @@ -107,10 +93,8 @@ class VectorisedGraph: Returns: A new vectorised graph containing the updated selection """ - def edges(self): """Return the edges present in the current selection""" - def expand(self, hops, window=None): """ Add all the documents `hops` hops away to the selection @@ -127,7 +111,6 @@ class VectorisedGraph: Returns: A new vectorised graph containing the updated selection """ - def expand_by_similarity(self, query, limit, window=None): """ Add the top `limit` adjacent documents with higher score for `query` to the selection @@ -148,7 +131,6 @@ class VectorisedGraph: Returns: A new vectorised graph containing the updated selection """ - def expand_edges_by_similarity(self, query, limit, window=None): """ Add the top `limit` adjacent edge documents with higher score for `query` to the selection @@ -163,7 +145,6 @@ class VectorisedGraph: Returns: A new vectorised graph containing the updated selection """ - def expand_nodes_by_similarity(self, query, limit, window=None): """ Add the top `limit` adjacent node documents with higher score for `query` to the selection @@ -178,18 +159,13 @@ class VectorisedGraph: Returns: A new vectorised graph containing the updated selection """ - def get_documents(self): """Return the documents present in the current selection""" - def get_documents_with_scores(self): """Return the documents alongside their scores present in the current selection""" - def nodes(self): """Return the nodes present in the current selection""" - def save_embeddings(self, file): """Save the embeddings present in this graph to `file` so they can be further used in a call to `vectorise`""" -def generate_property_list(entity, filter_out=..., force_static=...): - ... +def generate_property_list(entity, filter_out=..., force_static=...): ... diff --git a/python/tests/test_algorithms.py b/python/tests/test_algorithms.py index 35ea620beb..5fccc4851e 100644 --- a/python/tests/test_algorithms.py +++ b/python/tests/test_algorithms.py @@ -258,7 +258,11 @@ def test_temporal_reachability(): actual = algorithms.temporally_reachable_nodes(g, 20, 11, [1, 2], [4, 5]) expected = { "1": [(11, "start")], - "2": [(11, "start"), (12, "1"), (11, "1")], + "2": [ + (11, "1"), + (11, "start"), + (12, "1"), + ], "3": [], "4": [(12, "2")], "5": [(13, "2")], diff --git a/python/tests/test_disk_graph.py b/python/tests/test_disk_graph.py index 8d4a032f04..37b4e88269 100644 --- a/python/tests/test_disk_graph.py +++ b/python/tests/test_disk_graph.py @@ -1,4 +1,4 @@ -from raphtory import PyDirection, DiskGraphStorage +from raphtory import DiskGraphStorage from raphtory import algorithms import pandas as pd import tempfile @@ -35,17 +35,16 @@ ).sort_values(["src", "dst", "time"]) -def create_graph(edges, dir): - return DiskGraphStorage.load_from_pandas(dir, edges, "src", "dst", "time") - - # in every test use with to create a temporary directory that will be deleted automatically # after the with block ends def test_counts(): - dir = tempfile.TemporaryDirectory() - graph = create_graph(edges, dir.name).to_events() + graph_dir = tempfile.TemporaryDirectory() + graph = DiskGraphStorage.load_from_pandas( + graph_dir.name, edges, "time", "src", "dst" + ) + graph = graph.to_events() assert graph.count_nodes() == 5 assert graph.count_edges() == 20 @@ -140,6 +139,7 @@ def test_disk_graph(): ) assert len(list(actual.get_all_with_names())) == 1624 + def test_disk_graph_type_filter(): curr_dir = os.path.dirname(os.path.abspath(__file__)) rsc_dir = os.path.join(curr_dir, "..", "..", "pometry-storage-private", "resources") @@ -172,7 +172,7 @@ def test_disk_graph_type_filter(): read_chunk_size, concurrent_files, num_threads, - "node_type" + "node_type", ).to_events() assert g.count_nodes() == 1619 @@ -190,14 +190,20 @@ def test_disk_graph_type_filter(): assert g.nodes.type_filter([]).name.collect() == [] - neighbor_names = g.nodes.type_filter(["A"]).neighbours.type_filter(["B"]).name.collect() + neighbor_names = ( + g.nodes.type_filter(["A"]).neighbours.type_filter(["B"]).name.collect() + ) total_length = sum(len(names) for names in neighbor_names) assert total_length == 1023 - assert g.node("Comp175846").neighbours.type_filter(["A"]).name.collect() == ["Comp844043"] + assert g.node("Comp175846").neighbours.type_filter(["A"]).name.collect() == [ + "Comp844043" + ] assert g.node("Comp175846").neighbours.type_filter(["B"]).name.collect() == [] assert g.node("Comp175846").neighbours.type_filter([]).name.collect() == [] - assert g.node("Comp175846").neighbours.type_filter(["A", "B"]).name.collect() == ["Comp844043"] + assert g.node("Comp175846").neighbours.type_filter(["A", "B"]).name.collect() == [ + "Comp844043" + ] neighbor_names = g.node("Comp175846").neighbours.neighbours.name.collect() - assert len(neighbor_names) == 193 \ No newline at end of file + assert len(neighbor_names) == 193 diff --git a/python/tests/test_graph_conversions.py b/python/tests/test_graph_conversions.py index 5e24b394ed..c881c42dc7 100644 --- a/python/tests/test_graph_conversions.py +++ b/python/tests/test_graph_conversions.py @@ -21,23 +21,26 @@ def build_graph(): nodes_df["timestamp"] = pd.to_datetime(nodes_df["timestamp"]).astype( "datetime64[ms, UTC]" ) - - return Graph.load_from_pandas( - edge_df=edges_df, - edge_src="source", - edge_dst="destination", - edge_time="timestamp", - edge_properties=["data_size_MB"], - edge_layer="transaction_type", - edge_const_properties=["is_encrypted"], - edge_shared_const_properties={"datasource": "data/network_traffic_edges.csv"}, - node_df=nodes_df, - node_id="server_id", - node_time="timestamp", - node_properties=["OS_version", "primary_function", "uptime_days"], - node_const_properties=["server_name", "hardware_type"], - node_shared_const_properties={"datasource": "data/network_traffic_edges.csv"}, + g = Graph() + g.load_edges_from_pandas( + edges_df, + time="timestamp", + src="source", + dst="destination", + properties=["data_size_MB"], + layer_col="transaction_type", + constant_properties=["is_encrypted"], + shared_constant_properties={"datasource": "data/network_traffic_edges.csv"}, + ) + g.load_nodes_from_pandas( + df=nodes_df, + id="server_id", + time="timestamp", + properties=["OS_version", "primary_function", "uptime_days"], + constant_properties=["server_name", "hardware_type"], + shared_constant_properties={"datasource": "data/network_traffic_edges.csv"}, ) + return g def build_graph_without_datetime_type(): @@ -47,22 +50,26 @@ def build_graph_without_datetime_type(): nodes_df = pd.read_csv(base_dir / "data/network_traffic_nodes.csv") nodes_df["timestamp"] = pd.to_datetime(nodes_df["timestamp"]) - return Graph.load_from_pandas( - edge_df=edges_df, - edge_src="source", - edge_dst="destination", - edge_time="timestamp", - edge_properties=["data_size_MB"], - edge_layer="transaction_type", - edge_const_properties=["is_encrypted"], - edge_shared_const_properties={"datasource": "data/network_traffic_edges.csv"}, - node_df=nodes_df, - node_id="server_id", - node_time="timestamp", - node_properties=["OS_version", "primary_function", "uptime_days"], - node_const_properties=["server_name", "hardware_type"], - node_shared_const_properties={"datasource": "data/network_traffic_edges.csv"}, + g = Graph() + g.load_edges_from_pandas( + edges_df, + time="timestamp", + src="source", + dst="destination", + properties=["data_size_MB"], + layer_col="transaction_type", + constant_properties=["is_encrypted"], + shared_constant_properties={"datasource": "data/network_traffic_edges.csv"}, + ) + g.load_nodes_from_pandas( + df=nodes_df, + id="server_id", + time="timestamp", + properties=["OS_version", "primary_function", "uptime_days"], + constant_properties=["server_name", "hardware_type"], + shared_constant_properties={"datasource": "data/network_traffic_edges.csv"}, ) + return g def test_graph_timestamp_list_properties(): @@ -125,35 +132,35 @@ def test_py_vis(): [ { "color": "#97c2fc", - "id": 'ServerA', + "id": "ServerA", "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "ServerA", "shape": "dot", }, { "color": "#97c2fc", - "id": 'ServerB', + "id": "ServerB", "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "ServerB", "shape": "dot", }, { "color": "#97c2fc", - "id": 'ServerC', + "id": "ServerC", "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "ServerC", "shape": "dot", }, { "color": "#97c2fc", - "id": 'ServerD', + "id": "ServerD", "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "ServerD", "shape": "dot", }, { "color": "#97c2fc", - "id": 'ServerE', + "id": "ServerE", "image": "https://cdn-icons-png.flaticon.com/512/7584/7584620.png", "label": "ServerE", "shape": "dot", @@ -168,63 +175,63 @@ def test_py_vis(): "arrowStrikethrough": False, "arrows": "to", "color": "#000000", - "from": 'ServerA', + "from": "ServerA", "title": "", - "to": 'ServerB', + "to": "ServerB", "value": 0.0, }, { "arrowStrikethrough": False, "arrows": "to", "color": "#000000", - "from": 'ServerA', + "from": "ServerA", "title": "", - "to": 'ServerC', + "to": "ServerC", "value": 0.0, }, { "arrowStrikethrough": False, "arrows": "to", "color": "#000000", - "from": 'ServerB', + "from": "ServerB", "title": "", - "to": 'ServerD', + "to": "ServerD", "value": 0.0, }, { "arrowStrikethrough": False, "arrows": "to", "color": "#000000", - "from": 'ServerC', + "from": "ServerC", "title": "", - "to": 'ServerA', + "to": "ServerA", "value": 0.0, }, { "arrowStrikethrough": False, "arrows": "to", "color": "#000000", - "from": 'ServerD', + "from": "ServerD", "title": "", - "to": 'ServerC', + "to": "ServerC", "value": 0.0, }, { "arrowStrikethrough": False, "arrows": "to", "color": "#000000", - "from": 'ServerD', + "from": "ServerD", "title": "", - "to": 'ServerE', + "to": "ServerE", "value": 0.0, }, { "arrowStrikethrough": False, "arrows": "to", "color": "#000000", - "from": 'ServerE', + "from": "ServerE", "title": "", - "to": 'ServerB', + "to": "ServerB", "value": 0.0, }, ], diff --git a/python/tests/test_graphql.py b/python/tests/test_graphql.py index e0b9ec6c77..c27d87bbcf 100644 --- a/python/tests/test_graphql.py +++ b/python/tests/test_graphql.py @@ -1,18 +1,19 @@ import base64 import os +import pickle import tempfile import time import pytest -from raphtory.graphql import RaphtoryServer, RaphtoryClient +from raphtory.graphql import GraphServer, RaphtoryClient from raphtory import graph_loader from raphtory import Graph import json def normalize_path(path): - return path.replace('\\', '/') + return path.replace("\\", "/") def test_failed_server_start_in_time(): @@ -20,16 +21,22 @@ def test_failed_server_start_in_time(): server = None try: with pytest.raises(Exception) as excinfo: - server = RaphtoryServer(tmp_work_dir).start(timeout_ms=1) + server = GraphServer(tmp_work_dir).start(timeout_ms=1) assert str(excinfo.value) == "Failed to start server in 1 milliseconds" finally: if server: server.stop() +def test_wrong_url(): + with pytest.raises(Exception) as excinfo: + client = RaphtoryClient("http://broken_url.com") + assert str(excinfo.value) == "Could not connect to the given server - no response" + + def test_successful_server_start_in_time(): tmp_work_dir = tempfile.mkdtemp() - server = RaphtoryServer(tmp_work_dir).start(timeout_ms=3000) + server = GraphServer(tmp_work_dir).start(timeout_ms=3000) client = server.get_client() assert client.is_server_online() server.stop() @@ -43,14 +50,16 @@ def test_server_start_on_default_port(): g.add_edge(3, "ben", "haaroon") tmp_work_dir = tempfile.mkdtemp() - with RaphtoryServer(tmp_work_dir).start(): + with GraphServer(tmp_work_dir).start(): client = RaphtoryClient("http://localhost:1736") client.send_graph(path="g", graph=g) query = """{graph(path: "g") {nodes {list {name}}}}""" assert client.query(query) == { "graph": { - "nodes": {"list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}]} + "nodes": { + "list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}] + } } } @@ -62,14 +71,16 @@ def test_server_start_on_custom_port(): g.add_edge(3, "ben", "haaroon") tmp_work_dir = tempfile.mkdtemp() - with RaphtoryServer(tmp_work_dir).start(port=1737): + with GraphServer(tmp_work_dir).start(port=1737): client = RaphtoryClient("http://localhost:1737") client.send_graph(path="g", graph=g) query = """{graph(path: "g") {nodes {list {name}}}}""" assert client.query(query) == { "graph": { - "nodes": {"list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}]} + "nodes": { + "list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}] + } } } @@ -81,7 +92,7 @@ def test_send_graph_succeeds_if_no_graph_found_with_same_name(): g.add_edge(3, "ben", "haaroon") tmp_work_dir = tempfile.mkdtemp() - with RaphtoryServer(tmp_work_dir).start(): + with GraphServer(tmp_work_dir).start(): client = RaphtoryClient("http://localhost:1736") client.send_graph(path="g", graph=g) @@ -95,7 +106,7 @@ def test_send_graph_fails_if_graph_already_exists(): g.add_edge(3, "ben", "haaroon") g.save_to_file(os.path.join(tmp_work_dir, "g")) - with RaphtoryServer(tmp_work_dir).start(): + with GraphServer(tmp_work_dir).start(): client = RaphtoryClient("http://localhost:1736") with pytest.raises(Exception) as excinfo: client.send_graph(path="g", graph=g) @@ -111,7 +122,7 @@ def test_send_graph_succeeds_if_graph_already_exists_with_overwrite_enabled(): g.add_edge(3, "ben", "haaroon") g.save_to_file(os.path.join(tmp_work_dir, "g")) - with RaphtoryServer(tmp_work_dir).start(): + with GraphServer(tmp_work_dir).start(): client = RaphtoryClient("http://localhost:1736") g = Graph() @@ -124,7 +135,14 @@ def test_send_graph_succeeds_if_graph_already_exists_with_overwrite_enabled(): query = """{graph(path: "g") {nodes {list {name}}}}""" assert client.query(query) == { "graph": { - "nodes": {"list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}, {"name": "shivam"}]} + "nodes": { + "list": [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + {"name": "shivam"}, + ] + } } } @@ -136,7 +154,7 @@ def test_send_graph_succeeds_if_no_graph_found_with_same_name_at_namespace(): g.add_edge(3, "ben", "haaroon") tmp_work_dir = tempfile.mkdtemp() - with RaphtoryServer(tmp_work_dir).start(): + with GraphServer(tmp_work_dir).start(): client = RaphtoryClient("http://localhost:1736") client.send_graph(path="shivam/g", graph=g) @@ -151,7 +169,7 @@ def test_send_graph_fails_if_graph_already_exists_at_namespace(): os.makedirs(os.path.join(tmp_work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(tmp_work_dir, "shivam", "g")) - with RaphtoryServer(tmp_work_dir).start(): + with GraphServer(tmp_work_dir).start(): client = RaphtoryClient("http://localhost:1736") with pytest.raises(Exception) as excinfo: client.send_graph(path="shivam/g", graph=g) @@ -168,7 +186,7 @@ def test_send_graph_succeeds_if_graph_already_exists_at_namespace_with_overwrite os.makedirs(os.path.join(tmp_work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(tmp_work_dir, "shivam", "g")) - with RaphtoryServer(tmp_work_dir).start(): + with GraphServer(tmp_work_dir).start(): client = RaphtoryClient("http://localhost:1736") g = Graph() @@ -181,7 +199,14 @@ def test_send_graph_succeeds_if_graph_already_exists_at_namespace_with_overwrite query = """{graph(path: "shivam/g") {nodes {list {name}}}}""" assert client.query(query) == { "graph": { - "nodes": {"list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}, {"name": "shivam"}]} + "nodes": { + "list": [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + {"name": "shivam"}, + ] + } } } @@ -191,7 +216,9 @@ def assert_graph_fetch(path): query = f"""{{ graph(path: "{path}") {{ nodes {{ list {{ name }} }} }} }}""" assert client.query(query) == { "graph": { - "nodes": {"list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}]} + "nodes": { + "list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}] + } } } @@ -202,7 +229,7 @@ def assert_graph_fetch(path): path = "g" tmp_work_dir = tempfile.mkdtemp() - with RaphtoryServer(tmp_work_dir).start(): + with GraphServer(tmp_work_dir).start(): client = RaphtoryClient("http://localhost:1736") # Default namespace, graph is saved in the work dir @@ -237,17 +264,23 @@ def assert_graph_fetch(path): path = "../shivam/g" with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) - assert "References to the parent dir are not allowed within the path:" in str(excinfo.value) + assert "References to the parent dir are not allowed within the path:" in str( + excinfo.value + ) path = "./shivam/g" with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) - assert "References to the current dir are not allowed within the path" in str(excinfo.value) + assert "References to the current dir are not allowed within the path" in str( + excinfo.value + ) path = "shivam/../../../../investigation/g" with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) - assert "References to the parent dir are not allowed within the path:" in str(excinfo.value) + assert "References to the parent dir are not allowed within the path:" in str( + excinfo.value + ) path = "//shivam/investigation/g" with pytest.raises(Exception) as excinfo: @@ -264,7 +297,7 @@ def assert_graph_fetch(path): client.send_graph(path=path, graph=g, overwrite=True) assert "Backslash not allowed in path" in str(excinfo.value) - #Test if we can escape through a symlink + # Test if we can escape through a symlink tmp_dir2 = tempfile.mkdtemp() nested_dir = os.path.join(tmp_work_dir, "shivam", "graphs") os.makedirs(nested_dir) @@ -288,14 +321,16 @@ def test_upload_graph_succeeds_if_no_graph_found_with_same_name(): g.save_to_file(g_file_path) tmp_work_dir = tempfile.mkdtemp() - with RaphtoryServer(tmp_work_dir).start(): + with GraphServer(tmp_work_dir).start(): client = RaphtoryClient("http://localhost:1736") client.upload_graph(path="g", file_path=g_file_path, overwrite=False) query = """{graph(path: "g") {nodes {list {name}}}}""" assert client.query(query) == { "graph": { - "nodes": {"list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}]} + "nodes": { + "list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}] + } } } @@ -311,7 +346,7 @@ def test_upload_graph_fails_if_graph_already_exists(): tmp_work_dir = tempfile.mkdtemp() g.save_to_file(os.path.join(tmp_work_dir, "g")) - with RaphtoryServer(tmp_work_dir).start(): + with GraphServer(tmp_work_dir).start(): client = RaphtoryClient("http://localhost:1736") with pytest.raises(Exception) as excinfo: client.upload_graph(path="g", file_path=g_file_path) @@ -328,7 +363,7 @@ def test_upload_graph_succeeds_if_graph_already_exists_with_overwrite_enabled(): g.save_to_file(g_file_path) tmp_work_dir = tempfile.mkdtemp() - with RaphtoryServer(tmp_work_dir).start(): + with GraphServer(tmp_work_dir).start(): client = RaphtoryClient("http://localhost:1736") g = Graph() @@ -345,7 +380,14 @@ def test_upload_graph_succeeds_if_graph_already_exists_with_overwrite_enabled(): query = """{graph(path: "g") {nodes {list {name}}}}""" assert client.query(query) == { "graph": { - "nodes": {"list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}, {"name": "shivam"}]} + "nodes": { + "list": [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + {"name": "shivam"}, + ] + } } } @@ -361,14 +403,16 @@ def test_upload_graph_succeeds_if_no_graph_found_with_same_name_at_namespace(): g.save_to_file(g_file_path) tmp_work_dir = tempfile.mkdtemp() - with RaphtoryServer(tmp_work_dir).start(): + with GraphServer(tmp_work_dir).start(): client = RaphtoryClient("http://localhost:1736") client.upload_graph(path="shivam/g", file_path=g_file_path, overwrite=False) query = """{graph(path: "shivam/g") {nodes {list {name}}}}""" assert client.query(query) == { "graph": { - "nodes": {"list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}]} + "nodes": { + "list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}] + } } } @@ -385,7 +429,7 @@ def test_upload_graph_fails_if_graph_already_exists_at_namespace(): tmp_work_dir = tempfile.mkdtemp() os.makedirs(os.path.join(tmp_work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(tmp_work_dir, "shivam", "g")) - with RaphtoryServer(tmp_work_dir).start(): + with GraphServer(tmp_work_dir).start(): client = RaphtoryClient("http://localhost:1736") with pytest.raises(Exception) as excinfo: client.upload_graph(path="shivam/g", file_path=g_file_path, overwrite=False) @@ -401,7 +445,7 @@ def test_upload_graph_succeeds_if_graph_already_exists_at_namespace_with_overwri os.makedirs(os.path.join(tmp_work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(tmp_work_dir, "shivam", "g")) - with RaphtoryServer(tmp_work_dir).start(): + with GraphServer(tmp_work_dir).start(): client = RaphtoryClient("http://localhost:1736") g = Graph() @@ -418,13 +462,21 @@ def test_upload_graph_succeeds_if_graph_already_exists_at_namespace_with_overwri query = """{graph(path: "shivam/g") {nodes {list {name}}}}""" assert client.query(query) == { "graph": { - "nodes": {"list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}, {"name": "shivam"}]} + "nodes": { + "list": [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + {"name": "shivam"}, + ] + } } } + def test_get_graph_fails_if_graph_not_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """{ graph(path: "g1") { name, path, nodes { list { name } } } }""" @@ -435,10 +487,12 @@ def test_get_graph_fails_if_graph_not_found(): def test_get_graph_fails_if_graph_not_found_at_namespace(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() - query = """{ graph(path: "shivam/g1") { name, path, nodes { list { name } } } }""" + query = ( + """{ graph(path: "shivam/g1") { name, path, nodes { list { name } } } }""" + ) with pytest.raises(Exception) as excinfo: client.query(query) assert "Graph not found" in str(excinfo.value) @@ -446,7 +500,7 @@ def test_get_graph_fails_if_graph_not_found_at_namespace(): def test_get_graph_succeeds_if_graph_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() g = Graph() @@ -459,13 +513,19 @@ def test_get_graph_succeeds_if_graph_found(): query = """{ graph(path: "g1") { name, path, nodes { list { name } } } }""" assert client.query(query) == { - 'graph': {'name': 'g1', 'nodes': {'list': [{'name': 'ben'}, {'name': 'hamza'}, {'name': 'haaroon'}]}, - 'path': 'g1'}} + "graph": { + "name": "g1", + "nodes": { + "list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}] + }, + "path": "g1", + } + } def test_get_graph_succeeds_if_graph_found_at_namespace(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() g = Graph() @@ -476,28 +536,30 @@ def test_get_graph_succeeds_if_graph_found_at_namespace(): os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g2")) - query = """{ graph(path: "shivam/g2") { name, path, nodes { list { name } } } }""" + query = ( + """{ graph(path: "shivam/g2") { name, path, nodes { list { name } } } }""" + ) response = client.query(query) - assert response['graph']['name'] == 'g2' - assert response['graph']['nodes'] == {'list': [{'name': 'ben'}, {'name': 'hamza'}, {'name': 'haaroon'}]} - assert normalize_path(response['graph']['path']) == 'shivam/g2' + assert response["graph"]["name"] == "g2" + assert response["graph"]["nodes"] == { + "list": [{"name": "ben"}, {"name": "hamza"}, {"name": "haaroon"}] + } + assert normalize_path(response["graph"]["path"]) == "shivam/g2" def test_get_graphs_returns_emtpy_list_if_no_graphs_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() # Assert if no graphs are discoverable query = """{ graphs { name, path } }""" - assert client.query(query) == { - 'graphs': {'name': [], 'path': []} - } + assert client.query(query) == {"graphs": {"name": [], "path": []}} def test_get_graphs_returns_graph_list_if_graphs_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() g = Graph() @@ -514,22 +576,22 @@ def test_get_graphs_returns_graph_list_if_graphs_found(): query = """{ graphs { name, path } }""" response = client.query(query) sorted_response = { - 'graphs': { - 'name': sorted(response['graphs']['name']), - 'path': sorted(normalize_path(p) for p in response['graphs']['path']) + "graphs": { + "name": sorted(response["graphs"]["name"]), + "path": sorted(normalize_path(p) for p in response["graphs"]["path"]), } } assert sorted_response == { - 'graphs': { - 'name': ['g1', 'g2', 'g3'], - 'path': ['g1', 'shivam/g2', 'shivam/g3'] + "graphs": { + "name": ["g1", "g2", "g3"], + "path": ["g1", "shivam/g2", "shivam/g3"], } } def test_receive_graph_fails_if_no_graph_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """{ receiveGraph(path: "g2") }""" @@ -540,7 +602,7 @@ def test_receive_graph_fails_if_no_graph_found(): def test_receive_graph_succeeds_if_graph_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() g = Graph() @@ -551,17 +613,16 @@ def test_receive_graph_succeeds_if_graph_found(): g.save_to_file(os.path.join(work_dir, "g1")) query = """{ receiveGraph(path: "g1") }""" - received_graph = client.query(query)['receiveGraph'] + received_graph = client.query(query)["receiveGraph"] decoded_bytes = base64.b64decode(received_graph) - - g = Graph.from_bincode(decoded_bytes) + g = Graph.deserialise(decoded_bytes) assert g.nodes.name == ["ben", "hamza", "haaroon"] def test_receive_graph_using_client_api_succeeds_if_graph_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() g = Graph() @@ -576,7 +637,7 @@ def test_receive_graph_using_client_api_succeeds_if_graph_found(): def test_receive_graph_fails_if_no_graph_found_at_namespace(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """{ receiveGraph(path: "shivam/g2") }""" @@ -587,7 +648,7 @@ def test_receive_graph_fails_if_no_graph_found_at_namespace(): def test_receive_graph_succeeds_if_graph_found_at_namespace(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() g = Graph() @@ -599,17 +660,17 @@ def test_receive_graph_succeeds_if_graph_found_at_namespace(): g.save_to_file(os.path.join(work_dir, "shivam", "g2")) query = """{ receiveGraph(path: "shivam/g2") }""" - received_graph = client.query(query)['receiveGraph'] + received_graph = client.query(query)["receiveGraph"] decoded_bytes = base64.b64decode(received_graph) - g = Graph.from_bincode(decoded_bytes) + g = Graph.deserialise(decoded_bytes) assert g.nodes.name == ["ben", "hamza", "haaroon"] def test_move_graph_fails_if_graph_not_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") query = """mutation { @@ -634,7 +695,7 @@ def test_move_graph_fails_if_graph_with_same_name_already_exists(): g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "g6")) - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") query = """mutation { @@ -659,7 +720,7 @@ def test_move_graph_fails_if_graph_with_same_name_already_exists_at_same_namespa g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "ben", "g6")) - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") query = """mutation { @@ -685,7 +746,7 @@ def test_move_graph_fails_if_graph_with_same_name_already_exists_at_diff_namespa g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "shivam", "g6")) - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") query = """mutation { @@ -709,7 +770,7 @@ def test_move_graph_succeeds(): os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") # Assert if rename graph succeeds and old graph is deleted @@ -737,9 +798,18 @@ def test_move_graph_succeeds(): }}""" result = client.query(query) - assert result['graph']['nodes']['list'] == [{'name': 'ben'}, {"name": "hamza"}, {'name': 'haaroon'}] - assert result['graph']['properties']['constant']['lastUpdated']['value'] is not None - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None + assert result["graph"]["nodes"]["list"] == [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + ] + assert ( + result["graph"]["properties"]["constant"]["lastUpdated"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) def test_move_graph_using_client_api_succeeds(): @@ -752,7 +822,7 @@ def test_move_graph_using_client_api_succeeds(): os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") # Assert if rename graph succeeds and old graph is deleted @@ -774,9 +844,18 @@ def test_move_graph_using_client_api_succeeds(): }}""" result = client.query(query) - assert result['graph']['nodes']['list'] == [{'name': 'ben'}, {"name": "hamza"}, {'name': 'haaroon'}] - assert result['graph']['properties']['constant']['lastUpdated']['value'] is not None - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None + assert result["graph"]["nodes"]["list"] == [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + ] + assert ( + result["graph"]["properties"]["constant"]["lastUpdated"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) def test_move_graph_succeeds_at_same_namespace_as_graph(): @@ -790,7 +869,7 @@ def test_move_graph_succeeds_at_same_namespace_as_graph(): g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") # Assert if rename graph succeeds and old graph is deleted @@ -818,9 +897,18 @@ def test_move_graph_succeeds_at_same_namespace_as_graph(): }}""" result = client.query(query) - assert result['graph']['nodes']['list'] == [{'name': 'ben'}, {"name": "hamza"}, {'name': 'haaroon'}] - assert result['graph']['properties']['constant']['lastUpdated']['value'] is not None - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None + assert result["graph"]["nodes"]["list"] == [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + ] + assert ( + result["graph"]["properties"]["constant"]["lastUpdated"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) def test_move_graph_succeeds_at_diff_namespace_as_graph(): @@ -835,7 +923,7 @@ def test_move_graph_succeeds_at_diff_namespace_as_graph(): g.save_to_file(os.path.join(work_dir, "ben", "g3")) - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") # Assert if rename graph succeeds and old graph is deleted @@ -863,14 +951,23 @@ def test_move_graph_succeeds_at_diff_namespace_as_graph(): }}""" result = client.query(query) - assert result['graph']['nodes']['list'] == [{'name': 'ben'}, {"name": "hamza"}, {'name': 'haaroon'}] - assert result['graph']['properties']['constant']['lastUpdated']['value'] is not None - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None + assert result["graph"]["nodes"]["list"] == [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + ] + assert ( + result["graph"]["properties"]["constant"]["lastUpdated"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) def test_copy_graph_fails_if_graph_not_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") query = """mutation { @@ -895,7 +992,7 @@ def test_copy_graph_fails_if_graph_with_same_name_already_exists(): g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "g6")) - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") query = """mutation { @@ -920,7 +1017,7 @@ def test_copy_graph_fails_if_graph_with_same_name_already_exists_at_same_namespa g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "ben", "g6")) - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") query = """mutation { @@ -946,7 +1043,7 @@ def test_copy_graph_fails_if_graph_with_same_name_already_exists_at_diff_namespa g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "shivam", "g6")) - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") query = """mutation { @@ -970,7 +1067,7 @@ def test_copy_graph_succeeds(): os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") # Assert if copy graph succeeds and old graph is retained @@ -984,7 +1081,11 @@ def test_copy_graph_succeeds(): query = """{graph(path: "shivam/g3") { nodes {list {name}} }}""" result = client.query(query) - assert result['graph']['nodes']['list'] == [{'name': 'ben'}, {"name": "hamza"}, {'name': 'haaroon'}] + assert result["graph"]["nodes"]["list"] == [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + ] query = """{graph(path: "g4") { nodes {list {name}} @@ -996,8 +1097,14 @@ def test_copy_graph_succeeds(): }}""" result = client.query(query) - assert result['graph']['nodes']['list'] == [{'name': 'ben'}, {"name": "hamza"}, {'name': 'haaroon'}] - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None + assert result["graph"]["nodes"]["list"] == [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + ] + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) def test_copy_graph_using_client_api_succeeds(): @@ -1010,7 +1117,7 @@ def test_copy_graph_using_client_api_succeeds(): os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") # Assert if copy graph succeeds and old graph is retained @@ -1018,7 +1125,11 @@ def test_copy_graph_using_client_api_succeeds(): query = """{graph(path: "shivam/g3") { nodes {list {name}} }}""" result = client.query(query) - assert result['graph']['nodes']['list'] == [{'name': 'ben'}, {"name": "hamza"}, {'name': 'haaroon'}] + assert result["graph"]["nodes"]["list"] == [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + ] query = """{graph(path: "ben/g4") { nodes {list {name}} @@ -1030,8 +1141,14 @@ def test_copy_graph_using_client_api_succeeds(): }}""" result = client.query(query) - assert result['graph']['nodes']['list'] == [{'name': 'ben'}, {"name": "hamza"}, {'name': 'haaroon'}] - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None + assert result["graph"]["nodes"]["list"] == [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + ] + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) def test_copy_graph_succeeds_at_same_namespace_as_graph(): @@ -1045,7 +1162,7 @@ def test_copy_graph_succeeds_at_same_namespace_as_graph(): g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") # Assert if rename graph succeeds and old graph is deleted @@ -1059,7 +1176,11 @@ def test_copy_graph_succeeds_at_same_namespace_as_graph(): query = """{graph(path: "shivam/g3") { nodes {list {name}} }}""" result = client.query(query) - assert result['graph']['nodes']['list'] == [{'name': 'ben'}, {"name": "hamza"}, {'name': 'haaroon'}] + assert result["graph"]["nodes"]["list"] == [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + ] query = """{graph(path: "shivam/g4") { nodes {list {name}} @@ -1071,8 +1192,14 @@ def test_copy_graph_succeeds_at_same_namespace_as_graph(): }}""" result = client.query(query) - assert result['graph']['nodes']['list'] == [{'name': 'ben'}, {"name": "hamza"}, {'name': 'haaroon'}] - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None + assert result["graph"]["nodes"]["list"] == [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + ] + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) def test_copy_graph_succeeds_at_diff_namespace_as_graph(): @@ -1087,7 +1214,7 @@ def test_copy_graph_succeeds_at_diff_namespace_as_graph(): g.save_to_file(os.path.join(work_dir, "ben", "g3")) - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") # Assert if rename graph succeeds and old graph is deleted @@ -1101,7 +1228,11 @@ def test_copy_graph_succeeds_at_diff_namespace_as_graph(): query = """{graph(path: "ben/g3") { nodes {list {name}} }}""" result = client.query(query) - assert result['graph']['nodes']['list'] == [{'name': 'ben'}, {"name": "hamza"}, {'name': 'haaroon'}] + assert result["graph"]["nodes"]["list"] == [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + ] query = """{graph(path: "shivam/g4") { nodes {list {name}} @@ -1113,13 +1244,19 @@ def test_copy_graph_succeeds_at_diff_namespace_as_graph(): }}""" result = client.query(query) - assert result['graph']['nodes']['list'] == [{'name': 'ben'}, {"name": "hamza"}, {'name': 'haaroon'}] - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None + assert result["graph"]["nodes"]["list"] == [ + {"name": "ben"}, + {"name": "hamza"}, + {"name": "haaroon"}, + ] + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) def test_delete_graph_fails_if_graph_not_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") query = """mutation { @@ -1134,7 +1271,7 @@ def test_delete_graph_fails_if_graph_not_found(): def test_delete_graph_succeeds_if_graph_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") g = Graph() @@ -1159,7 +1296,7 @@ def test_delete_graph_succeeds_if_graph_found(): def test_delete_graph_using_client_api_succeeds_if_graph_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") g = Graph() @@ -1178,7 +1315,7 @@ def test_delete_graph_using_client_api_succeeds_if_graph_found(): def test_delete_graph_succeeds_if_graph_found_at_namespace(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start(): + with GraphServer(work_dir).start(): client = RaphtoryClient("http://localhost:1736") g = Graph() @@ -1203,7 +1340,7 @@ def test_delete_graph_succeeds_if_graph_found_at_namespace(): def test_create_graph_fail_if_parent_graph_not_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { createGraph( @@ -1221,7 +1358,7 @@ def test_create_graph_fail_if_parent_graph_not_found(): def test_create_graph_fail_if_parent_graph_not_found_at_namespace(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { createGraph( @@ -1244,7 +1381,7 @@ def test_create_graph_fail_if_graph_already_exists(): g.save_to_file(os.path.join(work_dir, "g0")) g.save_to_file(os.path.join(work_dir, "g3")) - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { createGraph( @@ -1263,12 +1400,12 @@ def test_create_graph_fail_if_graph_already_exists(): def test_create_graph_fail_if_graph_already_exists_at_namespace(): work_dir = tempfile.mkdtemp() os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) - + g = Graph() g.save_to_file(os.path.join(work_dir, "g0")) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - - with RaphtoryServer(work_dir).start() as server: + + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { createGraph( @@ -1297,7 +1434,7 @@ def test_create_graph_succeeds(): g.save_to_file(os.path.join(work_dir, "g1")) - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { @@ -1331,16 +1468,35 @@ def test_create_graph_succeeds(): }""" result = client.query(query) - assert result['graph']['nodes']['list'] == [ - {'name': 'ben', 'properties': {'temporal': {'get': {'values': ['engineering']}}}}, - {'name': 'hamza', 'properties': {'temporal': {'get': {'values': ['director']}}}} + assert result["graph"]["nodes"]["list"] == [ + { + "name": "ben", + "properties": {"temporal": {"get": {"values": ["engineering"]}}}, + }, + { + "name": "hamza", + "properties": {"temporal": {"get": {"values": ["director"]}}}, + }, + ] + assert result["graph"]["edges"]["list"] == [ + {"properties": {"temporal": {"get": {"values": ["1"]}}}} ] - assert result['graph']['edges']['list'] == [{'properties': {'temporal': {'get': {'values': ['1']}}}}] - assert result['graph']['properties']['constant']['creationTime']['value'] is not None - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None - assert result['graph']['properties']['constant']['lastUpdated']['value'] is not None - assert result['graph']['properties']['constant']['uiProps']['value'] == '{ "target": 6 : }' - assert result['graph']['properties']['constant']['isArchive']['value'] == 1 + assert ( + result["graph"]["properties"]["constant"]["creationTime"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastUpdated"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["uiProps"]["value"] + == '{ "target": 6 : }' + ) + assert result["graph"]["properties"]["constant"]["isArchive"]["value"] == 1 def test_create_graph_succeeds_at_namespace(): @@ -1356,7 +1512,7 @@ def test_create_graph_succeeds_at_namespace(): g.save_to_file(os.path.join(work_dir, "g1")) - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { @@ -1390,22 +1546,41 @@ def test_create_graph_succeeds_at_namespace(): }""" result = client.query(query) - assert result['graph']['nodes']['list'] == [ - {'name': 'ben', 'properties': {'temporal': {'get': {'values': ['engineering']}}}}, - {'name': 'hamza', 'properties': {'temporal': {'get': {'values': ['director']}}}} + assert result["graph"]["nodes"]["list"] == [ + { + "name": "ben", + "properties": {"temporal": {"get": {"values": ["engineering"]}}}, + }, + { + "name": "hamza", + "properties": {"temporal": {"get": {"values": ["director"]}}}, + }, + ] + assert result["graph"]["edges"]["list"] == [ + {"properties": {"temporal": {"get": {"values": ["1"]}}}} ] - assert result['graph']['edges']['list'] == [{'properties': {'temporal': {'get': {'values': ['1']}}}}] - assert result['graph']['properties']['constant']['creationTime']['value'] is not None - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None - assert result['graph']['properties']['constant']['lastUpdated']['value'] is not None - assert result['graph']['properties']['constant']['uiProps']['value'] == '{ "target": 6 : }' - assert result['graph']['properties']['constant']['isArchive']['value'] == 1 + assert ( + result["graph"]["properties"]["constant"]["creationTime"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastUpdated"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["uiProps"]["value"] + == '{ "target": 6 : }' + ) + assert result["graph"]["properties"]["constant"]["isArchive"]["value"] == 1 # Update Graph with new graph name tests (save as new graph name) def test_update_graph_with_new_graph_name_fails_if_parent_graph_not_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { updateGraph( @@ -1426,7 +1601,7 @@ def test_update_graph_with_new_graph_name_fails_if_current_graph_not_found(): g = Graph() work_dir = tempfile.mkdtemp() g.save_to_file(os.path.join(work_dir, "g1")) - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { updateGraph( @@ -1455,7 +1630,7 @@ def test_update_graph_with_new_graph_name_fails_if_new_graph_already_exists(): g.save_to_file(os.path.join(work_dir, "shivam", "g2")) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { @@ -1488,7 +1663,7 @@ def test_update_graph_with_new_graph_name_succeeds_if_parent_graph_belongs_to_di g.save_to_file(os.path.join(work_dir, "g1")) g.save_to_file(os.path.join(work_dir, "shivam", "g2")) - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { @@ -1523,16 +1698,35 @@ def test_update_graph_with_new_graph_name_succeeds_if_parent_graph_belongs_to_di }""" result = client.query(query) - assert result['graph']['nodes']['list'] == [ - {'name': 'ben', 'properties': {'temporal': {'get': {'values': ['engineering']}}}}, - {'name': 'hamza', 'properties': {'temporal': {'get': {'values': ['director']}}}} + assert result["graph"]["nodes"]["list"] == [ + { + "name": "ben", + "properties": {"temporal": {"get": {"values": ["engineering"]}}}, + }, + { + "name": "hamza", + "properties": {"temporal": {"get": {"values": ["director"]}}}, + }, + ] + assert result["graph"]["edges"]["list"] == [ + {"properties": {"temporal": {"get": {"values": ["1"]}}}} ] - assert result['graph']['edges']['list'] == [{'properties': {'temporal': {'get': {'values': ['1']}}}}] - assert result['graph']['properties']['constant']['creationTime']['value'] is not None - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None - assert result['graph']['properties']['constant']['lastUpdated']['value'] is not None - assert result['graph']['properties']['constant']['uiProps']['value'] == '{ "target": 6 : }' - assert result['graph']['properties']['constant']['isArchive']['value'] == 1 + assert ( + result["graph"]["properties"]["constant"]["creationTime"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastUpdated"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["uiProps"]["value"] + == '{ "target": 6 : }' + ) + assert result["graph"]["properties"]["constant"]["isArchive"]["value"] == 1 def test_update_graph_with_new_graph_name_succeeds_if_parent_graph_belongs_to_same_namespace(): @@ -1550,7 +1744,7 @@ def test_update_graph_with_new_graph_name_succeeds_if_parent_graph_belongs_to_sa g.save_to_file(os.path.join(work_dir, "shivam", "g2")) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { @@ -1584,16 +1778,35 @@ def test_update_graph_with_new_graph_name_succeeds_if_parent_graph_belongs_to_sa } }""" result = client.query(query) - assert result['graph']['nodes']['list'] == [ - {'name': 'ben', 'properties': {'temporal': {'get': {'values': ['engineering']}}}}, - {'name': 'hamza', 'properties': {'temporal': {'get': {'values': ['director']}}}} + assert result["graph"]["nodes"]["list"] == [ + { + "name": "ben", + "properties": {"temporal": {"get": {"values": ["engineering"]}}}, + }, + { + "name": "hamza", + "properties": {"temporal": {"get": {"values": ["director"]}}}, + }, ] - assert result['graph']['edges']['list'] == [{'properties': {'temporal': {'get': {'values': ['1']}}}}] - assert result['graph']['properties']['constant']['creationTime']['value'] is not None - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None - assert result['graph']['properties']['constant']['lastUpdated']['value'] is not None - assert result['graph']['properties']['constant']['uiProps']['value'] == '{ "target": 6 : }' - assert result['graph']['properties']['constant']['isArchive']['value'] == 1 + assert result["graph"]["edges"]["list"] == [ + {"properties": {"temporal": {"get": {"values": ["1"]}}}} + ] + assert ( + result["graph"]["properties"]["constant"]["creationTime"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastUpdated"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["uiProps"]["value"] + == '{ "target": 6 : }' + ) + assert result["graph"]["properties"]["constant"]["isArchive"]["value"] == 1 def test_update_graph_with_new_graph_name_succeeds_with_new_node_from_parent_graph_added_to_new_graph(): @@ -1620,7 +1833,7 @@ def test_update_graph_with_new_graph_name_succeeds_with_new_node_from_parent_gra os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g2")) - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { @@ -1655,16 +1868,33 @@ def test_update_graph_with_new_graph_name_succeeds_with_new_node_from_parent_gra }""" result = client.query(query) - assert result['graph']['nodes']['list'] == [ - {'name': 'ben', 'properties': {'temporal': {'get': {'values': ['engineering']}}}}, - {'name': 'shivam', 'properties': {'temporal': {'get': {'values': ['engineering']}}}} + assert result["graph"]["nodes"]["list"] == [ + { + "name": "ben", + "properties": {"temporal": {"get": {"values": ["engineering"]}}}, + }, + { + "name": "shivam", + "properties": {"temporal": {"get": {"values": ["engineering"]}}}, + }, ] - assert result['graph']['edges']['list'] == [] - assert result['graph']['properties']['constant']['creationTime']['value'] is not None - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None - assert result['graph']['properties']['constant']['lastUpdated']['value'] is not None - assert result['graph']['properties']['constant']['uiProps']['value'] == '{ "target": 6 : }' - assert result['graph']['properties']['constant']['isArchive']['value'] == 1 + assert result["graph"]["edges"]["list"] == [] + assert ( + result["graph"]["properties"]["constant"]["creationTime"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastUpdated"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["uiProps"]["value"] + == '{ "target": 6 : }' + ) + assert result["graph"]["properties"]["constant"]["isArchive"]["value"] == 1 def test_update_graph_with_new_graph_name_succeeds_with_new_node_removed_from_new_graph(): @@ -1682,7 +1912,7 @@ def test_update_graph_with_new_graph_name_succeeds_with_new_node_removed_from_ne g.save_to_file(os.path.join(work_dir, "g1")) g.save_to_file(os.path.join(work_dir, "shivam", "g2")) - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { @@ -1717,22 +1947,41 @@ def test_update_graph_with_new_graph_name_succeeds_with_new_node_removed_from_ne }""" result = client.query(query) - assert result['graph']['nodes']['list'] == [ - {'name': 'ben', 'properties': {'temporal': {'get': {'values': ['engineering']}}}}, - {'name': 'hamza', 'properties': {'temporal': {'get': {'values': ['director']}}}} + assert result["graph"]["nodes"]["list"] == [ + { + "name": "ben", + "properties": {"temporal": {"get": {"values": ["engineering"]}}}, + }, + { + "name": "hamza", + "properties": {"temporal": {"get": {"values": ["director"]}}}, + }, + ] + assert result["graph"]["edges"]["list"] == [ + {"properties": {"temporal": {"get": {"values": ["1"]}}}} ] - assert result['graph']['edges']['list'] == [{'properties': {'temporal': {'get': {'values': ['1']}}}}] - assert result['graph']['properties']['constant']['creationTime']['value'] is not None - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None - assert result['graph']['properties']['constant']['lastUpdated']['value'] is not None - assert result['graph']['properties']['constant']['uiProps']['value'] == '{ "target": 6 : }' - assert result['graph']['properties']['constant']['isArchive']['value'] == 1 + assert ( + result["graph"]["properties"]["constant"]["creationTime"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastUpdated"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["uiProps"]["value"] + == '{ "target": 6 : }' + ) + assert result["graph"]["properties"]["constant"]["isArchive"]["value"] == 1 # Update Graph tests (save graph as same graph name) def test_update_graph_fails_if_parent_graph_not_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { updateGraph( @@ -1753,7 +2002,7 @@ def test_update_graph_fails_if_current_graph_not_found(): g = Graph() work_dir = tempfile.mkdtemp() g.save_to_file(os.path.join(work_dir, "g1")) - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { updateGraph( @@ -1785,7 +2034,7 @@ def test_update_graph_succeeds_if_parent_graph_belongs_to_different_namespace(): g.save_to_file(os.path.join(work_dir, "g1")) g.save_to_file(os.path.join(work_dir, "shivam", "g2")) - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { @@ -1820,15 +2069,31 @@ def test_update_graph_succeeds_if_parent_graph_belongs_to_different_namespace(): }""" result = client.query(query) - assert result['graph']['nodes']['list'] == [ - {'name': 'ben', 'properties': {'temporal': {'get': {'values': ['engineering']}}}}, - {'name': 'hamza', 'properties': {'temporal': {'get': {'values': ['director']}}}} + assert result["graph"]["nodes"]["list"] == [ + { + "name": "ben", + "properties": {"temporal": {"get": {"values": ["engineering"]}}}, + }, + { + "name": "hamza", + "properties": {"temporal": {"get": {"values": ["director"]}}}, + }, + ] + assert result["graph"]["edges"]["list"] == [ + {"properties": {"temporal": {"get": {"values": ["1"]}}}} ] - assert result['graph']['edges']['list'] == [{'properties': {'temporal': {'get': {'values': ['1']}}}}] - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None - assert result['graph']['properties']['constant']['lastUpdated']['value'] is not None - assert result['graph']['properties']['constant']['uiProps']['value'] == '{ "target": 6 : }' - assert result['graph']['properties']['constant']['isArchive']['value'] == 1 + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastUpdated"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["uiProps"]["value"] + == '{ "target": 6 : }' + ) + assert result["graph"]["properties"]["constant"]["isArchive"]["value"] == 1 def test_update_graph_succeeds_if_parent_graph_belongs_to_same_namespace(): @@ -1846,7 +2111,7 @@ def test_update_graph_succeeds_if_parent_graph_belongs_to_same_namespace(): g.save_to_file(os.path.join(work_dir, "shivam", "g2")) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { @@ -1880,15 +2145,31 @@ def test_update_graph_succeeds_if_parent_graph_belongs_to_same_namespace(): } }""" result = client.query(query) - assert result['graph']['nodes']['list'] == [ - {'name': 'ben', 'properties': {'temporal': {'get': {'values': ['engineering']}}}}, - {'name': 'hamza', 'properties': {'temporal': {'get': {'values': ['director']}}}} + assert result["graph"]["nodes"]["list"] == [ + { + "name": "ben", + "properties": {"temporal": {"get": {"values": ["engineering"]}}}, + }, + { + "name": "hamza", + "properties": {"temporal": {"get": {"values": ["director"]}}}, + }, + ] + assert result["graph"]["edges"]["list"] == [ + {"properties": {"temporal": {"get": {"values": ["1"]}}}} ] - assert result['graph']['edges']['list'] == [{'properties': {'temporal': {'get': {'values': ['1']}}}}] - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None - assert result['graph']['properties']['constant']['lastUpdated']['value'] is not None - assert result['graph']['properties']['constant']['uiProps']['value'] == '{ "target": 6 : }' - assert result['graph']['properties']['constant']['isArchive']['value'] == 1 + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastUpdated"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["uiProps"]["value"] + == '{ "target": 6 : }' + ) + assert result["graph"]["properties"]["constant"]["isArchive"]["value"] == 1 def test_update_graph_succeeds_with_new_node_from_parent_graph_added_to_new_graph(): @@ -1914,7 +2195,7 @@ def test_update_graph_succeeds_with_new_node_from_parent_graph_added_to_new_grap os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g2")) - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { @@ -1949,15 +2230,29 @@ def test_update_graph_succeeds_with_new_node_from_parent_graph_added_to_new_grap }""" result = client.query(query) - assert result['graph']['nodes']['list'] == [ - {'name': 'ben', 'properties': {'temporal': {'get': {'values': ['engineering']}}}}, - {'name': 'shivam', 'properties': {'temporal': {'get': {'values': ['engineering']}}}} + assert result["graph"]["nodes"]["list"] == [ + { + "name": "ben", + "properties": {"temporal": {"get": {"values": ["engineering"]}}}, + }, + { + "name": "shivam", + "properties": {"temporal": {"get": {"values": ["engineering"]}}}, + }, ] - assert result['graph']['edges']['list'] == [] - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None - assert result['graph']['properties']['constant']['lastUpdated']['value'] is not None - assert result['graph']['properties']['constant']['uiProps']['value'] == '{ "target": 6 : }' - assert result['graph']['properties']['constant']['isArchive']['value'] == 1 + assert result["graph"]["edges"]["list"] == [] + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastUpdated"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["uiProps"]["value"] + == '{ "target": 6 : }' + ) + assert result["graph"]["properties"]["constant"]["isArchive"]["value"] == 1 def test_update_graph_succeeds_with_new_node_removed_from_new_graph(): @@ -1975,7 +2270,7 @@ def test_update_graph_succeeds_with_new_node_removed_from_new_graph(): g.save_to_file(os.path.join(work_dir, "g1")) g.save_to_file(os.path.join(work_dir, "shivam", "g2")) - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { @@ -2010,19 +2305,30 @@ def test_update_graph_succeeds_with_new_node_removed_from_new_graph(): }""" result = client.query(query) - assert result['graph']['nodes']['list'] == [ - {'name': 'ben', 'properties': {'temporal': {'get': {'values': ['engineering']}}}}, + assert result["graph"]["nodes"]["list"] == [ + { + "name": "ben", + "properties": {"temporal": {"get": {"values": ["engineering"]}}}, + }, ] - assert result['graph']['edges']['list'] == [] - assert result['graph']['properties']['constant']['lastOpened']['value'] is not None - assert result['graph']['properties']['constant']['lastUpdated']['value'] is not None - assert result['graph']['properties']['constant']['uiProps']['value'] == '{ "target": 6 : }' - assert result['graph']['properties']['constant']['isArchive']['value'] == 1 + assert result["graph"]["edges"]["list"] == [] + assert ( + result["graph"]["properties"]["constant"]["lastOpened"]["value"] is not None + ) + assert ( + result["graph"]["properties"]["constant"]["lastUpdated"]["value"] + is not None + ) + assert ( + result["graph"]["properties"]["constant"]["uiProps"]["value"] + == '{ "target": 6 : }' + ) + assert result["graph"]["properties"]["constant"]["isArchive"]["value"] == 1 def test_update_graph_last_opened_fails_if_graph_not_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { updateGraphLastOpened(path: "g1") }""" @@ -2033,7 +2339,7 @@ def test_update_graph_last_opened_fails_if_graph_not_found(): def test_update_graph_last_opened_fails_if_graph_not_found_at_namespace(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { updateGraphLastOpened(path: "shivam/g1") }""" @@ -2044,7 +2350,7 @@ def test_update_graph_last_opened_fails_if_graph_not_found_at_namespace(): def test_update_graph_last_opened_succeeds(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() g = Graph() @@ -2059,18 +2365,24 @@ def test_update_graph_last_opened_succeeds(): query_last_opened = """{ graph(path: "g1") { properties { constant { get(key: "lastOpened") { value } } } } }""" mutate_last_opened = """mutation { updateGraphLastOpened(path: "g1") }""" - assert client.query(query_last_opened) == {'graph': {'properties': {'constant': {'get': None}}}} - assert client.query(mutate_last_opened) == {'updateGraphLastOpened': True} - updated_last_opened1 = client.query(query_last_opened)['graph']['properties']['constant']['get']['value'] + assert client.query(query_last_opened) == { + "graph": {"properties": {"constant": {"get": None}}} + } + assert client.query(mutate_last_opened) == {"updateGraphLastOpened": True} + updated_last_opened1 = client.query(query_last_opened)["graph"]["properties"][ + "constant" + ]["get"]["value"] time.sleep(1) - assert client.query(mutate_last_opened) == {'updateGraphLastOpened': True} - updated_last_opened2 = client.query(query_last_opened)['graph']['properties']['constant']['get']['value'] + assert client.query(mutate_last_opened) == {"updateGraphLastOpened": True} + updated_last_opened2 = client.query(query_last_opened)["graph"]["properties"][ + "constant" + ]["get"]["value"] assert updated_last_opened2 > updated_last_opened1 def test_update_graph_last_opened_succeeds_at_namespace(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() g = Graph() @@ -2084,18 +2396,24 @@ def test_update_graph_last_opened_succeeds_at_namespace(): query_last_opened = """{ graph(path: "shivam/g2") { properties { constant { get(key: "lastOpened") { value } } } } }""" mutate_last_opened = """mutation { updateGraphLastOpened(path: "shivam/g2") }""" - assert client.query(query_last_opened) == {'graph': {'properties': {'constant': {'get': None}}}} - assert client.query(mutate_last_opened) == {'updateGraphLastOpened': True} - updated_last_opened1 = client.query(query_last_opened)['graph']['properties']['constant']['get']['value'] + assert client.query(query_last_opened) == { + "graph": {"properties": {"constant": {"get": None}}} + } + assert client.query(mutate_last_opened) == {"updateGraphLastOpened": True} + updated_last_opened1 = client.query(query_last_opened)["graph"]["properties"][ + "constant" + ]["get"]["value"] time.sleep(1) - assert client.query(mutate_last_opened) == {'updateGraphLastOpened': True} - updated_last_opened2 = client.query(query_last_opened)['graph']['properties']['constant']['get']['value'] + assert client.query(mutate_last_opened) == {"updateGraphLastOpened": True} + updated_last_opened2 = client.query(query_last_opened)["graph"]["properties"][ + "constant" + ]["get"]["value"] assert updated_last_opened2 > updated_last_opened1 def test_archive_graph_fails_if_graph_not_found(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { archiveGraph(path: "g1", isArchive: 0) }""" @@ -2106,7 +2424,7 @@ def test_archive_graph_fails_if_graph_not_found(): def test_archive_graph_fails_if_graph_not_found_at_namespace(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() query = """mutation { archiveGraph(path: "shivam/g1", isArchive: 0) }""" @@ -2117,7 +2435,7 @@ def test_archive_graph_fails_if_graph_not_found_at_namespace(): def test_archive_graph_succeeds(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() g = Graph() @@ -2130,18 +2448,30 @@ def test_archive_graph_succeeds(): g.save_to_file(os.path.join(work_dir, "shivam", "g2")) query_is_archive = """{ graph(path: "g1") { properties { constant { get(key: "isArchive") { value } } } } }""" - assert client.query(query_is_archive) == {'graph': {'properties': {'constant': {'get': None}}}} + assert client.query(query_is_archive) == { + "graph": {"properties": {"constant": {"get": None}}} + } update_archive_graph = """mutation { archiveGraph(path: "g1", isArchive: 0) }""" assert client.query(update_archive_graph) == {"archiveGraph": True} - assert client.query(query_is_archive)['graph']['properties']['constant']['get']['value'] == 0 + assert ( + client.query(query_is_archive)["graph"]["properties"]["constant"]["get"][ + "value" + ] + == 0 + ) update_archive_graph = """mutation { archiveGraph(path: "g1", isArchive: 1) }""" assert client.query(update_archive_graph) == {"archiveGraph": True} - assert client.query(query_is_archive)['graph']['properties']['constant']['get']['value'] == 1 + assert ( + client.query(query_is_archive)["graph"]["properties"]["constant"]["get"][ + "value" + ] + == 1 + ) def test_archive_graph_succeeds_at_namespace(): work_dir = tempfile.mkdtemp() - with RaphtoryServer(work_dir).start() as server: + with GraphServer(work_dir).start() as server: client = server.get_client() g = Graph() @@ -2154,13 +2484,29 @@ def test_archive_graph_succeeds_at_namespace(): g.save_to_file(os.path.join(work_dir, "shivam", "g2")) query_is_archive = """{ graph(path: "shivam/g2") { properties { constant { get(key: "isArchive") { value } } } } }""" - assert client.query(query_is_archive) == {'graph': {'properties': {'constant': {'get': None}}}} - update_archive_graph = """mutation { archiveGraph(path: "shivam/g2", isArchive: 0) }""" + assert client.query(query_is_archive) == { + "graph": {"properties": {"constant": {"get": None}}} + } + update_archive_graph = ( + """mutation { archiveGraph(path: "shivam/g2", isArchive: 0) }""" + ) assert client.query(update_archive_graph) == {"archiveGraph": True} - assert client.query(query_is_archive)['graph']['properties']['constant']['get']['value'] == 0 - update_archive_graph = """mutation { archiveGraph(path: "shivam/g2", isArchive: 1) }""" + assert ( + client.query(query_is_archive)["graph"]["properties"]["constant"]["get"][ + "value" + ] + == 0 + ) + update_archive_graph = ( + """mutation { archiveGraph(path: "shivam/g2", isArchive: 1) }""" + ) assert client.query(update_archive_graph) == {"archiveGraph": True} - assert client.query(query_is_archive)['graph']['properties']['constant']['get']['value'] == 1 + assert ( + client.query(query_is_archive)["graph"]["properties"]["constant"]["get"][ + "value" + ] + == 1 + ) def test_graph_windows_and_layers_query(): @@ -2172,7 +2518,7 @@ def test_graph_windows_and_layers_query(): g2.add_edge(1, 2, 3, layer="layer2") tmp_work_dir = tempfile.mkdtemp() - with RaphtoryServer(tmp_work_dir).start() as server: + with GraphServer(tmp_work_dir).start() as server: client = server.get_client() client.send_graph(path="lotr", graph=g1) client.send_graph(path="layers", graph=g2) @@ -2270,7 +2616,7 @@ def test_graph_properties_query(): n.add_constant_properties({"prop5": "val4"}) tmp_work_dir = tempfile.mkdtemp() - with RaphtoryServer(tmp_work_dir).start() as server: + with GraphServer(tmp_work_dir).start() as server: client = server.get_client() client.send_graph(path="g", graph=g) q = """ @@ -2347,6 +2693,7 @@ def test_graph_properties_query(): key=lambda x: x["key"], ) + # def test_disk_graph_name(): # import pandas as pd # from raphtory import DiskGraphStorage @@ -2380,6 +2727,6 @@ def test_graph_properties_query(): # ).sort_values(["src", "dst", "time"]) # g= DiskGraphStorage.load_from_pandas(dir, edges, "src", "dst", "time") # tmp_work_dir = tempfile.mkdtemp() -# with RaphtoryServer(tmp_work_dir).start() as server: +# with GraphServer(tmp_work_dir).start() as server: # client = server.get_client() -# client.upload_graph(path="g", graph=g) \ No newline at end of file +# client.upload_graph(path="g", graph=g) diff --git a/python/tests/test_iterables.py b/python/tests/test_iterables.py index 8e8874a63e..c1d0edb0cd 100644 --- a/python/tests/test_iterables.py +++ b/python/tests/test_iterables.py @@ -148,7 +148,7 @@ def test_propiterable(): assert sorted(total) == [2, 17, 18, 35, 38] total = dict(zip(g.nodes.id, g.nodes.out_edges.properties.get("value_dec").sum())) - assert total == {'1': 32, '2': 5, '3': 3, '4': 15, '5': None} + assert total == {"1": 32, "2": 5, "3": 3, "4": 15, "5": None} total = g.nodes.out_edges.properties.get("value_dec").sum().sum() assert total == 55 diff --git a/python/tests/test_load_from_pandas.py b/python/tests/test_load_from_pandas.py index e966d1a21b..6df04d40e9 100644 --- a/python/tests/test_load_from_pandas.py +++ b/python/tests/test_load_from_pandas.py @@ -19,9 +19,9 @@ def test_load_from_pandas(): df = pd.DataFrame( { + "time": [1, 2, 3, 4, 5], "src": [1, 2, 3, 4, 5], "dst": [2, 3, 4, 5, 6], - "time": [1, 2, 3, 4, 5], "weight": [1.0, 2.0, 3.0, 4.0, 5.0], "marbles": ["red", "blue", "green", "yellow", "purple"], } @@ -46,12 +46,12 @@ def assertions(g): assert g.nodes.id.collect() == expected_nodes assert edges == expected_edges - g = Graph.load_from_pandas(df, "src", "dst", "time", ["weight", "marbles"]) + g = Graph() + g.load_edges_from_pandas(df, "time", "src", "dst", ["weight", "marbles"]) assertions(g) - g = PersistentGraph.load_from_pandas( - df, "src", "dst", "time", ["weight", "marbles"] - ) + g = PersistentGraph() + g.load_edges_from_pandas(df, "time", "src", "dst", ["weight", "marbles"]) assertions(g) @@ -77,13 +77,13 @@ def assertions(exc_info): # Use pytest.raises to expect an exception with pytest.raises(Exception) as exc_info: - g = Graph.load_from_pandas(df, "src", "dst", "time", ["weight", "marbles"]) + g = Graph() + g.load_edges_from_pandas(df, "time", "src", "dst", ["weight", "marbles"]) assertions(exc_info) with pytest.raises(Exception) as exc_info: - g = PersistentGraph.load_from_pandas( - df, "src", "dst", "time", ["weight", "marbles"] - ) + g = PersistentGraph() + g.load_edges_from_pandas(df, "time", "src", "dst", ["weight", "marbles"]) assertions(exc_info) # Optionally, you can check the exception message or type @@ -148,13 +148,17 @@ def assertions(g): assert nodes == expected_nodes g = Graph() - g.load_nodes_from_pandas(nodes_df, "id", "time", "node_type", properties=["name"]) - g.load_edges_from_pandas(edges_df, "src", "dst", "time", ["weight", "marbles"]) + g.load_nodes_from_pandas( + nodes_df, "time", "id", node_type_col="node_type", properties=["name"] + ) + g.load_edges_from_pandas(edges_df, "time", "src", "dst", ["weight", "marbles"]) assertions(g) g = PersistentGraph() - g.load_nodes_from_pandas(nodes_df, "id", "time", "node_type", properties=["name"]) - g.load_edges_from_pandas(edges_df, "src", "dst", "time", ["weight", "marbles"]) + g.load_nodes_from_pandas( + nodes_df, "time", "id", node_type_col="node_type", properties=["name"] + ) + g.load_edges_from_pandas(edges_df, "time", "src", "dst", ["weight", "marbles"]) assertions(g) @@ -210,32 +214,23 @@ def assertions(g): assert g.nodes.id.collect() == expected_node_ids assert edges == expected_edges - g = Graph.load_from_pandas( - edges_df, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_properties=["weight", "marbles"], - node_df=nodes_df, - node_id="id", - node_time="time", - node_properties=["name"], - node_type="node_type", + g = Graph() + g.load_edges_from_pandas( + edges_df, time="time", src="src", dst="dst", properties=["weight", "marbles"] + ) + g.load_nodes_from_pandas( + df=nodes_df, time="time", id="id", properties=["name"], node_type="node_type" ) assertions(g) - g = PersistentGraph.load_from_pandas( - edges_df, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_properties=["weight", "marbles"], - node_df=nodes_df, - node_id="id", - node_time="time", - node_properties=["name"], - node_type="node_type", + g = PersistentGraph() + g.load_edges_from_pandas( + edges_df, time="time", src="src", dst="dst", properties=["weight", "marbles"] ) + g.load_nodes_from_pandas( + df=nodes_df, time="time", id="id", properties=["name"], node_type="node_type" + ) + assertions(g) @@ -296,22 +291,44 @@ def assertions1(g): g = Graph() g.load_nodes_from_pandas( nodes_df, - "id", "time", - "node_type", + "id", + node_type_col="node_type", properties=["name"], - shared_const_properties={"tag": "test_tag"}, + shared_constant_properties={"tag": "test_tag"}, ) assertions1(g) g = PersistentGraph() g.load_nodes_from_pandas( nodes_df, - "id", "time", - "node_type", + "id", + node_type_col="node_type", properties=["name"], - shared_const_properties={"tag": "test_tag"}, + shared_constant_properties={"tag": "test_tag"}, + ) + assertions1(g) + + g = Graph() + g.load_nodes_from_pandas(nodes_df, "time", "id") + g.load_node_props_from_pandas( + nodes_df, + "id", + node_type_col="node_type", + constant_properties=["name"], + shared_constant_properties={"tag": "test_tag"}, + ) + assertions1(g) + + g = PersistentGraph() + g.load_nodes_from_pandas(nodes_df, "time", "id") + g.load_node_props_from_pandas( + nodes_df, + "id", + node_type_col="node_type", + constant_properties=["name"], + shared_constant_properties={"tag": "test_tag"}, ) assertions1(g) @@ -328,11 +345,11 @@ def assertions2(g): g = Graph() g.load_nodes_from_pandas( nodes_df, - "id", "time", - "node_type", + "id", + node_type_col="node_type", properties=["name"], - const_properties=["type"], + constant_properties=["type"], ) assertions2(g) @@ -341,9 +358,9 @@ def assertions2(g): nodes_df, "id", "time", - "node_type", + node_type_col="node_type", properties=["name"], - const_properties=["type"], + constant_properties=["type"], ) assertions2(g) @@ -375,28 +392,26 @@ def assertions3(g): g = Graph() g.load_edges_from_pandas( edges_df, + "time", "src", "dst", - "time", properties=["weight", "marbles"], - const_properties=["marbles_const"], - shared_const_properties={"type": "Edge", "tag": "test_tag"}, + constant_properties=["marbles_const"], + shared_constant_properties={"type": "Edge", "tag": "test_tag"}, layer="test_layer", - layer_in_df=False, ) assertions3(g) g = PersistentGraph() g.load_edges_from_pandas( edges_df, + "time", "src", "dst", - "time", properties=["weight", "marbles"], - const_properties=["marbles_const"], - shared_const_properties={"type": "Edge", "tag": "test_tag"}, + constant_properties=["marbles_const"], + shared_constant_properties={"type": "Edge", "tag": "test_tag"}, layer="test_layer", - layer_in_df=False, ) assertions3(g) @@ -416,13 +431,13 @@ def assertions4(g): g = Graph() g.load_edges_from_pandas( - edges_df, "src", "dst", "time", ["weight", "marbles"], layer="layers" + edges_df, "time", "src", "dst", ["weight", "marbles"], layer_col="layers" ) assertions4(g) g = PersistentGraph() g.load_edges_from_pandas( - edges_df, "src", "dst", "time", ["weight", "marbles"], layer="layers" + edges_df, "time", "src", "dst", ["weight", "marbles"], layer_col="layers" ) assertions4(g) @@ -437,33 +452,37 @@ def assertions5(g): ] assert g.layers(["test_layer"]).edges.src.id.collect() == [1, 2, 3, 4, 5] - g = Graph.load_from_pandas( + g = Graph() + g.load_edges_from_pandas( edges_df, + "time", "src", "dst", - "time", - edge_layer="test_layer", - layer_in_df=False, - node_df=nodes_df, - node_id="id", - node_time="time", - node_properties=["name"], - node_shared_const_properties={"type": "Person"}, + layer="test_layer", + ) + g.load_nodes_from_pandas( + df=nodes_df, + time="time", + id="id", + properties=["name"], + shared_constant_properties={"type": "Person"}, ) assertions5(g) - g = PersistentGraph.load_from_pandas( + g = PersistentGraph() + g.load_edges_from_pandas( edges_df, + "time", "src", "dst", - "time", - edge_layer="test_layer", - layer_in_df=False, - node_df=nodes_df, - node_id="id", - node_time="time", - node_properties=["name"], - node_shared_const_properties={"type": "Person"}, + layer="test_layer", + ) + g.load_nodes_from_pandas( + df=nodes_df, + time="time", + id="id", + properties=["name"], + shared_constant_properties={"type": "Person"}, ) assertions5(g) @@ -489,31 +508,25 @@ def assertions6(g): 5, ] - g = Graph.load_from_pandas( - edges_df, - "src", - "dst", - "time", - edge_layer="layers", - node_df=nodes_df, - node_id="id", - node_time="time", - node_properties=["name"], - node_const_properties=["type"], + g = Graph() + g.load_edges_from_pandas(edges_df, "time", "src", "dst", layer_col="layers") + g.load_nodes_from_pandas( + df=nodes_df, + time="time", + id="id", + properties=["name"], + constant_properties=["type"], ) assertions6(g) - g = PersistentGraph.load_from_pandas( - edges_df, - "src", - "dst", - "time", - edge_layer="layers", - node_df=nodes_df, - node_id="id", - node_time="time", - node_properties=["name"], - node_const_properties=["type"], + g = PersistentGraph() + g.load_edges_from_pandas(edges_df, "time", "src", "dst", layer_col="layers") + g.load_nodes_from_pandas( + df=nodes_df, + time="time", + id="id", + properties=["name"], + constant_properties=["type"], ) assertions6(g) @@ -535,23 +548,21 @@ def assertions7(g): "test_tag", ] - g = Graph.load_from_pandas( + g = Graph() + g.load_edges_from_pandas( edges_df, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_properties=["weight", "marbles"], - node_df=nodes_df, - node_id="id", - node_time="time", - node_properties=["name"], - edge_layer="layers", + time="time", + src="src", + dst="dst", + properties=["weight", "marbles"], + layer_col="layers", ) + g.load_nodes_from_pandas(df=nodes_df, time="time", id="id", properties=["name"]) g.load_node_props_from_pandas( nodes_df, "id", - const_properties=["type"], - shared_const_properties={"tag": "test_tag"}, + constant_properties=["type"], + shared_constant_properties={"tag": "test_tag"}, ) assertions7(g) @@ -575,29 +586,28 @@ def assertions8(g): edges_df, "src", "dst", - const_properties=["marbles_const"], - shared_const_properties={"tag": "test_tag"}, - layer="layers", + constant_properties=["marbles_const"], + shared_constant_properties={"tag": "test_tag"}, + layer_col="layers", ) assertions8(g) - g = PersistentGraph.load_from_pandas( + g = PersistentGraph() + + g.load_edges_from_pandas( edges_df, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_properties=["weight", "marbles"], - node_df=nodes_df, - node_id="id", - node_time="time", - node_properties=["name"], - edge_layer="layers", + time="time", + src="src", + dst="dst", + properties=["weight", "marbles"], + layer_col="layers", ) + g.load_nodes_from_pandas(df=nodes_df, time="time", id="id", properties=["name"]) g.load_node_props_from_pandas( nodes_df, "id", - const_properties=["type"], - shared_const_properties={"tag": "test_tag"}, + constant_properties=["type"], + shared_constant_properties={"tag": "test_tag"}, ) assertions7(g) @@ -605,9 +615,9 @@ def assertions8(g): edges_df, "src", "dst", - const_properties=["marbles_const"], - shared_const_properties={"tag": "test_tag"}, - layer="layers", + constant_properties=["marbles_const"], + shared_constant_properties={"tag": "test_tag"}, + layer_col="layers", ) assertions8(g) @@ -624,35 +634,35 @@ def assertions_layers_in_df(g): assert g.layers(["layer 3"]).edges.src.id.collect() == [3] with pytest.raises( Exception, - match=re.escape("Invalid layer: test_layer. Valid layers: _default, layer 1, layer 2, layer 3, layer 4, layer 5"), + match=re.escape( + "Invalid layer: test_layer. Valid layers: _default, layer 1, layer 2, layer 3, layer 4, layer 5" + ), ): g.layers(["test_layer"]) g = Graph() g.load_edges_from_pandas( edges_df, + "time", "src", "dst", - "time", ["weight", "marbles"], - const_properties=["marbles_const"], - shared_const_properties={"type": "Edge", "tag": "test_tag"}, - layer="layers", - layer_in_df=True, + constant_properties=["marbles_const"], + shared_constant_properties={"type": "Edge", "tag": "test_tag"}, + layer_col="layers", ) assertions_layers_in_df(g) g = PersistentGraph() g.load_edges_from_pandas( edges_df, + "time", "src", "dst", - "time", ["weight", "marbles"], - const_properties=["marbles_const"], - shared_const_properties={"type": "Edge", "tag": "test_tag"}, - layer="layers", - layer_in_df=True, + constant_properties=["marbles_const"], + shared_constant_properties={"type": "Edge", "tag": "test_tag"}, + layer_col="layers", ) assertions_layers_in_df(g) @@ -682,11 +692,12 @@ def test_missing_columns(): "columns are not present within the dataframe: not_src, not_dst, not_time" ), ): - g = Graph.load_from_pandas( + g = Graph() + g.load_edges_from_pandas( edges_df, - edge_src="not_src", - edge_dst="not_dst", - edge_time="not_time", + time="not_time", + src="not_src", + dst="not_dst", ) with pytest.raises( @@ -695,11 +706,12 @@ def test_missing_columns(): "columns are not present within the dataframe: not_src, not_dst, not_time" ), ): - g = PersistentGraph.load_from_pandas( + g = PersistentGraph() + g.load_edges_from_pandas( edges_df, - edge_src="not_src", - edge_dst="not_dst", - edge_time="not_time", + time="not_time", + src="not_src", + dst="not_dst", ) with pytest.raises( @@ -708,18 +720,16 @@ def test_missing_columns(): "columns are not present within the dataframe: not_weight, bleep_bloop" ), ): - g = Graph.load_from_pandas( + g = Graph() + g.load_edges_from_pandas( edges_df, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_properties=["not_weight", "marbles"], - edge_const_properties=["bleep_bloop"], - node_df=nodes_df, - node_id="id", - node_time="time", - node_properties=["name"], + time="time", + src="src", + dst="dst", + properties=["not_weight", "marbles"], + constant_properties=["bleep_bloop"], ) + g.load_nodes_from_pandas(df=nodes_df, time="time", id="id", properties=["name"]) with pytest.raises( Exception, @@ -727,18 +737,16 @@ def test_missing_columns(): "columns are not present within the dataframe: not_weight, bleep_bloop" ), ): - g = PersistentGraph.load_from_pandas( + g = PersistentGraph() + g.load_edges_from_pandas( edges_df, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_properties=["not_weight", "marbles"], - edge_const_properties=["bleep_bloop"], - node_df=nodes_df, - node_id="id", - node_time="time", - node_properties=["name"], + time="time", + src="src", + dst="dst", + properties=["not_weight", "marbles"], + constant_properties=["bleep_bloop"], ) + g.load_nodes_from_pandas(df=nodes_df, time="time", id="id", properties=["name"]) with pytest.raises( Exception, @@ -746,16 +754,16 @@ def test_missing_columns(): "columns are not present within the dataframe: not_id, not_time, not_name" ), ): - g = Graph.load_from_pandas( + g = Graph() + g.load_edges_from_pandas( edges_df, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_properties=["weight", "marbles"], - node_df=nodes_df, - node_id="not_id", - node_time="not_time", - node_properties=["not_name"], + time="time", + src="src", + dst="dst", + properties=["weight", "marbles"], + ) + g.load_nodes_from_pandas( + df=nodes_df, time="not_time", id="not_id", properties=["not_name"] ) with pytest.raises( @@ -764,16 +772,16 @@ def test_missing_columns(): "columns are not present within the dataframe: not_id, not_time, not_name" ), ): - g = PersistentGraph.load_from_pandas( + g = PersistentGraph() + g.load_edges_from_pandas( edges_df, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_properties=["weight", "marbles"], - node_df=nodes_df, - node_id="not_id", - node_time="not_time", - node_properties=["not_name"], + time="time", + src="src", + dst="dst", + properties=["weight", "marbles"], + ) + g.load_nodes_from_pandas( + df=nodes_df, id="not_id", time="not_time", properties=["not_name"] ) with pytest.raises( @@ -787,7 +795,7 @@ def test_missing_columns(): edges_df, src="sauce", dst="dist", - const_properties=["wait", "marples"], + constant_properties=["wait", "marples"], ) with pytest.raises( @@ -801,7 +809,7 @@ def test_missing_columns(): edges_df, src="sauce", dst="dist", - const_properties=["wait", "marples"], + constant_properties=["wait", "marples"], ) with pytest.raises( @@ -814,7 +822,7 @@ def test_missing_columns(): g.load_node_props_from_pandas( nodes_df, id="sauce", - const_properties=["wait", "marples"], + constant_properties=["wait", "marples"], ) with pytest.raises( @@ -827,7 +835,7 @@ def test_missing_columns(): g.load_node_props_from_pandas( nodes_df, id="sauce", - const_properties=["wait", "marples"], + constant_properties=["wait", "marples"], ) @@ -838,12 +846,13 @@ def test_none_columns_edges(): with pytest.raises( Exception, match=re.escape("Ensure these contain no NaN, Null or None values.") ): - Graph.load_from_pandas(edges_df, "src", "dst", "time") + g = Graph() + g.load_edges_from_pandas(edges_df, "time", "src", "dst") with pytest.raises( Exception, match=re.escape("Ensure these contain no NaN, Null or None values.") ): - PersistentGraph.load_from_pandas(edges_df, "src", "dst", "time") + PersistentGraph().load_edges_from_pandas(edges_df, "time", "src", "dst") edges_df = pd.DataFrame( {"src": [1, 2, 3, 4, 5], "dst": [2, 3, 4, None, 6], "time": [1, 2, 3, 4, 5]} @@ -851,11 +860,11 @@ def test_none_columns_edges(): with pytest.raises( Exception, match=re.escape("Ensure these contain no NaN, Null or None values.") ): - Graph.load_from_pandas(edges_df, "src", "dst", "time") + Graph().load_edges_from_pandas(edges_df, "time", "src", "dst") with pytest.raises( Exception, match=re.escape("Ensure these contain no NaN, Null or None values.") ): - PersistentGraph.load_from_pandas(edges_df, "src", "dst", "time") + PersistentGraph().load_edges_from_pandas(edges_df, "time", "src", "dst") edges_df = pd.DataFrame( {"src": [1, 2, 3, 4, 5], "dst": [2, 3, 4, 5, 6], "time": [1, 2, None, 4, 5]} @@ -863,11 +872,11 @@ def test_none_columns_edges(): with pytest.raises( Exception, match=re.escape("Ensure these contain no NaN, Null or None values.") ): - Graph.load_from_pandas(edges_df, "src", "dst", "time") + Graph().load_edges_from_pandas(edges_df, "time", "src", "dst") with pytest.raises( Exception, match=re.escape("Ensure these contain no NaN, Null or None values.") ): - PersistentGraph.load_from_pandas(edges_df, "src", "dst", "time") + PersistentGraph().load_edges_from_pandas(edges_df, "time", "src", "dst") def test_loading_list_as_properties(): @@ -881,12 +890,13 @@ def test_loading_list_as_properties(): } ) - g = Graph.load_from_pandas( + g = Graph() + g.load_edges_from_pandas( df, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_properties=["marbles"], + time="time", + src="src", + dst="dst", + properties=["marbles"], ) assert g.edge(1, 2).properties["marbles"] == ["red"] @@ -902,8 +912,8 @@ def test_loading_list_as_properties(): g = Graph() g.load_nodes_from_pandas( df=df, - id="id", time="time", + id="id", properties=["marbles"], ) @@ -927,12 +937,12 @@ def test_unparsable_props(): """"Could not convert '2.0' with type str: tried to convert to double", 'Conversion failed for column weight with type object'""" ), ): - Graph.load_from_pandas( + Graph().load_edges_from_pandas( edges_df, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_properties=["weight"], + time="time", + src="src", + dst="dst", + properties=["weight"], ) with pytest.raises( Exception, @@ -940,12 +950,12 @@ def test_unparsable_props(): """"Could not convert '2.0' with type str: tried to convert to double", 'Conversion failed for column weight with type object'""" ), ): - PersistentGraph.load_from_pandas( + PersistentGraph().load_edges_from_pandas( edges_df, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_properties=["weight"], + time="time", + src="src", + dst="dst", + properties=["weight"], ) @@ -995,48 +1005,38 @@ def edges_assertions(g): assert g.count_nodes() == 6 g = Graph() - g.load_nodes_from_pandas(nodes_df, "id", "time") + g.load_nodes_from_pandas(nodes_df, "time", "id") nodes_assertions(g) g = PersistentGraph() - g.load_nodes_from_pandas(nodes_df, "id", "time") + g.load_nodes_from_pandas(nodes_df, "time", "id") nodes_assertions(g) g = Graph() - g.load_nodes_from_pandas( - nodes_df, "id", "time", node_type="node_type", node_type_in_df=False - ) + g.load_nodes_from_pandas(nodes_df, "time", "id", node_type="node_type") nodes_assertions2(g) g = PersistentGraph() - g.load_nodes_from_pandas( - nodes_df, "id", "time", node_type="node_type", node_type_in_df=False - ) + g.load_nodes_from_pandas(nodes_df, "time", "id", node_type="node_type") nodes_assertions2(g) g = Graph() - g.load_nodes_from_pandas( - nodes_df2, "id", "time", node_type="node_type", node_type_in_df=False - ) + g.load_nodes_from_pandas(nodes_df2, "time", "id", node_type="node_type") nodes_assertions2(g) g = PersistentGraph() - g.load_nodes_from_pandas( - nodes_df2, "id", "time", node_type="node_type", node_type_in_df=False - ) + g.load_nodes_from_pandas(nodes_df2, "time", "id", node_type="node_type") nodes_assertions2(g) g = Graph() - g.load_nodes_from_pandas( - nodes_df2, "id", "time", node_type="node_type", node_type_in_df=True - ) + g.load_nodes_from_pandas(nodes_df2, "time", "id", node_type_col="node_type") nodes_assertions3(g) g = PersistentGraph() - g.load_nodes_from_pandas( - nodes_df2, "id", "time", node_type="node_type", node_type_in_df=True - ) + g.load_nodes_from_pandas(nodes_df2, "time", "id", node_type_col="node_type") nodes_assertions3(g) - g = Graph.load_from_pandas(edges_df, "src", "dst", "time") + g = Graph() + g.load_edges_from_pandas(edges_df, "time", "src", "dst") edges_assertions(g) - g = Graph.load_from_pandas(edges_df, "src", "dst", "time") + g = Graph() + g.load_edges_from_pandas(edges_df, "time", "src", "dst") edges_assertions(g) @@ -1057,7 +1057,278 @@ def test_load_edge_deletions_from_pandas(): ) g = PersistentGraph() - g.load_edges_from_pandas(edges_df, "src", "dst", "time") + g.load_edges_from_pandas(edges_df, "time", "src", "dst") assert g.window(10, 12).edges.src.id.collect() == [1, 2, 3, 4, 5] - g.load_edges_deletions_from_pandas(edge_dels_df, "src", "dst", "time") + g.load_edge_deletions_from_pandas(edge_dels_df, "time", "src", "dst") assert g.window(10, 12).edges.src.id.collect() == [1, 2, 5] + + +def test_edge_both_option_failures_pandas(): + edges_df = pd.DataFrame( + { + "src": [1, 2, 3, 4, 5], + "dst": [2, 3, 4, 5, 6], + "time": [1, 2, 3, 4, 5], + "weight": [1.0, 2.0, 3.0, 4.0, 5.0], + "marbles": ["red", "blue", "green", "yellow", "purple"], + } + ) + # CHECK ALL EDGE FUNCTIONS ON GRAPH FAIL WITH BOTH LAYER AND LAYER_COL + g = Graph() + with pytest.raises( + Exception, + match=r"GraphLoadException\('WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)'\)", + ): + g.load_edges_from_pandas( + edges_df, "time", "src", "dst", layer="blah", layer_col="marbles" + ) + + with pytest.raises( + Exception, + match=r"GraphLoadException\('WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)'\)", + ): + g.load_edge_props_from_pandas( + edges_df, "src", "dst", layer="blah", layer_col="marbles" + ) + + # CHECK IF JUST LAYER WORKS + g = Graph() + g.load_edges_from_pandas(edges_df, "time", "src", "dst", layer="blah") + assert g.edges.layer_names.collect() == [ + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ] + assert g.unique_layers == ["_default", "blah"] + + g = Graph() + g.load_edges_from_pandas(edges_df, "time", "src", "dst", layer="blah") + g.load_edge_props_from_pandas( + edges_df, "src", "dst", layer="blah", constant_properties=["marbles"] + ) + assert g.edges.layer_names.collect() == [ + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ] + assert g.unique_layers == ["_default", "blah"] + assert g.layer("blah").edges.properties.get("marbles") == [ + "red", + "blue", + "green", + "yellow", + "purple", + ] + + # CHECK IF JUST LAYER_COL WORKS + g = Graph() + g.load_edges_from_pandas(edges_df, "time", "src", "dst", layer_col="marbles") + assert g.edges.layer_names.collect() == [ + ["red"], + ["blue"], + ["green"], + ["yellow"], + ["purple"], + ] + assert g.unique_layers == ["_default", "red", "blue", "green", "yellow", "purple"] + + g = Graph() + g.load_edges_from_pandas(edges_df, "time", "src", "dst", layer_col="marbles") + g.load_edge_props_from_pandas( + edges_df, "src", "dst", layer_col="marbles", constant_properties=["marbles"] + ) + assert g.edges.layer_names.collect() == [ + ["red"], + ["blue"], + ["green"], + ["yellow"], + ["purple"], + ] + assert g.unique_layers == ["_default", "red", "blue", "green", "yellow", "purple"] + assert g.edges.properties.get("marbles").collect() == [ + {"red": "red"}, + {"blue": "blue"}, + {"green": "green"}, + {"yellow": "yellow"}, + {"purple": "purple"}, + ] + + g = PersistentGraph() + with pytest.raises( + Exception, + match=r"GraphLoadException\('WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)'\)", + ): + g.load_edges_from_pandas( + edges_df, "time", "src", "dst", layer="blah", layer_col="marbles" + ) + + with pytest.raises( + Exception, + match=r"GraphLoadException\('WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)'\)", + ): + g.load_edge_props_from_pandas( + edges_df, "src", "dst", layer="blah", layer_col="marbles" + ) + + with pytest.raises( + Exception, + match=r"GraphLoadException\('WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)'\)", + ): + g.load_edge_deletions_from_pandas( + edges_df, "time", "src", "dst", layer="blah", layer_col="marbles" + ) + + # CHECK IF JUST LAYER WORKS + g = PersistentGraph() + g.load_edges_from_pandas(edges_df, "time", "src", "dst", layer="blah") + assert g.edges.layer_names.collect() == [ + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ] + assert g.unique_layers == ["_default", "blah"] + + g = PersistentGraph() + g.load_edges_from_pandas(edges_df, "time", "src", "dst", layer="blah") + g.load_edge_props_from_pandas( + edges_df, "src", "dst", layer="blah", constant_properties=["marbles"] + ) + assert g.edges.layer_names.collect() == [ + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ] + assert g.unique_layers == ["_default", "blah"] + assert g.layer("blah").edges.properties.get("marbles") == [ + "red", + "blue", + "green", + "yellow", + "purple", + ] + + g = PersistentGraph() + g.load_edge_deletions_from_pandas(edges_df, "time", "src", "dst", layer="blah") + assert g.edges.layer_names.collect() == [ + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ] + assert g.unique_layers == ["_default", "blah"] + + # CHECK IF JUST LAYER_COL WORKS + g = PersistentGraph() + g.load_edges_from_pandas(edges_df, "time", "src", "dst", layer_col="marbles") + assert g.edges.layer_names.collect() == [ + ["red"], + ["blue"], + ["green"], + ["yellow"], + ["purple"], + ] + assert g.unique_layers == ["_default", "red", "blue", "green", "yellow", "purple"] + + g = PersistentGraph() + g.load_edges_from_pandas(edges_df, "time", "src", "dst", layer_col="marbles") + g.load_edge_props_from_pandas( + edges_df, "src", "dst", layer_col="marbles", constant_properties=["marbles"] + ) + assert g.edges.layer_names.collect() == [ + ["red"], + ["blue"], + ["green"], + ["yellow"], + ["purple"], + ] + assert g.unique_layers == ["_default", "red", "blue", "green", "yellow", "purple"] + assert g.edges.properties.get("marbles").collect() == [ + {"red": "red"}, + {"blue": "blue"}, + {"green": "green"}, + {"yellow": "yellow"}, + {"purple": "purple"}, + ] + + g = PersistentGraph() + g.load_edge_deletions_from_pandas( + edges_df, "time", "src", "dst", layer_col="marbles" + ) + assert g.edges.layer_names.collect() == [ + ["red"], + ["blue"], + ["green"], + ["yellow"], + ["purple"], + ] + assert g.unique_layers == ["_default", "red", "blue", "green", "yellow", "purple"] + + +def test_node_both_option_failures_pandas(): + nodes_df = pd.DataFrame( + { + "id": [1, 2, 3, 4, 5, 6], + "name": ["Alice", "Bob", "Carol", "Dave", "Eve", "Frank"], + "time": [1, 2, 3, 4, 5, 6], + "node_type": ["P1", "P2", "P3", "P4", "P5", "P6"], + } + ) + # CHECK ALL NODE FUNCTIONS ON GRAPH FAIL WITH BOTH NODE_TYPE AND NODE_TYPE_COL + with pytest.raises( + Exception, + match=r"GraphLoadException\('WrongNumOfArgs\(\"node_type\", \"node_type_col\"\)'\)", + ): + g = Graph() + g.load_nodes_from_pandas( + nodes_df, "time", "id", node_type="node_type", node_type_col="node_type" + ) + + with pytest.raises( + Exception, + match=r"GraphLoadException\('WrongNumOfArgs\(\"node_type\", \"node_type_col\"\)'\)", + ): + g = Graph() + g.load_node_props_from_pandas( + nodes_df, "id", node_type="node_type", node_type_col="node_type" + ) + + # CHECK IF JUST NODE_TYPE WORKS + g = Graph() + g.load_nodes_from_pandas(nodes_df, "time", "id", node_type="node_type") + assert g.nodes.node_type.collect() == [ + "node_type", + "node_type", + "node_type", + "node_type", + "node_type", + "node_type", + ] + g = Graph() + g.load_nodes_from_pandas(nodes_df, "time", "id") + g.load_node_props_from_pandas(nodes_df, "id", node_type="node_type") + assert g.nodes.node_type.collect() == [ + "node_type", + "node_type", + "node_type", + "node_type", + "node_type", + "node_type", + ] + + # CHECK IF JUST NODE_TYPE_COL WORKS + g = Graph() + g.load_nodes_from_pandas(nodes_df, "time", "id", node_type_col="node_type") + assert g.nodes.node_type.collect() == ["P1", "P2", "P3", "P4", "P5", "P6"] + g = Graph() + g.load_nodes_from_pandas(nodes_df, "time", "id") + g.load_node_props_from_pandas(nodes_df, "id", node_type_col="node_type") + assert g.nodes.node_type.collect() == ["P1", "P2", "P3", "P4", "P5", "P6"] diff --git a/python/tests/test_load_from_parquet.py b/python/tests/test_load_from_parquet.py index 6785df097a..5b8b62e48a 100644 --- a/python/tests/test_load_from_parquet.py +++ b/python/tests/test_load_from_parquet.py @@ -1,10 +1,10 @@ import os import re -import tempfile - import pyarrow as pa import pyarrow.parquet as pq import pytest +import tempfile +import pandas as pd from raphtory import Graph, PersistentGraph @@ -14,7 +14,9 @@ def parquet_files(): dirname = tempfile.TemporaryDirectory() nodes_parquet_file_path = os.path.join(dirname.name, "parquet", "nodes.parquet") edges_parquet_file_path = os.path.join(dirname.name, "parquet", "edges.parquet") - edge_deletions_parquet_file_path = os.path.join(dirname.name, "parquet", "edges_deletions.parquet") + edge_deletions_parquet_file_path = os.path.join( + dirname.name, "parquet", "edges_deletions.parquet" + ) os.makedirs(os.path.dirname(nodes_parquet_file_path), exist_ok=True) @@ -22,8 +24,15 @@ def parquet_files(): "id": [1, 2, 3, 4, 5, 6], "name": ["Alice", "Bob", "Carol", "Dave", "Eve", "Frank"], "time": [1, 2, 3, 4, 5, 6], - "type": ["Person 1", "Person 2", "Person 3", "Person 4", "Person 5", "Person 6"], - "node_type": ["p", "p", "p", "p", "p", "p"], + "type": [ + "Person 1", + "Person 2", + "Person 3", + "Person 4", + "Person 5", + "Person 6", + ], + "node_type": ["p1", "p2", "p3", "p4", "p5", "p6"], } table = pa.table(data) @@ -52,7 +61,11 @@ def parquet_files(): table = pa.table(data) pq.write_table(table, edge_deletions_parquet_file_path) - print("""Created edges_deletions.parquet at loc = {}""".format(edge_deletions_parquet_file_path)) + print( + """Created edges_deletions.parquet at loc = {}""".format( + edge_deletions_parquet_file_path + ) + ) yield nodes_parquet_file_path, edges_parquet_file_path, edge_deletions_parquet_file_path @@ -96,12 +109,12 @@ def assert_expected_edges(g): def assert_expected_node_types(g): assert g.nodes.node_type == [ - "p", - "p", - "p", - "p", - "p", - "p", + "p1", + "p2", + "p3", + "p4", + "p5", + "p6", ] @@ -139,13 +152,13 @@ def assert_expected_node_property_dept(g): def assert_expected_edge_properties(g): - assert g.layers( - ["layer 1", "layer 2", "layer 3"] - ).edges.properties.constant.get("marbles_const").collect() == [ - {"layer 1": "red"}, - {"layer 2": "blue"}, - {"layer 3": "green"}, - ] + assert g.layers(["layer 1", "layer 2", "layer 3"]).edges.properties.constant.get( + "marbles_const" + ).collect() == [ + {"layer 1": "red"}, + {"layer 2": "blue"}, + {"layer 3": "green"}, + ] assert g.edges.properties.constant.get("tag").collect() == [ {"layer 1": "test_tag"}, {"layer 2": "test_tag"}, @@ -180,14 +193,31 @@ def assert_expected_edge_properties_test_layer(g): def assert_expected_layers(g): - assert g.unique_layers == ["_default", "layer 1", "layer 2", "layer 3", "layer 4", "layer 5"] + assert g.unique_layers == [ + "_default", + "layer 1", + "layer 2", + "layer 3", + "layer 4", + "layer 5", + ] assert g.layers(["layer 1"]).edges.src.id.collect() == [1] assert g.layers(["layer 1", "layer 2"]).edges.src.id.collect() == [1, 2] - assert g.layers(["layer 1", "layer 2", "layer 3"]).edges.src.id.collect() == [1, 2, 3] - assert g.layers(["layer 1", "layer 4", "layer 5"]).edges.src.id.collect() == [1, 4, 5] + assert g.layers(["layer 1", "layer 2", "layer 3"]).edges.src.id.collect() == [ + 1, + 2, + 3, + ] + assert g.layers(["layer 1", "layer 4", "layer 5"]).edges.src.id.collect() == [ + 1, + 4, + 5, + ] with pytest.raises( - Exception, - match=re.escape("Invalid layer: test_layer. Valid layers: _default, layer 1, layer 2, layer 3, layer 4, layer 5"), + Exception, + match=re.escape( + "Invalid layer: test_layer. Valid layers: _default, layer 1, layer 2, layer 3, layer 4, layer 5" + ), ): g.layers(["test_layer"]) @@ -198,19 +228,26 @@ def assert_expected_test_layer(g): def test_load_from_parquet_graphs(parquet_files): - nodes_parquet_file_path, edges_parquet_file_path, edges_deletions_parquet_file_path = parquet_files - - g = Graph.load_from_parquet( - edge_parquet_path=edges_parquet_file_path, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_properties=["weight", "marbles"], - node_parquet_path=nodes_parquet_file_path, - node_id="id", - node_time="time", - node_properties=["name"], - node_type="node_type", + ( + nodes_parquet_file_path, + edges_parquet_file_path, + edges_deletions_parquet_file_path, + ) = parquet_files + + g = Graph() + g.load_edges_from_parquet( + parquet_path=edges_parquet_file_path, + time="time", + src="src", + dst="dst", + properties=["weight", "marbles"], + ) + g.load_nodes_from_parquet( + parquet_path=nodes_parquet_file_path, + time="time", + id="id", + properties=["name"], + node_type_col="node_type", ) assert_expected_nodes(g) assert_expected_edges(g) @@ -218,10 +255,10 @@ def test_load_from_parquet_graphs(parquet_files): g = Graph() g.load_nodes_from_parquet( parquet_path=nodes_parquet_file_path, - id="id", time="time", - node_type="node_type", - properties=["name"] + id="id", + node_type_col="node_type", + properties=["name"], ) g.load_edges_from_parquet( parquet_path=edges_parquet_file_path, @@ -229,7 +266,7 @@ def test_load_from_parquet_graphs(parquet_files): dst="dst", time="time", properties=["weight", "marbles"], - layer="layers" + layer_col="layers", ) assert_expected_nodes(g) assert_expected_edges(g) @@ -238,8 +275,8 @@ def test_load_from_parquet_graphs(parquet_files): g.load_node_props_from_parquet( parquet_path=nodes_parquet_file_path, id="id", - const_properties=["type"], - shared_const_properties={"tag": "test_tag"}, + constant_properties=["type"], + shared_constant_properties={"tag": "test_tag"}, ) assert_expected_node_property_tag(g) assert_expected_node_property_type(g) @@ -248,9 +285,9 @@ def test_load_from_parquet_graphs(parquet_files): parquet_path=edges_parquet_file_path, src="src", dst="dst", - const_properties=["marbles_const"], - shared_const_properties={"tag": "test_tag"}, - layer="layers", + constant_properties=["marbles_const"], + shared_constant_properties={"tag": "test_tag"}, + layer_col="layers", ) assert_expected_edge_properties(g) assert_expected_layers(g) @@ -260,9 +297,9 @@ def test_load_from_parquet_graphs(parquet_files): parquet_path=nodes_parquet_file_path, id="id", time="time", - node_type="node_type", + node_type_col="node_type", properties=["name"], - shared_const_properties={"tag": "test_tag"}, + shared_constant_properties={"tag": "test_tag"}, ) assert_expected_node_types(g) assert_expected_node_property_tag(g) @@ -274,60 +311,71 @@ def test_load_from_parquet_graphs(parquet_files): dst="dst", time="time", properties=["weight", "marbles"], - const_properties=["marbles_const"], - shared_const_properties={"type": "Edge", "tag": "test_tag"}, + constant_properties=["marbles_const"], + shared_constant_properties={"type": "Edge", "tag": "test_tag"}, layer="test_layer", - layer_in_df=False, ) assert_expected_edge_properties_test_layer(g) assert_expected_test_layer(g) - g = Graph.load_from_parquet( - edge_parquet_path=edges_parquet_file_path, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_layer="test_layer", - layer_in_df=False, - node_parquet_path=nodes_parquet_file_path, - node_id="id", - node_time="time", - node_properties=["name"], - node_shared_const_properties={"dept": "Sales"}, + g = Graph() + g.load_edges_from_parquet( + parquet_path=edges_parquet_file_path, + time="time", + src="src", + dst="dst", + layer="test_layer", + ) + g.load_nodes_from_parquet( + parquet_path=nodes_parquet_file_path, + time="time", + id="id", + properties=["name"], + shared_constant_properties={"dept": "Sales"}, ) assert_expected_test_layer(g) assert_expected_node_property_dept(g) - g = Graph.load_from_parquet( - edge_parquet_path=edges_parquet_file_path, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_layer="layers", - node_parquet_path=nodes_parquet_file_path, - node_id="id", - node_time="time", - node_properties=["name"], - node_const_properties=["type"], + g = Graph() + g.load_edges_from_parquet( + parquet_path=edges_parquet_file_path, + src="src", + dst="dst", + time="time", + layer_col="layers", + ) + g.load_nodes_from_parquet( + parquet_path=nodes_parquet_file_path, + time="time", + id="id", + properties=["name"], + constant_properties=["type"], ) assert_expected_node_property_type(g) assert_expected_layers(g) def test_load_from_parquet_persistent_graphs(parquet_files): - nodes_parquet_file_path, edges_parquet_file_path, edges_deletions_parquet_file_path = parquet_files - - g = PersistentGraph.load_from_parquet( - edge_parquet_path=edges_parquet_file_path, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_properties=["weight", "marbles"], - node_parquet_path=nodes_parquet_file_path, - node_id="id", - node_time="time", - node_properties=["name"], - node_type="node_type", + ( + nodes_parquet_file_path, + edges_parquet_file_path, + edges_deletions_parquet_file_path, + ) = parquet_files + + g = PersistentGraph() + g.load_edges_from_parquet( + parquet_path=edges_parquet_file_path, + src="src", + dst="dst", + time="time", + properties=["weight", "marbles"], + ) + g.load_nodes_from_parquet( + parquet_path=nodes_parquet_file_path, + time="time", + id="id", + properties=["name"], + node_type_col="node_type", ) assert_expected_nodes(g) assert_expected_edges(g) @@ -338,7 +386,7 @@ def test_load_from_parquet_persistent_graphs(parquet_files): id="id", time="time", node_type="node_type", - properties=["name"] + properties=["name"], ) g.load_edges_from_parquet( parquet_path=edges_parquet_file_path, @@ -346,7 +394,7 @@ def test_load_from_parquet_persistent_graphs(parquet_files): dst="dst", time="time", properties=["weight", "marbles"], - layer="layers" + layer_col="layers", ) assert_expected_nodes(g) assert_expected_edges(g) @@ -355,8 +403,8 @@ def test_load_from_parquet_persistent_graphs(parquet_files): g.load_node_props_from_parquet( parquet_path=nodes_parquet_file_path, id="id", - const_properties=["type"], - shared_const_properties={"tag": "test_tag"}, + constant_properties=["type"], + shared_constant_properties={"tag": "test_tag"}, ) assert_expected_node_property_tag(g) assert_expected_node_property_type(g) @@ -365,9 +413,9 @@ def test_load_from_parquet_persistent_graphs(parquet_files): parquet_path=edges_parquet_file_path, src="src", dst="dst", - const_properties=["marbles_const"], - shared_const_properties={"tag": "test_tag"}, - layer="layers", + constant_properties=["marbles_const"], + shared_constant_properties={"tag": "test_tag"}, + layer_col="layers", ) assert_expected_edge_properties(g) assert_expected_layers(g) @@ -375,11 +423,11 @@ def test_load_from_parquet_persistent_graphs(parquet_files): g = PersistentGraph() g.load_nodes_from_parquet( parquet_path=nodes_parquet_file_path, - id="id", time="time", - node_type="node_type", + id="id", + node_type_col="node_type", properties=["name"], - shared_const_properties={"tag": "test_tag"}, + shared_constant_properties={"tag": "test_tag"}, ) assert_expected_node_types(g) assert_expected_node_property_tag(g) @@ -387,45 +435,49 @@ def test_load_from_parquet_persistent_graphs(parquet_files): g = PersistentGraph() g.load_edges_from_parquet( parquet_path=edges_parquet_file_path, + time="time", src="src", dst="dst", - time="time", properties=["weight", "marbles"], - const_properties=["marbles_const"], - shared_const_properties={"type": "Edge", "tag": "test_tag"}, + constant_properties=["marbles_const"], + shared_constant_properties={"type": "Edge", "tag": "test_tag"}, layer="test_layer", - layer_in_df=False, ) assert_expected_edge_properties_test_layer(g) assert_expected_test_layer(g) - g = Graph.load_from_parquet( - edge_parquet_path=edges_parquet_file_path, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_layer="test_layer", - layer_in_df=False, - node_parquet_path=nodes_parquet_file_path, - node_id="id", - node_time="time", - node_properties=["name"], - node_shared_const_properties={"dept": "Sales"}, + g = Graph() + g.load_edges_from_parquet( + parquet_path=edges_parquet_file_path, + src="src", + dst="dst", + time="time", + layer="test_layer", + ) + g.load_nodes_from_parquet( + parquet_path=nodes_parquet_file_path, + time="time", + id="id", + properties=["name"], + shared_constant_properties={"dept": "Sales"}, ) assert_expected_test_layer(g) assert_expected_node_property_dept(g) - g = PersistentGraph.load_from_parquet( - edge_parquet_path=edges_parquet_file_path, - edge_src="src", - edge_dst="dst", - edge_time="time", - edge_layer="layers", - node_parquet_path=nodes_parquet_file_path, - node_id="id", - node_time="time", - node_properties=["name"], - node_const_properties=["type"], + g = PersistentGraph() + g.load_edges_from_parquet( + parquet_path=edges_parquet_file_path, + src="src", + dst="dst", + time="time", + layer_col="layers", + ) + g.load_nodes_from_parquet( + parquet_path=nodes_parquet_file_path, + time="time", + id="id", + properties=["name"], + constant_properties=["type"], ) assert_expected_node_property_type(g) assert_expected_layers(g) @@ -433,16 +485,342 @@ def test_load_from_parquet_persistent_graphs(parquet_files): g = PersistentGraph() g.load_edges_from_parquet( parquet_path=edges_parquet_file_path, + time="time", src="src", dst="dst", - time="time", ) assert g.window(10, 12).edges.src.id.collect() == [1, 2, 3, 4, 5] - g.load_edges_deletions_from_parquet( + g.load_edge_deletions_from_parquet( parquet_path=edges_deletions_parquet_file_path, + time="time", src="src", dst="dst", - time="time" ) assert g.window(10, 12).edges.src.id.collect() == [1, 2, 5] + +def test_edge_both_option_failures_parquet(parquet_files): + ( + nodes_parquet_file_path, + edges_parquet_file_path, + edges_deletions_parquet_file_path, + ) = parquet_files + # CHECK ALL EDGE FUNCTIONS ON GRAPH FAIL WITH BOTH LAYER AND LAYER_COL + g = Graph() + with pytest.raises( + Exception, + match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", + ): + g.load_edges_from_parquet( + edges_parquet_file_path, + "time", + "src", + "dst", + layer="blah", + layer_col="marbles", + ) + + with pytest.raises( + Exception, + match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", + ): + g.load_edge_props_from_parquet( + edges_parquet_file_path, "src", "dst", layer="blah", layer_col="marbles" + ) + + # CHECK IF JUST LAYER WORKS + g = Graph() + g.load_edges_from_parquet( + edges_parquet_file_path, "time", "src", "dst", layer="blah" + ) + assert g.edges.layer_names.collect() == [ + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ] + assert g.unique_layers == ["_default", "blah"] + + g = Graph() + g.load_edges_from_parquet( + edges_parquet_file_path, "time", "src", "dst", layer="blah" + ) + g.load_edge_props_from_parquet( + edges_parquet_file_path, + "src", + "dst", + layer="blah", + constant_properties=["marbles"], + ) + assert g.edges.layer_names.collect() == [ + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ] + assert g.unique_layers == ["_default", "blah"] + assert g.layer("blah").edges.properties.get("marbles") == [ + "red", + "blue", + "green", + "yellow", + "purple", + ] + + # CHECK IF JUST LAYER_COL WORKS + g = Graph() + g.load_edges_from_parquet( + edges_parquet_file_path, "time", "src", "dst", layer_col="marbles" + ) + assert g.edges.layer_names.collect() == [ + ["red"], + ["blue"], + ["green"], + ["yellow"], + ["purple"], + ] + assert g.unique_layers == ["_default", "red", "blue", "green", "yellow", "purple"] + + g = Graph() + g.load_edges_from_parquet( + edges_parquet_file_path, "time", "src", "dst", layer_col="marbles" + ) + g.load_edge_props_from_parquet( + edges_parquet_file_path, + "src", + "dst", + layer_col="marbles", + constant_properties=["marbles"], + ) + assert g.edges.layer_names.collect() == [ + ["red"], + ["blue"], + ["green"], + ["yellow"], + ["purple"], + ] + assert g.unique_layers == ["_default", "red", "blue", "green", "yellow", "purple"] + assert g.edges.properties.get("marbles").collect() == [ + {"red": "red"}, + {"blue": "blue"}, + {"green": "green"}, + {"yellow": "yellow"}, + {"purple": "purple"}, + ] + + g = PersistentGraph() + with pytest.raises( + Exception, + match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", + ): + g.load_edges_from_parquet( + edges_parquet_file_path, + "time", + "src", + "dst", + layer="blah", + layer_col="marbles", + ) + + with pytest.raises( + Exception, + match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", + ): + g.load_edge_props_from_parquet( + edges_parquet_file_path, "src", "dst", layer="blah", layer_col="marbles" + ) + + with pytest.raises( + Exception, + match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"layer_name\", \"layer_col\"\)", + ): + g.load_edge_deletions_from_parquet( + edges_parquet_file_path, + "time", + "src", + "dst", + layer="blah", + layer_col="marbles", + ) + + # CHECK IF JUST LAYER WORKS + g = PersistentGraph() + g.load_edges_from_parquet( + edges_parquet_file_path, "time", "src", "dst", layer="blah" + ) + assert g.edges.layer_names.collect() == [ + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ] + assert g.unique_layers == ["_default", "blah"] + + g = PersistentGraph() + g.load_edges_from_parquet( + edges_parquet_file_path, "time", "src", "dst", layer="blah" + ) + g.load_edge_props_from_parquet( + edges_parquet_file_path, + "src", + "dst", + layer="blah", + constant_properties=["marbles"], + ) + assert g.edges.layer_names.collect() == [ + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ] + assert g.unique_layers == ["_default", "blah"] + assert g.layer("blah").edges.properties.get("marbles") == [ + "red", + "blue", + "green", + "yellow", + "purple", + ] + + g = PersistentGraph() + g.load_edge_deletions_from_parquet( + edges_parquet_file_path, "time", "src", "dst", layer="blah" + ) + assert g.edges.layer_names.collect() == [ + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ["blah"], + ] + assert g.unique_layers == ["_default", "blah"] + + # CHECK IF JUST LAYER_COL WORKS + g = PersistentGraph() + g.load_edges_from_parquet( + edges_parquet_file_path, "time", "src", "dst", layer_col="marbles" + ) + assert g.edges.layer_names.collect() == [ + ["red"], + ["blue"], + ["green"], + ["yellow"], + ["purple"], + ] + assert g.unique_layers == ["_default", "red", "blue", "green", "yellow", "purple"] + + g = PersistentGraph() + g.load_edges_from_parquet( + edges_parquet_file_path, "time", "src", "dst", layer_col="marbles" + ) + g.load_edge_props_from_parquet( + edges_parquet_file_path, + "src", + "dst", + layer_col="marbles", + constant_properties=["marbles"], + ) + assert g.edges.layer_names.collect() == [ + ["red"], + ["blue"], + ["green"], + ["yellow"], + ["purple"], + ] + assert g.unique_layers == ["_default", "red", "blue", "green", "yellow", "purple"] + assert g.edges.properties.get("marbles").collect() == [ + {"red": "red"}, + {"blue": "blue"}, + {"green": "green"}, + {"yellow": "yellow"}, + {"purple": "purple"}, + ] + + g = PersistentGraph() + g.load_edge_deletions_from_parquet( + edges_parquet_file_path, "time", "src", "dst", layer_col="marbles" + ) + assert g.edges.layer_names.collect() == [ + ["red"], + ["blue"], + ["green"], + ["yellow"], + ["purple"], + ] + assert g.unique_layers == ["_default", "red", "blue", "green", "yellow", "purple"] + + +def test_node_both_option_failures_parquet(parquet_files): + ( + nodes_parquet_file_path, + edges_parquet_file_path, + edges_deletions_parquet_file_path, + ) = parquet_files + + # CHECK ALL NODE FUNCTIONS ON GRAPH FAIL WITH BOTH NODE_TYPE AND NODE_TYPE_COL + with pytest.raises( + Exception, + match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"node_type\", \"node_type_col\"\)", + ): + g = Graph() + g.load_nodes_from_parquet( + nodes_parquet_file_path, + "time", + "id", + node_type="node_type", + node_type_col="node_type", + ) + + with pytest.raises( + Exception, + match=r"Failed to load graph: Failed to load graph WrongNumOfArgs\(\"node_type\", \"node_type_col\"\)", + ): + g = Graph() + g.load_node_props_from_parquet( + nodes_parquet_file_path, + "id", + node_type="node_type", + node_type_col="node_type", + ) + + # CHECK IF JUST NODE_TYPE WORKS + g = Graph() + g.load_nodes_from_parquet( + nodes_parquet_file_path, "time", "id", node_type="node_type" + ) + assert g.nodes.node_type.collect() == [ + "node_type", + "node_type", + "node_type", + "node_type", + "node_type", + "node_type", + ] + g = Graph() + g.load_nodes_from_parquet(nodes_parquet_file_path, "time", "id") + g.load_node_props_from_parquet(nodes_parquet_file_path, "id", node_type="node_type") + assert g.nodes.node_type.collect() == [ + "node_type", + "node_type", + "node_type", + "node_type", + "node_type", + "node_type", + ] + + # CHECK IF JUST NODE_TYPE_COL WORKS + g = Graph() + g.load_nodes_from_parquet( + nodes_parquet_file_path, "time", "id", node_type_col="node_type" + ) + assert g.nodes.node_type.collect() == ["p1", "p2", "p3", "p4", "p5", "p6"] + g = Graph() + g.load_nodes_from_parquet(nodes_parquet_file_path, "time", "id") + g.load_node_props_from_parquet( + nodes_parquet_file_path, "id", node_type_col="node_type" + ) + assert g.nodes.node_type.collect() == ["p1", "p2", "p3", "p4", "p5", "p6"] diff --git a/raphtory-api/Cargo.toml b/raphtory-api/Cargo.toml index 51f99f0e0d..aedc155f06 100644 --- a/raphtory-api/Cargo.toml +++ b/raphtory-api/Cargo.toml @@ -24,13 +24,14 @@ parking_lot = { workspace = true } pyo3 = { workspace = true, optional = true } rayon = { workspace = true } rand = { workspace = true } -quickcheck = { workspace = true } quickcheck_macros = { workspace = true } num-traits = { workspace = true } twox-hash.workspace = true [dev-dependencies] proptest.workspace = true +quickcheck.workspace = true +quickcheck_macros.workspace = true [features] default = [] diff --git a/raphtory-api/src/core/storage/dict_mapper.rs b/raphtory-api/src/core/storage/dict_mapper.rs index a80e14bbdf..58619246cc 100644 --- a/raphtory-api/src/core/storage/dict_mapper.rs +++ b/raphtory-api/src/core/storage/dict_mapper.rs @@ -1,7 +1,12 @@ use crate::core::storage::{arc_str::ArcStr, locked_vec::ArcReadLockedVec, FxDashMap}; +use dashmap::mapref::entry::Entry; use parking_lot::RwLock; use serde::{Deserialize, Serialize}; -use std::{borrow::Borrow, hash::Hash, sync::Arc}; +use std::{ + borrow::{Borrow, BorrowMut}, + hash::Hash, + sync::Arc, +}; #[derive(Serialize, Deserialize, Default, Debug)] pub struct DictMapper { @@ -9,24 +14,100 @@ pub struct DictMapper { reverse_map: Arc>>, //FIXME: a boxcar vector would be a great fit if it was serializable... } +#[derive(Copy, Clone, Debug)] +pub enum MaybeNew { + New(Index), + Existing(Index), +} + +impl PartialEq for MaybeNew +where + Index: PartialEq, + T: Borrow, +{ + fn eq(&self, other: &T) -> bool { + other.borrow() == self.borrow() + } +} + +impl MaybeNew { + #[inline] + pub fn inner(self) -> Index { + match self { + MaybeNew::New(inner) => inner, + MaybeNew::Existing(inner) => inner, + } + } + + #[inline] + pub fn map(self, map_fn: impl FnOnce(Index) -> R) -> MaybeNew { + match self { + MaybeNew::New(inner) => MaybeNew::New(map_fn(inner)), + MaybeNew::Existing(inner) => MaybeNew::Existing(map_fn(inner)), + } + } + + #[inline] + pub fn as_ref(&self) -> MaybeNew<&Index> { + match self { + MaybeNew::New(inner) => MaybeNew::New(inner), + MaybeNew::Existing(inner) => MaybeNew::Existing(inner), + } + } + + #[inline] + pub fn as_mut(&mut self) -> MaybeNew<&mut Index> { + match self { + MaybeNew::New(inner) => MaybeNew::New(inner), + MaybeNew::Existing(inner) => MaybeNew::Existing(inner), + } + } + + #[inline] + pub fn if_new(self, map_fn: impl FnOnce(Index) -> R) -> Option { + match self { + MaybeNew::New(inner) => Some(map_fn(inner)), + MaybeNew::Existing(_) => None, + } + } +} + +impl Borrow for MaybeNew { + #[inline] + fn borrow(&self) -> &Index { + self.as_ref().inner() + } +} + +impl BorrowMut for MaybeNew { + #[inline] + fn borrow_mut(&mut self) -> &mut Index { + self.as_mut().inner() + } +} + impl DictMapper { - pub fn get_or_create_id(&self, name: &Q) -> usize + pub fn get_or_create_id(&self, name: &Q) -> MaybeNew where Q: Hash + Eq + ?Sized + ToOwned + Borrow, T: Into, { if let Some(existing_id) = self.map.get(name.borrow()) { - return *existing_id; + return MaybeNew::Existing(*existing_id); } let name = name.to_owned().into(); - let new_id = self.map.entry(name.clone()).or_insert_with(|| { - let mut reverse = self.reverse_map.write(); - let id = reverse.len(); - reverse.push(name); - id - }); - *new_id + let new_id = match self.map.entry(name.clone()) { + Entry::Occupied(entry) => MaybeNew::Existing(*entry.get()), + Entry::Vacant(entry) => { + let mut reverse = self.reverse_map.write(); + let id = reverse.len(); + reverse.push(name); + entry.insert(id); + MaybeNew::New(id) + } + }; + new_id } pub fn get_id(&self, name: &str) -> Option { @@ -88,11 +169,11 @@ mod test { #[test] fn test_dict_mapper() { let mapper = DictMapper::default(); - assert_eq!(mapper.get_or_create_id("test"), 0); - assert_eq!(mapper.get_or_create_id("test"), 0); - assert_eq!(mapper.get_or_create_id("test2"), 1); - assert_eq!(mapper.get_or_create_id("test2"), 1); - assert_eq!(mapper.get_or_create_id("test"), 0); + assert_eq!(mapper.get_or_create_id("test"), 0usize); + assert_eq!(mapper.get_or_create_id("test").inner(), 0); + assert_eq!(mapper.get_or_create_id("test2").inner(), 1); + assert_eq!(mapper.get_or_create_id("test2").inner(), 1); + assert_eq!(mapper.get_or_create_id("test").inner(), 0); } #[quickcheck] @@ -110,7 +191,7 @@ mod test { write_s.shuffle(&mut rng); for s in write_s { let id = mapper.get_or_create_id(s.as_str()); - ids.insert(s, id); + ids.insert(s, id.inner()); } ids }) @@ -147,7 +228,7 @@ mod test { let mut actual = vec!["test", "test2", "test3", "test4", "test5"] .into_iter() - .map(|name| mapper.get_or_create_id(name)) + .map(|name| mapper.get_or_create_id(name).inner()) .collect::>(); actual.sort(); diff --git a/raphtory-benchmark/Cargo.toml b/raphtory-benchmark/Cargo.toml index 08ca97469f..aae91a280d 100644 --- a/raphtory-benchmark/Cargo.toml +++ b/raphtory-benchmark/Cargo.toml @@ -7,20 +7,16 @@ edition = "2021" [dependencies] criterion = { workspace = true } -raphtory = { path = "../raphtory", features = ["io"], version = "0.10.0" } -raphtory-api = { path = "../raphtory-api", version = "0.10.0" } -raphtory-graphql = { path = "../raphtory-graphql", version = "0.10.0" } +raphtory = { path = "../raphtory", features = ["io"], version = "0.11.0" } +raphtory-api = { path = "../raphtory-api", version = "0.11.0" } pometry-storage.workspace = true sorted_vector_map = { workspace = true } rand = { workspace = true } rayon = { workspace = true } -polars-arrow = { workspace = true } tempfile = { workspace = true } -chrono = { workspace = true } clap = { workspace = true } csv = { workspace = true } flate2 = { workspace = true } -tokio = { workspace = true } [[bench]] name = "tgraph_benchmarks" diff --git a/raphtory-benchmark/bin/run_graphql.py b/raphtory-benchmark/bin/run_graphql.py index c22f7acad0..b0953c5812 100644 --- a/raphtory-benchmark/bin/run_graphql.py +++ b/raphtory-benchmark/bin/run_graphql.py @@ -1,4 +1,4 @@ -from raphtory.graphql import RaphtoryServer +from raphtory.graphql import GraphServer import logging logging.basicConfig(level=logging.INFO) @@ -6,7 +6,7 @@ def load_gql_server(): logging.info("Loading GQL server") - RaphtoryServer(graph_dir="./graphs/").run() + GraphServer(graph_dir="./graphs/").run() load_gql_server() diff --git a/raphtory-benchmark/bin/run_server.rs b/raphtory-benchmark/bin/run_server.rs index 53768e82f2..de6f64062b 100644 --- a/raphtory-benchmark/bin/run_server.rs +++ b/raphtory-benchmark/bin/run_server.rs @@ -1,10 +1,10 @@ -use raphtory_graphql::RaphtoryServer; +use raphtory_graphql::GraphServer; #[tokio::main(flavor = "multi_thread", worker_threads = 63)] async fn main() { let graph_directory = "/Users/haaroony/Documents/dev/Raphtory/comparison-benchmark/graphql-benchmark/graphs"; - RaphtoryServer::from_directory(&graph_directory) + GraphServer::from_directory(&graph_directory) .run("INFO", false) .await .unwrap() diff --git a/raphtory-cypher/Cargo.toml b/raphtory-cypher/Cargo.toml index 7eec1c729f..52dfd956f4 100644 --- a/raphtory-cypher/Cargo.toml +++ b/raphtory-cypher/Cargo.toml @@ -42,7 +42,6 @@ itertools.workspace = true proptest.workspace = true pretty_assertions.workspace = true tempfile.workspace = true -rand.workspace = true tokio.workspace = true clap.workspace = true diff --git a/raphtory-cypher/src/lib.rs b/raphtory-cypher/src/lib.rs index 6661c86a67..23a1f5fa52 100644 --- a/raphtory-cypher/src/lib.rs +++ b/raphtory-cypher/src/lib.rs @@ -312,7 +312,7 @@ mod cypher { let edge_lists = vec![chunk]; let graph = - DiskGraphStorage::load_from_edge_lists(&edge_lists, 20, 20, graph_dir, 0, 1, 2) + DiskGraphStorage::load_from_edge_lists(&edge_lists, 20, 20, graph_dir, 2, 0, 1) .unwrap(); let df = run_cypher("match ()-[e]->() RETURN *", &graph, true) diff --git a/raphtory-graphql/Cargo.toml b/raphtory-graphql/Cargo.toml index ba20d8ee88..25b3b3b59d 100644 --- a/raphtory-graphql/Cargo.toml +++ b/raphtory-graphql/Cargo.toml @@ -13,26 +13,22 @@ readme.workspace = true homepage.workspace = true [dependencies] -raphtory = { path = "../raphtory", version = "0.10.0", features = ['vectors', 'search', "io"] } -raphtory-api = { path = "../raphtory-api", version = "0.10.0" } -bincode = { workspace = true } +raphtory = { path = "../raphtory", version = "0.11.0", features = ['vectors', 'search', "io"] } +raphtory-api = { path = "../raphtory-api", version = "0.11.0" } base64 = { workspace = true } thiserror = { workspace = true } -dotenv = { workspace = true } itertools = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } once_cell = { workspace = true } poem = { workspace = true } -poem-openapi = { workspace = true } oauth2 = { workspace = true } tokio = { workspace = true } -async-graphql = { workspace = true, features=["apollo_tracing"] } +async-graphql = { workspace = true, features = ["apollo_tracing"] } dynamic-graphql = { workspace = true } async-graphql-poem = { workspace = true } parking_lot = { workspace = true } futures-util = { workspace = true } -async-stream = { workspace = true } jsonwebtoken = { workspace = true } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true } @@ -45,13 +41,12 @@ ordered-float = { workspace = true } uuid = { workspace = true } chrono = { workspace = true } config = { workspace = true } -toml = { workspace = true } url = { workspace = true } base64-compat = { workspace = true } -time = { workspace = true } reqwest = { workspace = true } moka = { workspace = true } + # python binding optional dependencies pyo3 = { workspace = true, optional = true } crossbeam-channel = { workspace = true } diff --git a/raphtory-graphql/src/data.rs b/raphtory-graphql/src/data.rs index c62971b50f..d926d9ff75 100644 --- a/raphtory-graphql/src/data.rs +++ b/raphtory-graphql/src/data.rs @@ -1,4 +1,7 @@ -use crate::{model::algorithms::global_plugins::GlobalPlugins, server_config::AppConfig}; +use crate::{ + model::{algorithms::global_plugins::GlobalPlugins, create_dirs_if_not_present, GqlGraphType}, + server_config::AppConfig, +}; use moka::sync::Cache; #[cfg(feature = "storage")] use raphtory::disk_graph::DiskGraphStorage; @@ -11,12 +14,15 @@ use raphtory::{ SymlinkNotAllowed, }, }, - db::api::view::MaterializedGraph, + db::{api::view::MaterializedGraph, graph::views::deletion_graph::PersistentGraph}, + prelude::*, search::IndexedGraph, }; use std::{ collections::HashMap, fs, + fs::File, + io::Write, path::{Component, Path, PathBuf, StripPrefixError}, sync::Arc, }; @@ -32,9 +38,14 @@ impl Data { pub fn new(work_dir: &Path, configs: &AppConfig) -> Self { let cache_configs = &configs.cache; - let graphs_cache_builder = Cache::builder() + let graphs_cache_builder = Cache::<_, IndexedGraph>::builder() .max_capacity(cache_configs.capacity) .time_to_idle(std::time::Duration::from_secs(cache_configs.tti_seconds)) + .eviction_listener(|_, value, _| { + value + .write_updates() + .unwrap_or_else(|err| println!("Write on eviction failed: {err:?}")) + }) .build(); let graphs_cache: Cache> = graphs_cache_builder; @@ -61,6 +72,24 @@ impl Data { } } + pub fn new_graph(&self, path: &Path, graph_type: GqlGraphType) -> Result<(), GraphError> { + let full_path = self.construct_graph_full_path(path)?; + if full_path.exists() { + return Err(GraphError::GraphNameAlreadyExists(path.to_path_buf()).into()); + } + create_dirs_if_not_present(&full_path)?; + let mut cache = File::create_new(full_path)?; + match graph_type { + GqlGraphType::Persistent => { + cache.write_all(&PersistentGraph::new().encode_to_vec())?; + } + GqlGraphType::Event => { + cache.write_all(&Graph::new().encode_to_vec())?; + } + } + Ok(()) + } + pub fn get_graph_names_paths(&self) -> Result, GraphError> { let mut paths = vec![]; for path in self.get_graph_paths() { @@ -250,9 +279,9 @@ fn get_graph_from_path(path: &Path) -> Result, G return Err(PathIsDirectory(path.to_path_buf()).into()); } } else { - let graph = load_bincode_graph(path)?; + let graph = MaterializedGraph::load_cached(path)?; println!("Graph loaded = {}", path.display()); - Ok(IndexedGraph::from_graph(&graph.into())?) + Ok(IndexedGraph::from_graph(&graph)?) } } @@ -297,11 +326,6 @@ pub(crate) fn get_graph_name(path: &Path) -> Result { } //should not happen, but means we always get a name } -pub(crate) fn load_bincode_graph(path: &Path) -> Result { - let graph = MaterializedGraph::load_from_file(path, false)?; - Ok(graph) -} - #[cfg(feature = "storage")] fn load_disk_graph(path: &Path) -> Result { let disk_graph = DiskGraphStorage::load_from_dir(path) @@ -336,7 +360,7 @@ pub(crate) mod data_tests { use raphtory::core::utils::errors::{GraphError, InvalidPathReason}; #[cfg(feature = "storage")] use raphtory::{ - db::api::storage::storage_ops::GraphStorage, db::api::view::internal::CoreGraphOps, + db::api::storage::graph::storage_ops::GraphStorage, db::api::view::internal::CoreGraphOps, disk_graph::DiskGraphStorage, }; #[cfg(feature = "storage")] @@ -390,12 +414,12 @@ pub(crate) mod data_tests { #[cfg(feature = "storage")] copy_dir_recursive(disk_graph_path, &full_path)?; } else { - graph.save_to_path(&full_path)?; + graph.encode(&full_path)?; } #[cfg(not(feature = "storage"))] { - graph.save_to_path(&full_path)?; + graph.encode(&full_path)?; } } Ok(()) @@ -499,13 +523,9 @@ pub(crate) mod data_tests { .add_edge(0, 1, 3, [("name", "test_e2")], None) .unwrap(); - graph - .save_to_file(&tmp_work_dir.path().join("test_g")) - .unwrap(); + graph.encode(&tmp_work_dir.path().join("test_g")).unwrap(); let _ = DiskGraphStorage::from_graph(&graph, &tmp_work_dir.path().join("test_dg")).unwrap(); - graph - .save_to_file(&tmp_work_dir.path().join("test_g2")) - .unwrap(); + graph.encode(&tmp_work_dir.path().join("test_g2")).unwrap(); let configs = AppConfigBuilder::new() .with_cache_capacity(1) diff --git a/raphtory-graphql/src/lib.rs b/raphtory-graphql/src/lib.rs index 6ff3ccc6b6..5b44c609ae 100644 --- a/raphtory-graphql/src/lib.rs +++ b/raphtory-graphql/src/lib.rs @@ -1,4 +1,4 @@ -pub use crate::server::RaphtoryServer; +pub use crate::server::GraphServer; pub mod azure_auth; mod data; @@ -666,7 +666,7 @@ mod graphql_test { g.add_node(0, 1, NO_PROPS, None).unwrap(); let tmp_file = tempfile::NamedTempFile::new().unwrap(); let path = tmp_file.path(); - g.save_to_file(path).unwrap(); + g.encode(path).unwrap(); let file = std::fs::File::open(path).unwrap(); let upload_val = UploadValue { filename: "test".into(), diff --git a/raphtory-graphql/src/main.rs b/raphtory-graphql/src/main.rs index dd5a4965c0..db2fa78ab1 100644 --- a/raphtory-graphql/src/main.rs +++ b/raphtory-graphql/src/main.rs @@ -1,4 +1,4 @@ -use raphtory_graphql::RaphtoryServer; +use raphtory_graphql::GraphServer; use std::{ env, path::{Path, PathBuf}, @@ -11,7 +11,7 @@ async fn main() -> IoResult<()> { let work_dir = env::var("GRAPH_DIRECTORY").unwrap_or(default_path.display().to_string()); let work_dir = PathBuf::from(&work_dir); - RaphtoryServer::new(work_dir, None, None)?.run().await?; + GraphServer::new(work_dir, None, None)?.run().await?; Ok(()) } diff --git a/raphtory-graphql/src/model/graph/graph.rs b/raphtory-graphql/src/model/graph/graph.rs index 385a3358e2..e231ea2853 100644 --- a/raphtory-graphql/src/model/graph/graph.rs +++ b/raphtory-graphql/src/model/graph/graph.rs @@ -1,5 +1,5 @@ use crate::{ - data::get_graph_name, + data::{get_graph_name, Data}, model::{ algorithms::graph_algorithms::GraphAlgorithms, graph::{ @@ -8,6 +8,7 @@ use crate::{ schema::graph_schema::GraphSchema, }, }; +use async_graphql::{Context, DataContext}; use dynamic_graphql::{ResolvedObject, ResolvedObjectFields}; use itertools::Itertools; use raphtory::{ @@ -25,7 +26,7 @@ use raphtory::{ prelude::*, search::{into_indexed::DynamicIndexedGraph, IndexedGraph}, }; -use std::{collections::HashSet, convert::Into, path::PathBuf}; +use std::{collections::HashSet, convert::Into, path::PathBuf, sync::Arc}; #[derive(ResolvedObject)] pub(crate) struct GqlGraph { @@ -362,4 +363,18 @@ impl GqlGraph { None => vec![], } } + + /// Export all nodes and edges from this graph view to another existing graph + async fn export_to<'a>( + &'a self, + ctx: &Context<'a>, + path: String, + ) -> Result> { + let data = ctx.data_unchecked::(); + let other_g = data.get_graph(path.as_ref())?; + other_g.import_nodes(self.graph.nodes(), true)?; + other_g.import_edges(self.graph.edges(), true)?; + other_g.write_updates()?; + Ok(true) + } } diff --git a/raphtory-graphql/src/model/graph/graphs.rs b/raphtory-graphql/src/model/graph/graphs.rs index 3b7400b42a..e980452b00 100644 --- a/raphtory-graphql/src/model/graph/graphs.rs +++ b/raphtory-graphql/src/model/graph/graphs.rs @@ -1,5 +1,4 @@ use crate::data::get_graph_name; -use async_graphql::parser::Error; use dynamic_graphql::{ResolvedObject, ResolvedObjectFields}; use itertools::Itertools; use std::path::PathBuf; diff --git a/raphtory-graphql/src/model/graph/mod.rs b/raphtory-graphql/src/model/graph/mod.rs index 7d67eb65c9..4c65a1b6c9 100644 --- a/raphtory-graphql/src/model/graph/mod.rs +++ b/raphtory-graphql/src/model/graph/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod edge; mod edges; pub(crate) mod graph; pub(crate) mod graphs; +pub(crate) mod mutable_graph; pub(crate) mod node; mod nodes; mod path_from_node; diff --git a/raphtory-graphql/src/model/graph/mutable_graph.rs b/raphtory-graphql/src/model/graph/mutable_graph.rs new file mode 100644 index 0000000000..0006c7ca01 --- /dev/null +++ b/raphtory-graphql/src/model/graph/mutable_graph.rs @@ -0,0 +1,368 @@ +use crate::model::graph::{edge::Edge, graph::GqlGraph, node::Node, property::GqlPropValue}; +use dynamic_graphql::{InputObject, ResolvedObject, ResolvedObjectFields}; +use raphtory::{ + core::utils::errors::GraphError, + db::{ + api::view::MaterializedGraph, + graph::{edge::EdgeView, node::NodeView}, + }, + prelude::*, + search::IndexedGraph, +}; +use raphtory_api::core::storage::arc_str::OptionAsStr; +use std::path::PathBuf; + +#[derive(InputObject)] +pub struct GqlPropInput { + key: String, + value: GqlPropValue, +} + +#[derive(InputObject)] +pub struct TPropInput { + time: i64, + properties: Option>, +} + +#[derive(InputObject)] +pub struct NodeAddition { + name: String, + node_type: Option, + constant_properties: Option>, + updates: Option>, +} + +#[derive(InputObject)] +pub struct EdgeAddition { + src: String, + dst: String, + layer: Option, + constant_properties: Option>, + updates: Option>, +} + +#[derive(ResolvedObject)] +pub struct GqlMutableGraph { + path: PathBuf, + graph: IndexedGraph, +} + +impl GqlMutableGraph { + pub(crate) fn new(path: impl Into, graph: IndexedGraph) -> Self { + Self { + path: path.into(), + graph, + } + } +} + +fn as_properties(properties: Vec) -> impl Iterator { + properties.into_iter().map(|p| (p.key, p.value.0)) +} + +#[ResolvedObjectFields] +impl GqlMutableGraph { + /// Get the non-mutable graph + async fn graph(&self) -> GqlGraph { + GqlGraph::new(self.path.clone(), self.graph.clone()) + } + + /// Get mutable existing node + async fn node(&self, name: String) -> Option { + self.graph.node(name).map(|n| n.into()) + } + + /// Add a new node or add updates to an existing node + async fn add_node( + &self, + time: i64, + name: String, + properties: Option>, + node_type: Option, + ) -> Result { + let node = self.graph.add_node( + time, + name, + as_properties(properties.unwrap_or(vec![])), + node_type.as_str(), + )?; + self.graph.write_updates()?; + Ok(node.into()) + } + + /// Add a batch of nodes + async fn add_nodes(&self, nodes: Vec) -> Result { + for node in nodes { + let name = node.name.as_str(); + let node_type = node.node_type.as_str(); + for prop in node.updates.unwrap_or(vec![]) { + self.graph.add_node( + prop.time, + name, + as_properties(prop.properties.unwrap_or(vec![])), + node_type, + )?; + } + let constant_props = node.constant_properties.unwrap_or(vec![]); + if !constant_props.is_empty() { + let node_view = self + .graph + .node(name) + .ok_or(GraphError::NodeNameError(node.name))?; + node_view.add_constant_properties(as_properties(constant_props))?; + } + } + self.graph.write_updates()?; + Ok(true) + } + + /// Get a mutable existing edge + async fn edge(&self, src: String, dst: String) -> Option { + self.graph.edge(src, dst).map(|e| e.into()) + } + + /// Add a new edge or add updates to an existing edge + async fn add_edge( + &self, + time: i64, + src: String, + dst: String, + properties: Option>, + layer: Option, + ) -> Result { + let edge = self.graph.add_edge( + time, + src, + dst, + as_properties(properties.unwrap_or(vec![])), + layer.as_str(), + )?; + self.graph.write_updates()?; + Ok(edge.into()) + } + + /// Add a batch of edges + async fn add_edges(&self, edges: Vec) -> Result { + for edge in edges { + let src = edge.src.as_str(); + let dst = edge.dst.as_str(); + let layer = edge.layer.as_str(); + for prop in edge.updates.unwrap_or(vec![]) { + self.graph.add_edge( + prop.time, + src, + dst, + as_properties(prop.properties.unwrap_or(vec![])), + layer, + )?; + } + let constant_props = edge.constant_properties.unwrap_or(vec![]); + if !constant_props.is_empty() { + let edge_view = self.graph.edge(src, dst).ok_or(GraphError::EdgeNameError { + src: edge.src, + dst: edge.dst, + })?; + edge_view.add_constant_properties(as_properties(constant_props), layer)?; + } + } + self.graph.write_updates()?; + Ok(true) + } + + /// Mark an edge as deleted (creates the edge if it did not exist) + async fn delete_edge( + &self, + time: i64, + src: String, + dst: String, + layer: Option, + ) -> Result { + let edge = self.graph.delete_edge(time, src, dst, layer.as_str())?; + self.graph.write_updates()?; + Ok(edge.into()) + } + + /// Add temporal properties to graph + async fn add_properties( + &self, + t: i64, + properties: Vec, + ) -> Result { + self.graph.add_properties(t, as_properties(properties))?; + self.graph.write_updates()?; + Ok(true) + } + + /// Add constant properties to graph (errors if the property already exists) + async fn add_constant_properties( + &self, + properties: Vec, + ) -> Result { + self.graph + .add_constant_properties(as_properties(properties))?; + self.graph.write_updates()?; + Ok(true) + } + + /// Update constant properties of the graph (overwrites existing values) + async fn update_constant_properties( + &self, + properties: Vec, + ) -> Result { + self.graph + .update_constant_properties(as_properties(properties))?; + self.graph.write_updates()?; + Ok(true) + } +} + +#[derive(ResolvedObject)] +pub struct GqlMutableNode { + node: NodeView>, +} + +impl From>> for GqlMutableNode { + fn from(node: NodeView>) -> Self { + Self { node } + } +} + +#[ResolvedObjectFields] +impl GqlMutableNode { + /// Use to check if adding the node was successful + async fn success(&self) -> bool { + true + } + + /// Get the non-mutable `Node` + async fn node(&self) -> Node { + self.node.clone().into() + } + + /// Add constant properties to the node (errors if the property already exists) + async fn add_constant_properties( + &self, + properties: Vec, + ) -> Result { + self.node + .add_constant_properties(as_properties(properties))?; + self.node.graph.write_updates()?; + Ok(true) + } + + /// Set the node type (errors if the node already has a non-default type) + async fn set_node_type(&self, new_type: String) -> Result { + self.node.set_node_type(&new_type)?; + self.node.graph.write_updates()?; + Ok(true) + } + + /// Update constant properties of the node (overwrites existing property values) + async fn update_constant_properties( + &self, + properties: Vec, + ) -> Result { + self.node + .update_constant_properties(as_properties(properties))?; + self.node.graph.write_updates()?; + Ok(true) + } + + /// Add temporal property updates to the node + async fn add_updates( + &self, + time: i64, + properties: Vec, + ) -> Result { + self.node.add_updates(time, as_properties(properties))?; + self.node.graph.write_updates()?; + Ok(true) + } +} + +#[derive(ResolvedObject)] +pub struct GqlMutableEdge { + edge: EdgeView>, +} + +impl From>> for GqlMutableEdge { + fn from(edge: EdgeView>) -> Self { + Self { edge } + } +} + +#[ResolvedObjectFields] +impl GqlMutableEdge { + /// Use to check if adding the edge was successful + async fn success(&self) -> bool { + true + } + + /// Get the non-mutable edge for querying + async fn edge(&self) -> Edge { + self.edge.clone().into() + } + + /// Get the mutable source node of the edge + async fn src(&self) -> GqlMutableNode { + self.edge.src().into() + } + + /// Get the mutable destination node of the edge + async fn dst(&self) -> GqlMutableNode { + self.edge.dst().into() + } + + /// Mark the edge as deleted at time `time` + async fn delete(&self, time: i64, layer: Option) -> Result { + self.edge.delete(time, layer.as_str())?; + self.edge.graph.write_updates()?; + Ok(true) + } + + /// Add constant properties to the edge (errors if the value already exists) + /// + /// If this is called after `add_edge`, the layer is inherited from the `add_edge` and does not + /// need to be specified again. + async fn add_constant_properties( + &self, + properties: Vec, + layer: Option, + ) -> Result { + self.edge + .add_constant_properties(as_properties(properties), layer.as_str())?; + self.edge.graph.write_updates()?; + Ok(true) + } + + /// Update constant properties of the edge (existing values are overwritten) + /// + /// If this is called after `add_edge`, the layer is inherited from the `add_edge` and does not + /// need to be specified again. + async fn update_constant_properties( + &self, + properties: Vec, + layer: Option, + ) -> Result { + self.edge + .update_constant_properties(as_properties(properties), layer.as_str())?; + self.edge.graph.write_updates()?; + Ok(true) + } + + /// Add temporal property updates to the edge + /// + /// If this is called after `add_edge`, the layer is inherited from the `add_edge` and does not + /// need to be specified again. + async fn add_updates( + &self, + time: i64, + properties: Vec, + layer: Option, + ) -> Result { + self.edge + .add_updates(time, as_properties(properties), layer.as_str())?; + self.edge.graph.write_updates()?; + Ok(true) + } +} diff --git a/raphtory-graphql/src/model/mod.rs b/raphtory-graphql/src/model/mod.rs index 174ba90ea5..41017ec7da 100644 --- a/raphtory-graphql/src/model/mod.rs +++ b/raphtory-graphql/src/model/mod.rs @@ -2,32 +2,36 @@ use crate::{ data::Data, model::{ algorithms::global_plugins::GlobalPlugins, - graph::{graph::GqlGraph, graphs::GqlGraphs, vectorised_graph::GqlVectorisedGraph}, + graph::{ + graph::GqlGraph, graphs::GqlGraphs, mutable_graph::GqlMutableGraph, + vectorised_graph::GqlVectorisedGraph, + }, }, - url_encode::url_decode_graph, + url_encode::{url_decode_graph, url_encode_graph}, }; use async_graphql::Context; -use base64::{engine::general_purpose::STANDARD, Engine}; use chrono::Utc; use dynamic_graphql::{ - App, Mutation, MutationFields, MutationRoot, ResolvedObject, ResolvedObjectFields, Result, - Upload, + App, Enum, Mutation, MutationFields, MutationRoot, ResolvedObject, ResolvedObjectFields, + Result, Upload, }; use itertools::Itertools; #[cfg(feature = "storage")] -use raphtory::db::api::{storage::storage_ops::GraphStorage, view::internal::CoreGraphOps}; +use raphtory::db::api::{storage::graph::storage_ops::GraphStorage, view::internal::CoreGraphOps}; use raphtory::{ core::{utils::errors::GraphError, Prop}, db::api::view::MaterializedGraph, - prelude::{GraphViewOps, ImportOps, PropertyAdditionOps}, + prelude::*, }; use raphtory_api::core::storage::arc_str::ArcStr; use std::{ error::Error, fmt::{Display, Formatter}, fs, - io::Read, + fs::File, + io::copy, path::Path, + sync::Arc, }; pub mod algorithms; @@ -59,6 +63,12 @@ pub enum GqlGraphError { FailedToCreateDir(String), } +#[derive(Enum)] +pub enum GqlGraphType { + Persistent, + Event, +} + #[derive(ResolvedObject)] #[graphql(root)] pub(crate) struct QueryRoot; @@ -78,6 +88,14 @@ impl QueryRoot { .map(|g| GqlGraph::new(path.to_path_buf(), g))?) } + async fn update_graph<'a>(ctx: &Context<'a>, path: String) -> Result { + let data = ctx.data_unchecked::(); + let graph = data + .get_graph(path.as_ref()) + .map(|g| GqlMutableGraph::new(path, g))?; + Ok(graph) + } + async fn vectorised_graph<'a>(ctx: &Context<'a>, path: String) -> Option { let data = ctx.data_unchecked::(); let g = data @@ -100,11 +118,12 @@ impl QueryRoot { data.global_plugins.clone() } - async fn receive_graph<'a>(ctx: &Context<'a>, path: String) -> Result { - let path = Path::new(&path); + async fn receive_graph<'a>(ctx: &Context<'a>, path: String) -> Result> { + let path = path.as_ref(); let data = ctx.data_unchecked::(); - let g = data.get_graph(path)?.materialize()?; - Ok(STANDARD.encode(g.bincode()?)) + let g = data.get_graph(path)?.graph.clone(); + let res = url_encode_graph(g)?; + Ok(res) } } @@ -131,6 +150,16 @@ impl Mut { Ok(true) } + async fn new_graph<'a>( + ctx: &Context<'a>, + path: String, + graph_type: GqlGraphType, + ) -> Result { + let data = ctx.data_unchecked::(); + data.new_graph(path.as_ref(), graph_type)?; + Ok(true) + } + // If namespace is not provided, it will be set to the current working directory. // This applies to both the graph namespace and new graph namespace. async fn move_graph<'a>(ctx: &Context<'a>, path: String, new_path: String) -> Result { @@ -160,7 +189,7 @@ impl Mut { graph.update_constant_properties([("lastUpdated", Prop::I64(timestamp * 1000))])?; graph.update_constant_properties([("lastOpened", Prop::I64(timestamp * 1000))])?; create_dirs_if_not_present(&new_full_path)?; - graph.save_to_file(&new_full_path)?; + graph.graph.encode(&new_full_path)?; delete_graph(&full_path)?; data.graphs.remove(&path.to_path_buf()); @@ -197,7 +226,7 @@ impl Mut { let new_graph = graph.materialize()?; new_graph.update_constant_properties([("lastOpened", Prop::I64(timestamp * 1000))])?; create_dirs_if_not_present(&new_full_path)?; - new_graph.save_to_file(&new_full_path)?; + new_graph.encode(&new_full_path)?; } Ok(true) @@ -217,9 +246,7 @@ impl Mut { graph.update_constant_properties([("lastOpened", Prop::I64(timestamp * 1000))])?; - let full_path = data.construct_graph_full_path(path)?; - graph.save_to_file(full_path)?; - data.graphs.insert(path.to_path_buf(), graph); + graph.write_updates()?; Ok(true) } @@ -261,7 +288,7 @@ impl Mut { new_subgraph.update_constant_properties([("isArchive", Prop::U8(is_archive))])?; create_dirs_if_not_present(&new_graph_full_path)?; - new_subgraph.save_to_file(new_graph_full_path)?; + new_subgraph.cache(new_graph_full_path)?; data.graphs .insert(new_graph_path.to_path_buf(), new_subgraph.into()); @@ -354,7 +381,7 @@ impl Mut { new_subgraph.update_constant_properties([("uiProps", Prop::Str(props.into()))])?; new_subgraph.update_constant_properties([("isArchive", Prop::U8(is_archive))])?; - new_subgraph.save_to_file(new_graph_full_path)?; + new_subgraph.cache(new_graph_full_path)?; data.graphs.remove(&graph_path.to_path_buf()); data.graphs @@ -398,12 +425,11 @@ impl Mut { return Err(GraphError::GraphNameAlreadyExists(path.to_path_buf()).into()); } - let mut buffer = Vec::new(); - let mut buff_read = graph.value(ctx)?.content; - buff_read.read_to_end(&mut buffer)?; - let g: MaterializedGraph = MaterializedGraph::from_bincode(&buffer)?; + let mut in_file = graph.value(ctx)?.content; create_dirs_if_not_present(&full_path)?; - g.save_to_file(&full_path)?; + let mut out_file = File::create(&full_path)?; + copy(&mut in_file, &mut out_file)?; + let g = MaterializedGraph::load_cached(&full_path)?; data.graphs.insert(path.to_path_buf(), g.into()); Ok(path.display().to_string()) } @@ -411,7 +437,7 @@ impl Mut { /// Send graph bincode as base64 encoded string /// /// Returns:: - /// name of the new graph + /// path of the new graph async fn send_graph<'a>( ctx: &Context<'a>, path: String, @@ -426,7 +452,7 @@ impl Mut { } let g: MaterializedGraph = url_decode_graph(graph)?; create_dirs_if_not_present(&full_path)?; - g.save_to_file(&full_path)?; + g.cache(&full_path)?; data.graphs.insert(path.to_path_buf(), g.into()); Ok(path.display().to_string()) } @@ -441,12 +467,7 @@ impl Mut { } graph.update_constant_properties([("isArchive", Prop::U8(is_archive))])?; - - let full_path = data.construct_graph_full_path(path)?; - graph.save_to_file(full_path)?; - - data.graphs.insert(path.to_path_buf(), graph); - + graph.write_updates()?; Ok(true) } } diff --git a/raphtory-graphql/src/python/graphql.rs b/raphtory-graphql/src/python/graphql.rs index f0a58c0bb0..1074604340 100644 --- a/raphtory-graphql/src/python/graphql.rs +++ b/raphtory-graphql/src/python/graphql.rs @@ -4,18 +4,18 @@ use crate::{ global_plugins::GlobalPlugins, vector_algorithms::VectorAlgorithms, }, server_config::*, - url_encode::url_encode_graph, - RaphtoryServer, + url_encode::{url_decode_graph, url_encode_graph}, + GraphServer, }; use async_graphql::{ dynamic::{Field, FieldFuture, FieldValue, InputValue, Object, TypeRef, ValueAccessor}, Value as GraphqlValue, }; -use base64::{engine::general_purpose, Engine as _}; use crossbeam_channel::Sender as CrossbeamSender; use dynamic_graphql::internal::{Registry, TypeName}; use itertools::intersperse; use pyo3::{ + exceptions, exceptions::{PyAttributeError, PyException, PyTypeError, PyValueError}, prelude::*, types::{IntoPyDict, PyDict, PyFunction, PyList}, @@ -116,11 +116,11 @@ impl PyGlobalPlugins { } /// A class for defining and running a Raphtory GraphQL server -#[pyclass(name = "RaphtoryServer")] -pub struct PyRaphtoryServer(Option); +#[pyclass(name = "GraphServer")] +pub struct PyGraphServer(Option); -impl PyRaphtoryServer { - fn new(server: RaphtoryServer) -> Self { +impl PyGraphServer { + fn new(server: GraphServer) -> Self { Self(Some(server)) } @@ -222,7 +222,7 @@ impl PyRaphtoryServer { } #[pymethods] -impl PyRaphtoryServer { +impl PyGraphServer { #[new] #[pyo3( signature = (work_dir, cache_capacity = None, cache_tti_seconds = None, log_level = None, config_path = None) @@ -246,8 +246,8 @@ impl PyRaphtoryServer { } let app_config = Some(app_config_builder.build()); - let server = RaphtoryServer::new(work_dir, app_config, config_path)?; - Ok(PyRaphtoryServer::new(server)) + let server = GraphServer::new(work_dir, app_config, config_path)?; + Ok(PyGraphServer::new(server)) } /// Vectorise a subset of the graphs of the server. @@ -323,7 +323,7 @@ impl PyRaphtoryServer { ) -> PyResult { let adapter = |entry_point: &VectorAlgorithms, py: Python| entry_point.graph.clone().into_py(py); - PyRaphtoryServer::with_generic_document_search_function(slf, name, input, function, adapter) + PyGraphServer::with_generic_document_search_function(slf, name, input, function, adapter) } /// Register a function in the GraphQL schema for document search among all the graphs. @@ -349,7 +349,7 @@ impl PyRaphtoryServer { let adapter = |entry_point: &GlobalPlugins, py: Python| { PyGlobalPlugins(entry_point.clone()).into_py(py) }; - PyRaphtoryServer::with_generic_document_search_function(slf, name, input, function, adapter) + PyGraphServer::with_generic_document_search_function(slf, name, input, function, adapter) } /// Start the server and return a handle to it. @@ -365,7 +365,7 @@ impl PyRaphtoryServer { py: Python, port: u16, timeout_ms: Option, - ) -> PyResult { + ) -> PyResult { let (sender, receiver) = crossbeam_channel::bounded::(1); let server = take_server_ownership(slf)?; @@ -394,15 +394,15 @@ impl PyRaphtoryServer { }) }); - let mut server = PyRunningRaphtoryServer::new(join_handle, sender, port); + let mut server = PyRunningGraphServer::new(join_handle, sender, port)?; if let Some(server_handler) = &server.server_handler { - match PyRunningRaphtoryServer::wait_for_server_online( + match PyRunningGraphServer::wait_for_server_online( &server_handler.client.url, timeout_ms, ) { Ok(_) => return Ok(server), Err(e) => { - PyRunningRaphtoryServer::stop_server(&mut server, py)?; + PyRunningGraphServer::stop_server(&mut server, py)?; Err(e) } } @@ -446,7 +446,7 @@ fn adapt_graphql_value(value: &ValueAccessor, py: Python) -> PyObject { } } -fn take_server_ownership(mut server: PyRefMut) -> PyResult { +fn take_server_ownership(mut server: PyRefMut) -> PyResult { let new_server = server.0.take().ok_or_else(|| { PyException::new_err( "Server object has already been used, please create another one from scratch", @@ -470,8 +470,8 @@ const RUNNING_SERVER_CONSUMED_MSG: &str = "Running server object has already been used, please create another one from scratch"; /// A Raphtory server handler that also enables querying the server -#[pyclass(name = "RunningRaphtoryServer")] -pub struct PyRunningRaphtoryServer { +#[pyclass(name = "RunningGraphServer")] +pub struct PyRunningGraphServer { server_handler: Option, } @@ -486,20 +486,20 @@ struct ServerHandler { client: PyRaphtoryClient, } -impl PyRunningRaphtoryServer { +impl PyRunningGraphServer { fn new( join_handle: JoinHandle>, sender: CrossbeamSender, port: u16, - ) -> Self { + ) -> PyResult { let url = format!("http://localhost:{port}"); let server_handler = Some(ServerHandler { join_handle, sender, - client: PyRaphtoryClient::new(url), + client: PyRaphtoryClient::new(url)?, }); - PyRunningRaphtoryServer { server_handler } + Ok(PyRunningGraphServer { server_handler }) } fn apply_if_alive(&self, function: F) -> PyResult @@ -544,7 +544,7 @@ impl PyRunningRaphtoryServer { } #[pymethods] -impl PyRunningRaphtoryServer { +impl PyRunningGraphServer { pub(crate) fn get_client(&self) -> PyResult { self.apply_if_alive(|handler| Ok(handler.client.clone())) } @@ -639,8 +639,22 @@ const WAIT_CHECK_INTERVAL_MILLIS: u64 = 200; #[pymethods] impl PyRaphtoryClient { #[new] - fn new(url: String) -> Self { - Self { url } + fn new(url: String) -> PyResult { + match reqwest::blocking::get(url.clone()) { + Ok(response) => { + if response.status() == 200 { + Ok(Self { url }) + } else { + Err(PyValueError::new_err(format!( + "Could not connect to the given server - response {}", + response.status() + ))) + } + } + Err(_) => Err(PyValueError::new_err( + "Could not connect to the given server - no response", + )), + } } /// Check if the server is online. @@ -951,10 +965,7 @@ impl PyRaphtoryClient { let data = self.query_with_json_variables(query.clone(), variables.into())?; match data.get("receiveGraph") { Some(JsonValue::String(graph)) => { - let decoded_bytes = general_purpose::STANDARD - .decode(graph.clone()) - .map_err(|err| PyException::new_err(format!("Base64 decode error: {}", err)))?; - let mat_graph = MaterializedGraph::from_bincode(&decoded_bytes)?; + let mat_graph = url_decode_graph(graph)?; Ok(mat_graph) } _ => Err(PyException::new_err(format!( diff --git a/raphtory-graphql/src/python/mod.rs b/raphtory-graphql/src/python/mod.rs index 51dfed10cd..363054a62e 100644 --- a/raphtory-graphql/src/python/mod.rs +++ b/raphtory-graphql/src/python/mod.rs @@ -1,2 +1,12 @@ +use crate::url_encode::UrlDecodeError; +use pyo3::PyErr; +use raphtory::python::utils::errors::adapt_err_value; + pub mod graphql; pub mod pymodule; + +impl From for PyErr { + fn from(value: UrlDecodeError) -> Self { + adapt_err_value(&value) + } +} diff --git a/raphtory-graphql/src/python/pymodule.rs b/raphtory-graphql/src/python/pymodule.rs index c8697ae8b2..ce79277fa6 100644 --- a/raphtory-graphql/src/python/pymodule.rs +++ b/raphtory-graphql/src/python/pymodule.rs @@ -1,13 +1,13 @@ use crate::python::graphql::{ - PyGlobalPlugins, PyRaphtoryClient, PyRaphtoryServer, PyRunningRaphtoryServer, + PyGlobalPlugins, PyGraphServer, PyRaphtoryClient, PyRunningGraphServer, }; use pyo3::{prelude::PyModule, PyErr, Python}; pub fn base_graphql_module(py: Python<'_>) -> Result<&PyModule, PyErr> { let graphql_module = PyModule::new(py, "graphql")?; graphql_module.add_class::()?; - graphql_module.add_class::()?; - graphql_module.add_class::()?; + graphql_module.add_class::()?; + graphql_module.add_class::()?; graphql_module.add_class::()?; return Ok(graphql_module); } diff --git a/raphtory-graphql/src/server.rs b/raphtory-graphql/src/server.rs index f5d2606829..e194bfee86 100644 --- a/raphtory-graphql/src/server.rs +++ b/raphtory-graphql/src/server.rs @@ -74,12 +74,12 @@ impl From for io::Error { } /// A struct for defining and running a Raphtory GraphQL server -pub struct RaphtoryServer { +pub struct GraphServer { data: Data, configs: AppConfig, } -impl RaphtoryServer { +impl GraphServer { pub fn new( work_dir: PathBuf, app_config: Option, @@ -174,12 +174,12 @@ impl RaphtoryServer { } /// Start the server on the default port and return a handle to it. - pub async fn start(self) -> IoResult { + pub async fn start(self) -> IoResult { self.start_with_port(1736).await } /// Start the server on the port `port` and return a handle to it. - pub async fn start_with_port(self, port: u16) -> IoResult { + pub async fn start_with_port(self, port: u16) -> IoResult { fn configure_logger(configs: &LoggingConfig) { let log_level = &configs.log_level; let filter = EnvFilter::new(log_level); @@ -217,7 +217,7 @@ impl RaphtoryServer { .run_with_graceful_shutdown(app, server_termination(signal_receiver), None); let server_result = tokio::spawn(server_task); - Ok(RunningRaphtoryServer { + Ok(RunningGraphServer { signal_sender, server_result, }) @@ -346,12 +346,12 @@ impl RaphtoryServer { } /// A Raphtory server handler -pub struct RunningRaphtoryServer { +pub struct RunningGraphServer { signal_sender: Sender<()>, server_result: JoinHandle>, } -impl RunningRaphtoryServer { +impl RunningGraphServer { /// Stop the server. pub async fn stop(&self) { let _ignored = self.signal_sender.send(()).await; @@ -405,14 +405,14 @@ async fn server_termination(mut internal_signal: Receiver<()>) { mod server_tests { extern crate chrono; - use crate::server::RaphtoryServer; + use crate::server::GraphServer; use chrono::prelude::*; use tokio::time::{sleep, Duration}; #[tokio::test] async fn test_server_start_stop() { let tmp_dir = tempfile::tempdir().unwrap(); - let server = RaphtoryServer::new(tmp_dir.path().to_path_buf(), None, None).unwrap(); + let server = GraphServer::new(tmp_dir.path().to_path_buf(), None, None).unwrap(); println!("Calling start at time {}", Local::now()); let handler = server.start_with_port(0); sleep(Duration::from_secs(1)).await; diff --git a/raphtory-graphql/src/url_encode.rs b/raphtory-graphql/src/url_encode.rs index c51688a011..95a43fd02f 100644 --- a/raphtory-graphql/src/url_encode.rs +++ b/raphtory-graphql/src/url_encode.rs @@ -1,5 +1,9 @@ use base64::{prelude::BASE64_URL_SAFE, DecodeError, Engine}; -use raphtory::{core::utils::errors::GraphError, db::api::view::MaterializedGraph}; +use raphtory::{ + core::utils::errors::GraphError, + db::api::view::MaterializedGraph, + serialise::{StableDecode, StableEncode}, +}; #[derive(thiserror::Error, Debug)] pub enum UrlDecodeError { @@ -17,11 +21,11 @@ pub enum UrlDecodeError { pub fn url_encode_graph>(graph: G) -> Result { let g: MaterializedGraph = graph.into(); - Ok(BASE64_URL_SAFE.encode(g.bincode()?)) + Ok(BASE64_URL_SAFE.encode(g.encode_to_vec())) } pub fn url_decode_graph>(graph: T) -> Result { - Ok(MaterializedGraph::from_bincode( + Ok(MaterializedGraph::decode_from_bytes( &BASE64_URL_SAFE.decode(graph)?, )?) } diff --git a/raphtory/Cargo.toml b/raphtory/Cargo.toml index 8a2b23efdb..4001dae7ab 100644 --- a/raphtory/Cargo.toml +++ b/raphtory/Cargo.toml @@ -15,8 +15,7 @@ homepage.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -raphtory-api = { path = "../raphtory-api", version = "0.10.0" } -bincode = { workspace = true } +raphtory-api = { path = "../raphtory-api", version = "0.11.0" } chrono = { workspace = true } itertools = { workspace = true } num-traits = { workspace = true } @@ -31,7 +30,6 @@ rustc-hash = { workspace = true } serde = { workspace = true } sorted_vector_map = { workspace = true } thiserror = { workspace = true } -twox-hash = { workspace = true } lock_api = { workspace = true } dashmap = { workspace = true } enum_dispatch = { workspace = true } @@ -41,6 +39,8 @@ quad-rand = { workspace = true } serde_json = { workspace = true } ouroboros = { workspace = true } either = { workspace = true } +kdam = { workspace = true } + # io optional dependencies csv = { workspace = true, optional = true } @@ -58,6 +58,7 @@ tantivy = { workspace = true, optional = true } futures-util = { workspace = true, optional = true } async-trait = { workspace = true, optional = true } async-openai = { workspace = true, optional = true } +bincode = { workspace = true, optional = true } # python binding optional dependencies pyo3 = { workspace = true, optional = true } @@ -65,19 +66,12 @@ num = { workspace = true, optional = true } display-error-chain = { workspace = true, optional = true } polars-arrow = { workspace = true, optional = true } polars-parquet = { workspace = true, optional = true } -polars-utils = { workspace = true, optional = true } -kdam = { workspace = true, optional = true } memmap2 = { workspace = true, optional = true } -ahash = { workspace = true, optional = true } tempfile = { workspace = true, optional = true } -bytemuck = { workspace = true, optional = true } -rpds = { workspace = true, optional = true } -thread_local = { workspace = true, optional = true } pometry-storage = { workspace = true, optional = true } -prost = { workspace = true, optional = true} -prost-types = { workspace = true, optional = true} -bytes = { workspace = true, optional = true} +prost = { workspace = true, optional = true } +prost-types = { workspace = true, optional = true } [dev-dependencies] csv = { workspace = true } @@ -85,14 +79,13 @@ pretty_assertions = { workspace = true } quickcheck = { workspace = true } quickcheck_macros = { workspace = true } tempfile = { workspace = true } -tempdir = { workspace = true } tokio = { workspace = true } # for vector testing dotenv = { workspace = true } # for vector testing streaming-stats = { workspace = true } proptest = { workspace = true } [build-dependencies] -prost-build = { workspace = true, optional = true} +prost-build = { workspace = true, optional = true } [features] default = [] @@ -105,12 +98,13 @@ io = [ "dep:csv", "dep:reqwest", "dep:tokio", + "proto", ] # search search = ["dep:tantivy"] # vectors -vectors = ["dep:futures-util", "dep:async-trait", "dep:async-openai"] +vectors = ["dep:futures-util", "dep:async-trait", "dep:async-openai", "dep:bincode"] # Enables generating the pyo3 python bindings python = [ @@ -124,22 +118,15 @@ python = [ "dep:display-error-chain", "polars-arrow?/compute", "raphtory-api/python", - "dep:rpds", - "dep:kdam", - "kdam?/notebook" + "kdam/notebook" ] # storage storage = [ "arrow", "pometry-storage", - "dep:polars-utils", "dep:memmap2", - "dep:ahash", "dep:tempfile", - "dep:bytemuck", - "dep:rpds", - "dep:thread_local", "polars-arrow?/io_ipc", "polars-arrow?/arrow_rs", ] @@ -152,7 +139,6 @@ arrow = [ proto = [ "dep:prost", "dep:prost-types", - "dep:bytes", "dep:prost-build", "dep:memmap2", ] diff --git a/raphtory/build.rs b/raphtory/build.rs index 1ff0547e1d..be1eda9fde 100644 --- a/raphtory/build.rs +++ b/raphtory/build.rs @@ -1,7 +1,8 @@ use std::io::Result; #[cfg(feature = "proto")] fn main() -> Result<()> { - prost_build::compile_protos(&["src/graph.proto"], &["src/"])?; + prost_build::compile_protos(&["src/serialise/graph.proto"], &["src/serialise"])?; + println!("cargo::rerun-if-changed=src/serialise/graph.proto"); Ok(()) } diff --git a/raphtory/src/algorithms/pathing/temporal_reachability.rs b/raphtory/src/algorithms/pathing/temporal_reachability.rs index df487e4fea..62a61d7ffc 100644 --- a/raphtory/src/algorithms/pathing/temporal_reachability.rs +++ b/raphtory/src/algorithms/pathing/temporal_reachability.rs @@ -191,10 +191,12 @@ pub fn temporally_reachable_nodes( None, |_, ess, _, _| { ess.finalize(&taint_history, |taint_history| { - taint_history + let mut hist = taint_history .into_iter() .map(|tmsg| (tmsg.event_time, tmsg.src_node)) - .collect_vec() + .collect_vec(); + hist.sort(); + hist }) }, threads, diff --git a/raphtory/src/core/entities/edges/edge_store.rs b/raphtory/src/core/entities/edges/edge_store.rs index cb2376f026..c4defa93d8 100644 --- a/raphtory/src/core/entities/edges/edge_store.rs +++ b/raphtory/src/core/entities/edges/edge_store.rs @@ -13,7 +13,7 @@ use crate::{ Prop, }, db::api::{ - storage::edges::edge_storage_ops::{EdgeStorageIntoOps, EdgeStorageOps}, + storage::graph::edges::edge_storage_ops::{EdgeStorageIntoOps, EdgeStorageOps}, view::IntoDynBoxed, }, }; diff --git a/raphtory/src/core/entities/graph/logical_to_physical.rs b/raphtory/src/core/entities/graph/logical_to_physical.rs index 374fe98c86..559baa3e1d 100644 --- a/raphtory/src/core/entities/graph/logical_to_physical.rs +++ b/raphtory/src/core/entities/graph/logical_to_physical.rs @@ -1,12 +1,12 @@ +use crate::core::utils::errors::{GraphError, MutateGraphError}; +use dashmap::mapref::entry::Entry; use once_cell::sync::OnceCell; - use raphtory_api::core::{ entities::{GidRef, VID}, - storage::FxDashMap, + storage::{dict_mapper::MaybeNew, FxDashMap}, }; use serde::{Deserialize, Deserializer, Serialize}; - -use crate::core::utils::errors::{GraphError, MutateGraphError}; +use std::hash::Hash; #[derive(Debug, Deserialize, Serialize)] enum Map { @@ -52,13 +52,13 @@ impl Mapping { &self, gid: GidRef, f_init: impl FnOnce() -> VID, - ) -> Result { + ) -> Result, GraphError> { let map = self.map.get_or_init(|| match &gid { GidRef::U64(_) => Map::U64(FxDashMap::default()), GidRef::Str(_) => Map::Str(FxDashMap::default()), }); let vid = match gid { - GidRef::U64(id) => map.as_u64().map(|m| *(m.entry(id).or_insert_with(f_init))), + GidRef::U64(id) => map.as_u64().map(|m| get_or_new(m, id, f_init)), GidRef::Str(id) => map.as_str().map(|m| optim_get_or_insert(m, id, f_init)), }; vid.ok_or(GraphError::FailedToMutateGraph { @@ -77,10 +77,31 @@ impl Mapping { } } -fn optim_get_or_insert(m: &FxDashMap, id: &str, f_init: impl FnOnce() -> VID) -> VID { +#[inline] +fn optim_get_or_insert( + m: &FxDashMap, + id: &str, + f_init: impl FnOnce() -> VID, +) -> MaybeNew { m.get(id) - .map(|vid| *vid) - .unwrap_or_else(|| *(m.entry(id.to_owned()).or_insert_with(f_init))) + .map(|vid| MaybeNew::Existing(*vid)) + .unwrap_or_else(|| get_or_new(m, id.to_owned(), f_init)) +} + +#[inline] +fn get_or_new( + m: &FxDashMap, + id: K, + f_init: impl FnOnce() -> VID, +) -> MaybeNew { + match m.entry(id) { + Entry::Occupied(entry) => MaybeNew::Existing(*entry.get()), + Entry::Vacant(entry) => { + let id = f_init(); + entry.insert(id); + MaybeNew::New(id) + } + } } impl<'de> Deserialize<'de> for Mapping { diff --git a/raphtory/src/core/entities/graph/mod.rs b/raphtory/src/core/entities/graph/mod.rs index a101097477..2c0c4e997e 100644 --- a/raphtory/src/core/entities/graph/mod.rs +++ b/raphtory/src/core/entities/graph/mod.rs @@ -5,37 +5,27 @@ pub(crate) mod timer; #[cfg(test)] mod test { - use crate::{ - core::PropType, - db::api::{mutation::internal::InternalAdditionOps, storage::storage_ops::GraphStorage}, - prelude::*, - }; + use crate::{core::PropType, db::api::mutation::internal::InternalAdditionOps, prelude::*}; #[test] fn test_neighbours_multiple_layers() { - let g = GraphStorage::default(); - let l_btc = g.resolve_layer(Some("btc")).unwrap(); - let l_eth = g.resolve_layer(Some("eth")).unwrap(); - let l_tether = g.resolve_layer(Some("tether")).unwrap(); - let v1 = g.resolve_node(1).unwrap(); - let v2 = g.resolve_node(2).unwrap(); + let g = Graph::default(); + let l_btc = g.resolve_layer(Some("btc")).unwrap().inner(); + let l_eth = g.resolve_layer(Some("eth")).unwrap().inner(); + let l_tether = g.resolve_layer(Some("tether")).unwrap().inner(); + let v1 = g.resolve_node(1).unwrap().inner(); + let v2 = g.resolve_node(2).unwrap().inner(); let tx_sent_id = g .resolve_edge_property("tx_sent", PropType::I32, false) + .unwrap() + .inner(); + g.internal_add_edge(1.into(), v1, v2, &[(tx_sent_id, Prop::I32(10))], l_btc) .unwrap(); - g.internal_add_edge(1.into(), v1, v2, vec![(tx_sent_id, Prop::I32(10))], l_btc) + g.internal_add_edge(1.into(), v1, v2, &[(tx_sent_id, Prop::I32(20))], l_eth) .unwrap(); - g.internal_add_edge(1.into(), v1, v2, vec![(tx_sent_id, Prop::I32(20))], l_eth) + g.internal_add_edge(1.into(), v1, v2, &[(tx_sent_id, Prop::I32(70))], l_tether) .unwrap(); - g.internal_add_edge( - 1.into(), - v1, - v2, - vec![(tx_sent_id, Prop::I32(70))], - l_tether, - ) - .unwrap(); - let g = Graph::from_internal_graph(g); let first = g .node(v1) .and_then(|n| n.layers(["btc", "eth"]).ok()) diff --git a/raphtory/src/core/entities/graph/tgraph.rs b/raphtory/src/core/entities/graph/tgraph.rs index a0fd464221..e10434c3c1 100644 --- a/raphtory/src/core/entities/graph/tgraph.rs +++ b/raphtory/src/core/entities/graph/tgraph.rs @@ -19,14 +19,14 @@ use crate::{ utils::errors::GraphError, Direction, Prop, }, - db::api::{storage::edges::edge_storage_ops::EdgeStorageOps, view::Layer}, + db::api::{storage::graph::edges::edge_storage_ops::EdgeStorageOps, view::Layer}, }; use dashmap::DashSet; use itertools::Itertools; use raphtory_api::core::{ entities::{edges::edge_ref::EdgeRef, GidRef}, input::input_node::InputNode, - storage::arc_str::ArcStr, + storage::{arc_str::ArcStr, dict_mapper::MaybeNew}, }; use rustc_hash::FxHasher; use serde::{Deserialize, Serialize}; @@ -98,10 +98,10 @@ impl TemporalGraph { } } - pub(crate) fn process_prop_value(&self, prop: Prop) -> Prop { + pub(crate) fn process_prop_value(&self, prop: &Prop) -> Prop { match prop { Prop::Str(value) => Prop::Str(self.resolve_str(value)), - _ => prop, + _ => prop.clone(), } } @@ -365,15 +365,15 @@ impl TemporalGraph { t: TimeIndexEntry, layer: usize, edge_fn: F, - ) -> Result { + ) -> Result, GraphError> { let mut node_pair = self.storage.pair_node_mut(src_id, dst_id); let src = node_pair.get_i(); let mut edge = match src.find_edge_eid(dst_id, &LayerIds::All) { - Some(edge_id) => self.storage.get_edge_mut(edge_id), - None => self.storage.push_edge(EdgeStore::new(src_id, dst_id)), + Some(edge_id) => MaybeNew::Existing(self.storage.get_edge_mut(edge_id)), + None => MaybeNew::New(self.storage.push_edge(EdgeStore::new(src_id, dst_id))), }; - self.link_nodes_inner(&mut node_pair, &mut edge, t, layer, edge_fn)?; - Ok(edge.edge_store().eid) + self.link_nodes_inner(&mut node_pair, edge.as_mut().inner(), t, layer, edge_fn)?; + Ok(edge.map(|e| e.edge_store().eid)) } pub(crate) fn resolve_node_ref(&self, v: NodeRef) -> Option { @@ -389,18 +389,15 @@ impl TemporalGraph { /// Checks if the same string value already exists and returns a pointer to the same existing value if it exists, /// otherwise adds the string to the pool. - fn resolve_str(&self, value: ArcStr) -> ArcStr { - match self.string_pool.get(&value) { + fn resolve_str(&self, value: &ArcStr) -> ArcStr { + match self.string_pool.get(value) { Some(value) => value.clone(), None => { - if self.string_pool.insert(value.clone()) { - value - } else { - self.string_pool - .get(&value) - .expect("value exists due to insert above returning false") - .clone() - } + self.string_pool.insert(value.clone()); + self.string_pool + .get(value) + .expect("value should exist as inserted above") + .clone() } } } diff --git a/raphtory/src/core/entities/nodes/node_ref.rs b/raphtory/src/core/entities/nodes/node_ref.rs index 00073c93c8..a6d7280ee4 100644 --- a/raphtory/src/core/entities/nodes/node_ref.rs +++ b/raphtory/src/core/entities/nodes/node_ref.rs @@ -72,6 +72,12 @@ impl AsNodeRef for GID { } } +impl<'a> AsNodeRef for GidRef<'a> { + fn as_node_ref(&self) -> NodeRef { + NodeRef::External(*self) + } +} + impl<'a> NodeRef<'a> { /// Makes a new node reference from an internal `VID`. /// Values are unchecked and the node is assumed to exist so use with caution! diff --git a/raphtory/src/core/entities/properties/graph_meta.rs b/raphtory/src/core/entities/properties/graph_meta.rs index 98ee907f86..ae4afc4613 100644 --- a/raphtory/src/core/entities/properties/graph_meta.rs +++ b/raphtory/src/core/entities/properties/graph_meta.rs @@ -5,7 +5,10 @@ use crate::core::{ Prop, PropType, }; use raphtory_api::core::storage::{ - arc_str::ArcStr, dict_mapper::DictMapper, locked_vec::ArcReadLockedVec, FxDashMap, + arc_str::ArcStr, + dict_mapper::{DictMapper, MaybeNew}, + locked_vec::ArcReadLockedVec, + FxDashMap, }; use serde::{Deserialize, Serialize}; use std::ops::{Deref, DerefMut}; @@ -44,7 +47,7 @@ impl GraphMeta { name: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { if is_static { Ok(self.constant_mapper.get_or_create_id(name)) } else { diff --git a/raphtory/src/core/entities/properties/props.rs b/raphtory/src/core/entities/properties/props.rs index b56c581e51..aa694c85cf 100644 --- a/raphtory/src/core/entities/properties/props.rs +++ b/raphtory/src/core/entities/properties/props.rs @@ -8,11 +8,13 @@ use crate::{ utils::errors::GraphError, Prop, PropType, }, - db::api::storage::tprop_storage_ops::TPropOps, + db::api::storage::graph::tprop_storage_ops::TPropOps, }; use parking_lot::RwLock; use raphtory_api::core::storage::{ - arc_str::ArcStr, dict_mapper::DictMapper, locked_vec::ArcReadLockedVec, + arc_str::ArcStr, + dict_mapper::{DictMapper, MaybeNew}, + locked_vec::ArcReadLockedVec, }; use serde::{Deserialize, Serialize}; use std::{fmt::Debug, hash::Hash, ops::Deref, sync::Arc}; @@ -153,7 +155,7 @@ impl Meta { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { if is_static { self.meta_prop_constant .get_or_create_and_validate(prop, dtype) @@ -173,7 +175,7 @@ impl Meta { } #[inline] - pub fn get_or_create_layer_id(&self, name: &str) -> usize { + pub fn get_or_create_layer_id(&self, name: &str) -> MaybeNew { self.meta_layer.get_or_create_id(name) } @@ -183,7 +185,7 @@ impl Meta { } #[inline] - pub fn get_or_create_node_type_id(&self, node_type: &str) -> usize { + pub fn get_or_create_node_type_id(&self, node_type: &str) -> MaybeNew { self.meta_node_type.get_or_create_id(node_type) } @@ -264,13 +266,14 @@ impl PropMapper { &self, prop: &str, dtype: PropType, - ) -> Result { - let id = self.id_mapper.get_or_create_id(prop); + ) -> Result, GraphError> { + let wrapped_id = self.id_mapper.get_or_create_id(prop); + let id = wrapped_id.inner(); let dtype_read = self.dtypes.read_recursive(); if let Some(old_type) = dtype_read.get(id) { if !matches!(old_type, PropType::Empty) { return if *old_type == dtype { - Ok(id) + Ok(wrapped_id) } else { Err(GraphError::PropertyTypeError { name: prop.to_owned(), @@ -287,11 +290,11 @@ impl PropMapper { if matches!(old_type, PropType::Empty) { // vector already resized but this id is not filled yet, set the dtype and return id dtype_write[id] = dtype; - Ok(id) + Ok(wrapped_id) } else { // already filled because a different thread won the race for this id, check the type matches if old_type == dtype { - Ok(id) + Ok(wrapped_id) } else { Err(GraphError::PropertyTypeError { name: prop.to_owned(), @@ -305,7 +308,7 @@ impl PropMapper { // vector not resized yet, resize it and set the dtype and return id dtype_write.resize(id + 1, PropType::Empty); dtype_write[id] = dtype; - Ok(id) + Ok(wrapped_id) } } } diff --git a/raphtory/src/core/entities/properties/tprop.rs b/raphtory/src/core/entities/properties/tprop.rs index 223854d356..3a8351e4f7 100644 --- a/raphtory/src/core/entities/properties/tprop.rs +++ b/raphtory/src/core/entities/properties/tprop.rs @@ -4,7 +4,7 @@ use crate::{ utils::errors::GraphError, DocumentInput, Prop, PropType, }, db::{ - api::storage::tprop_storage_ops::TPropOps, + api::storage::graph::tprop_storage_ops::TPropOps, graph::{graph::Graph, views::deletion_graph::PersistentGraph}, }, }; diff --git a/raphtory/src/core/storage/locked_view.rs b/raphtory/src/core/storage/locked_view.rs index a0addb3224..ee1d5df9f7 100644 --- a/raphtory/src/core/storage/locked_view.rs +++ b/raphtory/src/core/storage/locked_view.rs @@ -1,18 +1,17 @@ use dashmap::mapref::one::Ref; use parking_lot::{MappedRwLockReadGuard, RwLockReadGuard}; -use rustc_hash::FxHasher; use std::{ borrow::Borrow, cmp::Ordering, fmt::{Debug, Formatter}, - hash::{BuildHasherDefault, Hash, Hasher}, + hash::{Hash, Hasher}, ops::Deref, }; pub enum LockedView<'a, T> { LockMapped(MappedRwLockReadGuard<'a, T>), Locked(RwLockReadGuard<'a, T>), - DashMap(Ref<'a, usize, T, BuildHasherDefault>), + DashMap(Ref<'a, usize, T>), } impl<'a, T, O> AsRef for LockedView<'a, O> @@ -77,8 +76,8 @@ impl<'a, T> From> for LockedView<'a, T> { } } -impl<'a, T> From>> for LockedView<'a, T> { - fn from(value: Ref<'a, usize, T, BuildHasherDefault>) -> Self { +impl<'a, T> From> for LockedView<'a, T> { + fn from(value: Ref<'a, usize, T>) -> Self { Self::DashMap(value) } } diff --git a/raphtory/src/core/storage/raw_edges.rs b/raphtory/src/core/storage/raw_edges.rs index 5d16e08602..b2edcf5e77 100644 --- a/raphtory/src/core/storage/raw_edges.rs +++ b/raphtory/src/core/storage/raw_edges.rs @@ -18,7 +18,7 @@ use crate::{ edges::edge_store::{EdgeDataLike, EdgeLayer, EdgeStore}, LayerIds, }, - db::api::storage::edges::edge_storage_ops::{EdgeStorageOps, MemEdge}, + db::api::storage::graph::edges::edge_storage_ops::{EdgeStorageOps, MemEdge}, }; use super::{resolve, timeindex::TimeIndex}; diff --git a/raphtory/src/core/utils/errors.rs b/raphtory/src/core/utils/errors.rs index 8f41d0ac8b..4dfcb66f57 100644 --- a/raphtory/src/core/utils/errors.rs +++ b/raphtory/src/core/utils/errors.rs @@ -1,6 +1,10 @@ use crate::core::{utils::time::error::ParseTimeError, Prop, PropType}; #[cfg(feature = "arrow")] use polars_arrow::legacy::error; +#[cfg(feature = "storage")] +use pometry_storage::RAError; +#[cfg(feature = "python")] +use pyo3::PyErr; use raphtory_api::core::{entities::GID, storage::arc_str::ArcStr}; use std::path::PathBuf; #[cfg(feature = "search")] @@ -36,6 +40,8 @@ pub enum InvalidPathReason { #[derive(thiserror::Error, Debug)] pub enum GraphError { + #[error("You cannot set ‘{0}’ and ‘{1}’ at the same time. Please pick one or the other.")] + WrongNumOfArgs(String, String), #[cfg(feature = "arrow")] #[error("Arrow error: {0}")] Arrow(#[from] error::PolarsError), @@ -118,12 +124,6 @@ pub enum GraphError { src: String, dst: String, }, - #[error("Bincode operation failed")] - BinCodeError { - #[from] - source: Box, - }, - #[error("The loaded graph is of the wrong type. Did you mean Graph / PersistentGraph?")] GraphLoadError, @@ -143,6 +143,10 @@ pub enum GraphError { )] ColumnDoesNotExist(String), + #[cfg(feature = "storage")] + #[error("Raphtory Arrow Error: {0}")] + DiskGraphError(#[from] RAError), + #[cfg(feature = "search")] #[error("Index operation failed")] IndexError { @@ -157,11 +161,6 @@ pub enum GraphError { source: QueryParserError, }, - #[error( - "Failed to load the graph as the bincode version {0} is different to supported version {1}" - )] - BincodeVersionError(u32, u32), - #[error("The layer_name function is only available once an edge has been exploded via .explode_layers() or .explode(). If you want to retrieve the layers for this edge you can use .layer_names")] LayerNameAPIError, @@ -183,8 +182,18 @@ pub enum GraphError { #[error("Failed to deserialise graph: {0}")] DeserialisationError(String), + #[cfg(feature = "proto")] + #[error("Cache is not initialised")] + CacheNotInnitialised, + #[error("Immutable graph is .. immutable!")] AttemptToMutateImmutableGraph, + + #[cfg(feature = "python")] + #[error("Python error occurred: {0}")] + PythonError(#[from] PyErr), + #[error("An error with Tdqm occurred")] + TqdmError, } impl GraphError { diff --git a/raphtory/src/db/api/mutation/addition_ops.rs b/raphtory/src/db/api/mutation/addition_ops.rs index fc31c05364..6068fab448 100644 --- a/raphtory/src/db/api/mutation/addition_ops.rs +++ b/raphtory/src/db/api/mutation/addition_ops.rs @@ -108,14 +108,18 @@ impl AdditionOps for G { props: PI, node_type: Option<&str>, ) -> Result, GraphError> { - let properties = props - .collect_properties(|name, dtype| self.resolve_node_property(name, dtype, false))?; let ti = time_from_input(self, t)?; - let v_id = self.resolve_node(v)?; - if let Some(node_type) = node_type { - self.set_node_type(v_id, node_type)?; - } - self.internal_add_node(ti, v_id, properties)?; + let properties = props.collect_properties(|name, dtype| { + Ok(self.resolve_node_property(name, dtype, false)?.inner()) + })?; + let v_id = match node_type { + None => self.resolve_node(v)?.inner(), + Some(node_type) => { + let (v_id, _) = self.resolve_node_and_type(v, node_type)?.inner(); + v_id.inner() + } + }; + self.internal_add_node(ti, v_id, &properties)?; Ok(NodeView::new_internal(self.clone(), v_id)) } @@ -128,13 +132,16 @@ impl AdditionOps for G { layer: Option<&str>, ) -> Result, GraphError> { let ti = time_from_input(self, t)?; - let src_id = self.resolve_node(src)?; - let dst_id = self.resolve_node(dst)?; - let layer_id = self.resolve_layer(layer)?; + let src_id = self.resolve_node(src)?.inner(); + let dst_id = self.resolve_node(dst)?.inner(); + let layer_id = self.resolve_layer(layer)?.inner(); - let properties: Vec<(usize, Prop)> = props - .collect_properties(|name, dtype| self.resolve_edge_property(name, dtype, false))?; - let eid = self.internal_add_edge(ti, src_id, dst_id, properties, layer_id)?; + let properties: Vec<(usize, Prop)> = props.collect_properties(|name, dtype| { + Ok(self.resolve_edge_property(name, dtype, false)?.inner()) + })?; + let eid = self + .internal_add_edge(ti, src_id, dst_id, &properties, layer_id)? + .inner(); Ok(EdgeView::new( self.clone(), EdgeRef::new_outgoing(eid, src_id, dst_id).at_layer(layer_id), diff --git a/raphtory/src/db/api/mutation/deletion_ops.rs b/raphtory/src/db/api/mutation/deletion_ops.rs index 007cdf4925..f3b040fc03 100644 --- a/raphtory/src/db/api/mutation/deletion_ops.rs +++ b/raphtory/src/db/api/mutation/deletion_ops.rs @@ -4,25 +4,40 @@ use crate::{ entities::nodes::node_ref::AsNodeRef, utils::{errors::GraphError, time::IntoTimeWithFormat}, }, - db::api::mutation::{ - internal::{InternalAdditionOps, InternalDeletionOps}, - TryIntoInputTime, + db::{ + api::{ + mutation::{ + internal::{InternalAdditionOps, InternalDeletionOps}, + TryIntoInputTime, + }, + view::StaticGraphViewOps, + }, + graph::edge::EdgeView, }, }; +use raphtory_api::core::entities::edges::edge_ref::EdgeRef; -pub trait DeletionOps: InternalDeletionOps + InternalAdditionOps + Sized { +pub trait DeletionOps: + InternalDeletionOps + InternalAdditionOps + StaticGraphViewOps + Sized +{ fn delete_edge( &self, t: T, src: V, dst: V, layer: Option<&str>, - ) -> Result<(), GraphError> { + ) -> Result, GraphError> { let ti = time_from_input(self, t)?; - let src_id = self.resolve_node(src)?; - let dst_id = self.resolve_node(dst)?; - let layer = self.resolve_layer(layer)?; - self.internal_delete_edge(ti, src_id, dst_id, layer) + let src_id = self.resolve_node(src)?.inner(); + let dst_id = self.resolve_node(dst)?.inner(); + let layer = self.resolve_layer(layer)?.inner(); + let eid = self + .internal_delete_edge(ti, src_id, dst_id, layer)? + .inner(); + Ok(EdgeView::new( + self.clone(), + EdgeRef::new_outgoing(eid, src_id, dst_id).at_layer(layer), + )) } fn delete_edge_with_custom_time_format( @@ -32,7 +47,7 @@ pub trait DeletionOps: InternalDeletionOps + InternalAdditionOps + Sized { src: V, dst: V, layer: Option<&str>, - ) -> Result<(), GraphError> { + ) -> Result, GraphError> { let time: i64 = t.parse_time(fmt)?; self.delete_edge(time, src, dst, layer) } diff --git a/raphtory/src/db/api/mutation/import_ops.rs b/raphtory/src/db/api/mutation/import_ops.rs index c5f1b32850..02cc94ce42 100644 --- a/raphtory/src/db/api/mutation/import_ops.rs +++ b/raphtory/src/db/api/mutation/import_ops.rs @@ -128,15 +128,17 @@ impl< if !force && self.node(node.id()).is_some() { return Err(NodeExistsError(node.id())); } - - let node_internal = self.resolve_node(node.id())?; - if let Some(node_type) = node.node_type().as_str() { - self.set_node_type(node_internal, node_type)?; - } + let node_internal = match node.node_type().as_str() { + None => self.resolve_node(node.id())?.inner(), + Some(node_type) => { + let (node_internal, _) = self.resolve_node_and_type(node.id(), node_type)?.inner(); + node_internal.inner() + } + }; for h in node.history() { let t = time_from_input(self, h)?; - self.internal_add_node(t, node_internal, vec![])?; + self.internal_add_node(t, node_internal, &[])?; } for (name, prop_view) in node.properties().temporal().iter() { let old_prop_id = node @@ -151,10 +153,10 @@ impl< .temporal_prop_meta() .get_dtype(old_prop_id) .unwrap(); - let new_prop_id = self.resolve_node_property(&name, dtype, false)?; + let new_prop_id = self.resolve_node_property(&name, dtype, false)?.inner(); for (h, prop) in prop_view.iter() { let t = time_from_input(self, h)?; - self.internal_add_node(t, node_internal, vec![(new_prop_id, prop)])?; + self.internal_add_node(t, node_internal, &[(new_prop_id, prop)])?; } } self.node(node.id()) @@ -212,9 +214,9 @@ impl< if self.include_deletions() { for t in edge.graph.edge_deletion_history(edge.edge, &layer_ids) { let ti = time_from_input(self, t)?; - let src_id = self.resolve_node(edge.src().id())?; - let dst_id = self.resolve_node(edge.dst().id())?; - let layer = self.resolve_layer(layer_name)?; + let src_id = self.resolve_node(edge.src().id())?.inner(); + let dst_id = self.resolve_node(edge.dst().id())?.inner(); + let layer = self.resolve_layer(layer_name)?.inner(); self.internal_delete_edge(ti, src_id, dst_id, layer)?; } } diff --git a/raphtory/src/db/api/mutation/internal/internal_addition_ops.rs b/raphtory/src/db/api/mutation/internal/internal_addition_ops.rs index b3a02bf3d3..e5ec60bbda 100644 --- a/raphtory/src/db/api/mutation/internal/internal_addition_ops.rs +++ b/raphtory/src/db/api/mutation/internal/internal_addition_ops.rs @@ -8,6 +8,7 @@ use crate::{ db::api::view::internal::Base, }; use enum_dispatch::enum_dispatch; +use raphtory_api::core::storage::dict_mapper::MaybeNew; #[enum_dispatch] pub trait InternalAdditionOps { @@ -15,12 +16,17 @@ pub trait InternalAdditionOps { fn next_event_id(&self) -> Result; /// map layer name to id and allocate a new layer if needed - fn resolve_layer(&self, layer: Option<&str>) -> Result; - - fn set_node_type(&self, v_id: VID, node_type: &str) -> Result<(), GraphError>; + fn resolve_layer(&self, layer: Option<&str>) -> Result, GraphError>; /// map external node id to internal id, allocating a new empty node if needed - fn resolve_node(&self, id: V) -> Result; + fn resolve_node(&self, id: V) -> Result, GraphError>; + + /// resolve a node and corresponding type, outer MaybeNew tracks whether the type assignment is new for the node even if both node and type already existed. + fn resolve_node_and_type( + &self, + id: V, + node_type: &str, + ) -> Result, MaybeNew)>, GraphError>; /// map property key to internal id, allocating new property if needed fn resolve_graph_property( @@ -28,7 +34,7 @@ pub trait InternalAdditionOps { prop: &str, dtype: PropType, is_static: bool, - ) -> Result; + ) -> Result, GraphError>; /// map property key to internal id, allocating new property if needed and checking property type. /// returns `None` if the type does not match @@ -37,21 +43,21 @@ pub trait InternalAdditionOps { prop: &str, dtype: PropType, is_static: bool, - ) -> Result; + ) -> Result, GraphError>; fn resolve_edge_property( &self, prop: &str, dtype: PropType, is_static: bool, - ) -> Result; + ) -> Result, GraphError>; /// add node update fn internal_add_node( &self, t: TimeIndexEntry, v: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError>; /// add edge update @@ -60,16 +66,16 @@ pub trait InternalAdditionOps { t: TimeIndexEntry, src: VID, dst: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, - ) -> Result; + ) -> Result, GraphError>; /// add update for an existing edge fn internal_add_edge_update( &self, t: TimeIndexEntry, edge: EID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, ) -> Result<(), GraphError>; } @@ -99,18 +105,22 @@ impl InternalAdditionOps for G { } #[inline] - fn resolve_layer(&self, layer: Option<&str>) -> Result { + fn resolve_layer(&self, layer: Option<&str>) -> Result, GraphError> { self.graph().resolve_layer(layer) } #[inline] - fn set_node_type(&self, v_id: VID, node_type: &str) -> Result<(), GraphError> { - self.graph().set_node_type(v_id, node_type) + fn resolve_node(&self, n: V) -> Result, GraphError> { + self.graph().resolve_node(n) } #[inline] - fn resolve_node(&self, n: V) -> Result { - self.graph().resolve_node(n) + fn resolve_node_and_type( + &self, + id: V, + node_type: &str, + ) -> Result, MaybeNew)>, GraphError> { + self.graph().resolve_node_and_type(id, node_type) } #[inline] @@ -119,7 +129,7 @@ impl InternalAdditionOps for G { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.graph().resolve_graph_property(prop, dtype, is_static) } @@ -129,7 +139,7 @@ impl InternalAdditionOps for G { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.graph().resolve_node_property(prop, dtype, is_static) } @@ -139,7 +149,7 @@ impl InternalAdditionOps for G { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.graph().resolve_edge_property(prop, dtype, is_static) } @@ -148,7 +158,7 @@ impl InternalAdditionOps for G { &self, t: TimeIndexEntry, v: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.graph().internal_add_node(t, v, props) } @@ -159,9 +169,9 @@ impl InternalAdditionOps for G { t: TimeIndexEntry, src: VID, dst: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, - ) -> Result { + ) -> Result, GraphError> { self.graph().internal_add_edge(t, src, dst, props, layer) } @@ -170,7 +180,7 @@ impl InternalAdditionOps for G { &self, t: TimeIndexEntry, edge: EID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, ) -> Result<(), GraphError> { self.graph().internal_add_edge_update(t, edge, props, layer) diff --git a/raphtory/src/db/api/mutation/internal/internal_deletion_ops.rs b/raphtory/src/db/api/mutation/internal/internal_deletion_ops.rs index 4585c62668..cd60c40ed5 100644 --- a/raphtory/src/db/api/mutation/internal/internal_deletion_ops.rs +++ b/raphtory/src/db/api/mutation/internal/internal_deletion_ops.rs @@ -2,7 +2,7 @@ use crate::{ core::{entities::VID, storage::timeindex::TimeIndexEntry, utils::errors::GraphError}, db::api::view::internal::Base, }; -use raphtory_api::core::entities::EID; +use raphtory_api::core::{entities::EID, storage::dict_mapper::MaybeNew}; pub trait InternalDeletionOps { fn internal_delete_edge( @@ -11,7 +11,7 @@ pub trait InternalDeletionOps { src: VID, dst: VID, layer: usize, - ) -> Result<(), GraphError>; + ) -> Result, GraphError>; fn internal_delete_existing_edge( &self, @@ -48,7 +48,7 @@ impl InternalDeletionOps for G { src: VID, dst: VID, layer: usize, - ) -> Result<(), GraphError> { + ) -> Result, GraphError> { self.graph().internal_delete_edge(t, src, dst, layer) } diff --git a/raphtory/src/db/api/mutation/internal/internal_property_additions_ops.rs b/raphtory/src/db/api/mutation/internal/internal_property_additions_ops.rs index f688fb612b..c5916ed316 100644 --- a/raphtory/src/db/api/mutation/internal/internal_property_additions_ops.rs +++ b/raphtory/src/db/api/mutation/internal/internal_property_additions_ops.rs @@ -15,41 +15,40 @@ pub trait InternalPropertyAdditionOps { fn internal_add_properties( &self, t: TimeIndexEntry, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError>; - fn internal_add_constant_properties(&self, props: Vec<(usize, Prop)>) - -> Result<(), GraphError>; + fn internal_add_constant_properties(&self, props: &[(usize, Prop)]) -> Result<(), GraphError>; fn internal_update_constant_properties( &self, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError>; fn internal_add_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError>; fn internal_update_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError>; fn internal_add_constant_edge_properties( &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError>; fn internal_update_constant_edge_properties( &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError>; } @@ -77,23 +76,20 @@ impl InternalPropertyAdditionOps for G { fn internal_add_properties( &self, t: TimeIndexEntry, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.graph().internal_add_properties(t, props) } #[inline(always)] - fn internal_add_constant_properties( - &self, - props: Vec<(usize, Prop)>, - ) -> Result<(), GraphError> { + fn internal_add_constant_properties(&self, props: &[(usize, Prop)]) -> Result<(), GraphError> { self.graph().internal_add_constant_properties(props) } #[inline(always)] fn internal_update_constant_properties( &self, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.graph().internal_update_constant_properties(props) } @@ -102,7 +98,7 @@ impl InternalPropertyAdditionOps for G { fn internal_add_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.graph() .internal_add_constant_node_properties(vid, props) @@ -112,7 +108,7 @@ impl InternalPropertyAdditionOps for G { fn internal_update_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.graph() .internal_update_constant_node_properties(vid, props) @@ -123,7 +119,7 @@ impl InternalPropertyAdditionOps for G { &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.graph() .internal_add_constant_edge_properties(eid, layer, props) @@ -134,7 +130,7 @@ impl InternalPropertyAdditionOps for G { &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.graph() .internal_update_constant_edge_properties(eid, layer, props) diff --git a/raphtory/src/db/api/mutation/property_addition_ops.rs b/raphtory/src/db/api/mutation/property_addition_ops.rs index abd6f0a1bb..7aec6632f6 100644 --- a/raphtory/src/db/api/mutation/property_addition_ops.rs +++ b/raphtory/src/db/api/mutation/property_addition_ops.rs @@ -29,23 +29,26 @@ impl PropertyAdditionOps f props: PI, ) -> Result<(), GraphError> { let ti = time_from_input(self, t)?; - let properties: Vec<_> = props - .collect_properties(|name, dtype| self.resolve_graph_property(name, dtype, false))?; - self.internal_add_properties(ti, properties) + let properties: Vec<_> = props.collect_properties(|name, dtype| { + Ok(self.resolve_graph_property(name, dtype, false)?.inner()) + })?; + self.internal_add_properties(ti, &properties) } fn add_constant_properties(&self, props: PI) -> Result<(), GraphError> { - let properties: Vec<_> = props - .collect_properties(|name, dtype| self.resolve_graph_property(name, dtype, true))?; - self.internal_add_constant_properties(properties) + let properties: Vec<_> = props.collect_properties(|name, dtype| { + Ok(self.resolve_graph_property(name, dtype, true)?.inner()) + })?; + self.internal_add_constant_properties(&properties) } fn update_constant_properties( &self, props: PI, ) -> Result<(), GraphError> { - let properties: Vec<_> = props - .collect_properties(|name, dtype| self.resolve_graph_property(name, dtype, true))?; - self.internal_update_constant_properties(properties) + let properties: Vec<_> = props.collect_properties(|name, dtype| { + Ok(self.resolve_graph_property(name, dtype, true)?.inner()) + })?; + self.internal_update_constant_properties(&properties) } } diff --git a/raphtory/src/db/api/state/lazy_node_state.rs b/raphtory/src/db/api/state/lazy_node_state.rs index 3470473fda..713fd20901 100644 --- a/raphtory/src/db/api/state/lazy_node_state.rs +++ b/raphtory/src/db/api/state/lazy_node_state.rs @@ -3,7 +3,7 @@ use crate::{ db::{ api::{ state::{NodeState, NodeStateOps}, - storage::{nodes::node_storage_ops::NodeStorageOps, storage_ops::GraphStorage}, + storage::graph::{nodes::node_storage_ops::NodeStorageOps, storage_ops::GraphStorage}, view::{internal::NodeList, IntoDynBoxed}, }, graph::node::NodeView, diff --git a/raphtory/src/db/api/storage/edges/edge_entry.rs b/raphtory/src/db/api/storage/graph/edges/edge_entry.rs similarity index 99% rename from raphtory/src/db/api/storage/edges/edge_entry.rs rename to raphtory/src/db/api/storage/graph/edges/edge_entry.rs index ccce6d6ae2..19949f36ca 100644 --- a/raphtory/src/db/api/storage/edges/edge_entry.rs +++ b/raphtory/src/db/api/storage/graph/edges/edge_entry.rs @@ -11,7 +11,7 @@ use crate::{ storage::raw_edges::EdgeRGuard, Prop, }, - db::api::storage::{ + db::api::storage::graph::{ edges::{ edge_ref::EdgeStorageRef, edge_storage_ops::{EdgeStorageOps, TimeIndexRef}, diff --git a/raphtory/src/db/api/storage/edges/edge_owned_entry.rs b/raphtory/src/db/api/storage/graph/edges/edge_owned_entry.rs similarity index 98% rename from raphtory/src/db/api/storage/edges/edge_owned_entry.rs rename to raphtory/src/db/api/storage/graph/edges/edge_owned_entry.rs index 51ab8b8c7e..fdd61040f7 100644 --- a/raphtory/src/db/api/storage/edges/edge_owned_entry.rs +++ b/raphtory/src/db/api/storage/graph/edges/edge_owned_entry.rs @@ -1,20 +1,10 @@ -use std::ops::Range; - -use rayon::iter::ParallelIterator; - -use raphtory_api::core::storage::timeindex::TimeIndexEntry; - -#[cfg(feature = "storage")] -use crate::db::api::storage::variants::storage_variants::StorageVariants; -#[cfg(feature = "storage")] -use crate::disk_graph::storage_interface::edge::DiskOwnedEdge; use crate::{ core::{ entities::{edges::edge_ref::EdgeRef, LayerIds, VID}, storage::raw_edges::EdgeArcGuard, Prop, }, - db::api::storage::{ + db::api::storage::graph::{ edges::{ edge_ref::EdgeStorageRef, edge_storage_ops::{EdgeStorageIntoOps, EdgeStorageOps, TimeIndexRef}, @@ -22,6 +12,14 @@ use crate::{ tprop_storage_ops::TPropOps, }, }; +use raphtory_api::core::storage::timeindex::TimeIndexEntry; +use rayon::iter::ParallelIterator; +use std::ops::Range; + +#[cfg(feature = "storage")] +use crate::db::api::storage::graph::variants::storage_variants::StorageVariants; +#[cfg(feature = "storage")] +use crate::disk_graph::storage_interface::edge::DiskOwnedEdge; #[derive(Debug, Clone)] pub enum EdgeOwnedEntry { diff --git a/raphtory/src/db/api/storage/edges/edge_ref.rs b/raphtory/src/db/api/storage/graph/edges/edge_ref.rs similarity index 97% rename from raphtory/src/db/api/storage/edges/edge_ref.rs rename to raphtory/src/db/api/storage/graph/edges/edge_ref.rs index b5c7ce4b0e..37e4047228 100644 --- a/raphtory/src/db/api/storage/edges/edge_ref.rs +++ b/raphtory/src/db/api/storage/graph/edges/edge_ref.rs @@ -1,22 +1,21 @@ -use std::ops::Range; - -use rayon::prelude::*; - use super::edge_storage_ops::MemEdge; -#[cfg(feature = "storage")] -use crate::db::api::storage::variants::storage_variants::StorageVariants; -#[cfg(feature = "storage")] -use crate::disk_graph::storage_interface::edge::DiskEdge; use crate::{ core::{ entities::{edges::edge_ref::EdgeRef, LayerIds, EID, VID}, Prop, }, - db::api::storage::{ + db::api::storage::graph::{ edges::edge_storage_ops::{EdgeStorageOps, TimeIndexRef}, tprop_storage_ops::TPropOps, }, }; +use rayon::prelude::*; +use std::ops::Range; + +#[cfg(feature = "storage")] +use crate::db::api::storage::graph::variants::storage_variants::StorageVariants; +#[cfg(feature = "storage")] +use crate::disk_graph::storage_interface::edge::DiskEdge; macro_rules! for_all { ($value:expr, $pattern:pat => $result:expr) => { diff --git a/raphtory/src/db/api/storage/edges/edge_storage_ops.rs b/raphtory/src/db/api/storage/graph/edges/edge_storage_ops.rs similarity index 99% rename from raphtory/src/db/api/storage/edges/edge_storage_ops.rs rename to raphtory/src/db/api/storage/graph/edges/edge_storage_ops.rs index 178748ac2b..81e134caad 100644 --- a/raphtory/src/db/api/storage/edges/edge_storage_ops.rs +++ b/raphtory/src/db/api/storage/graph/edges/edge_storage_ops.rs @@ -12,7 +12,7 @@ use crate::{ Prop, }, db::api::{ - storage::{tprop_storage_ops::TPropOps, variants::layer_variants::LayerVariants}, + storage::graph::{tprop_storage_ops::TPropOps, variants::layer_variants::LayerVariants}, view::IntoDynBoxed, }, }; diff --git a/raphtory/src/db/api/storage/edges/edges.rs b/raphtory/src/db/api/storage/graph/edges/edges.rs similarity index 99% rename from raphtory/src/db/api/storage/edges/edges.rs rename to raphtory/src/db/api/storage/graph/edges/edges.rs index ef56ea2778..01d08a4bf9 100644 --- a/raphtory/src/db/api/storage/edges/edges.rs +++ b/raphtory/src/db/api/storage/graph/edges/edges.rs @@ -3,7 +3,7 @@ use super::{edge_entry::EdgeStorageEntry, unlocked::UnlockedEdges}; use crate::disk_graph::storage_interface::{edges::DiskEdges, edges_ref::DiskEdgesRef}; use crate::{ core::{entities::LayerIds, storage::raw_edges::LockedEdges}, - db::api::storage::{ + db::api::storage::graph::{ edges::edge_storage_ops::EdgeStorageOps, variants::storage_variants3::StorageVariants, }, }; diff --git a/raphtory/src/db/api/storage/edges/mod.rs b/raphtory/src/db/api/storage/graph/edges/mod.rs similarity index 100% rename from raphtory/src/db/api/storage/edges/mod.rs rename to raphtory/src/db/api/storage/graph/edges/mod.rs diff --git a/raphtory/src/db/api/storage/edges/unlocked.rs b/raphtory/src/db/api/storage/graph/edges/unlocked.rs similarity index 100% rename from raphtory/src/db/api/storage/edges/unlocked.rs rename to raphtory/src/db/api/storage/graph/edges/unlocked.rs diff --git a/raphtory/src/db/api/storage/locked.rs b/raphtory/src/db/api/storage/graph/locked.rs similarity index 100% rename from raphtory/src/db/api/storage/locked.rs rename to raphtory/src/db/api/storage/graph/locked.rs diff --git a/raphtory/src/db/api/storage/graph/mod.rs b/raphtory/src/db/api/storage/graph/mod.rs new file mode 100644 index 0000000000..7f8f256ae9 --- /dev/null +++ b/raphtory/src/db/api/storage/graph/mod.rs @@ -0,0 +1,6 @@ +pub mod edges; +pub mod locked; +pub mod nodes; +pub mod storage_ops; +pub mod tprop_storage_ops; +pub mod variants; diff --git a/raphtory/src/db/api/storage/nodes/mod.rs b/raphtory/src/db/api/storage/graph/nodes/mod.rs similarity index 100% rename from raphtory/src/db/api/storage/nodes/mod.rs rename to raphtory/src/db/api/storage/graph/nodes/mod.rs diff --git a/raphtory/src/db/api/storage/nodes/node_entry.rs b/raphtory/src/db/api/storage/graph/nodes/node_entry.rs similarity index 99% rename from raphtory/src/db/api/storage/nodes/node_entry.rs rename to raphtory/src/db/api/storage/graph/nodes/node_entry.rs index 4c1401185b..4210dd7161 100644 --- a/raphtory/src/db/api/storage/nodes/node_entry.rs +++ b/raphtory/src/db/api/storage/graph/nodes/node_entry.rs @@ -10,7 +10,7 @@ use crate::{ Direction, }, db::api::{ - storage::{ + storage::graph::{ nodes::{node_ref::NodeStorageRef, node_storage_ops::NodeStorageOps}, tprop_storage_ops::TPropOps, variants::storage_variants3::StorageVariants, diff --git a/raphtory/src/db/api/storage/nodes/node_owned_entry.rs b/raphtory/src/db/api/storage/graph/nodes/node_owned_entry.rs similarity index 93% rename from raphtory/src/db/api/storage/nodes/node_owned_entry.rs rename to raphtory/src/db/api/storage/graph/nodes/node_owned_entry.rs index 70b16b6b27..b223e29217 100644 --- a/raphtory/src/db/api/storage/nodes/node_owned_entry.rs +++ b/raphtory/src/db/api/storage/graph/nodes/node_owned_entry.rs @@ -10,7 +10,7 @@ use crate::{ storage::ArcEntry, Direction, }, - db::api::storage::nodes::node_storage_ops::NodeStorageIntoOps, + db::api::storage::graph::nodes::node_storage_ops::NodeStorageIntoOps, }; pub enum NodeOwnedEntry { diff --git a/raphtory/src/db/api/storage/nodes/node_ref.rs b/raphtory/src/db/api/storage/graph/nodes/node_ref.rs similarity index 94% rename from raphtory/src/db/api/storage/nodes/node_ref.rs rename to raphtory/src/db/api/storage/graph/nodes/node_ref.rs index 4e094a6691..e4cf012ef4 100644 --- a/raphtory/src/db/api/storage/nodes/node_ref.rs +++ b/raphtory/src/db/api/storage/graph/nodes/node_ref.rs @@ -1,23 +1,21 @@ -use std::borrow::Cow; - -use raphtory_api::core::entities::GidRef; - -#[cfg(feature = "storage")] -use crate::db::api::storage::variants::storage_variants::StorageVariants; - -#[cfg(feature = "storage")] -use crate::disk_graph::storage_interface::node::DiskNode; use crate::{ core::{ entities::{edges::edge_ref::EdgeRef, nodes::node_store::NodeStore, LayerIds, VID}, Direction, }, db::api::{ - storage::{nodes::node_storage_ops::NodeStorageOps, tprop_storage_ops::TPropOps}, + storage::graph::{nodes::node_storage_ops::NodeStorageOps, tprop_storage_ops::TPropOps}, view::internal::NodeAdditions, }, prelude::Prop, }; +use raphtory_api::core::entities::GidRef; +use std::borrow::Cow; + +#[cfg(feature = "storage")] +use crate::db::api::storage::graph::variants::storage_variants::StorageVariants; +#[cfg(feature = "storage")] +use crate::disk_graph::storage_interface::node::DiskNode; #[derive(Copy, Clone, Debug)] pub enum NodeStorageRef<'a> { diff --git a/raphtory/src/db/api/storage/nodes/node_storage_ops.rs b/raphtory/src/db/api/storage/graph/nodes/node_storage_ops.rs similarity index 96% rename from raphtory/src/db/api/storage/nodes/node_storage_ops.rs rename to raphtory/src/db/api/storage/graph/nodes/node_storage_ops.rs index b997680135..eca6e956c2 100644 --- a/raphtory/src/db/api/storage/nodes/node_storage_ops.rs +++ b/raphtory/src/db/api/storage/graph/nodes/node_storage_ops.rs @@ -9,7 +9,7 @@ use crate::{ storage::ArcEntry, Direction, }, - db::api::{storage::tprop_storage_ops::TPropOps, view::internal::NodeAdditions}, + db::api::{storage::graph::tprop_storage_ops::TPropOps, view::internal::NodeAdditions}, prelude::Prop, }; use itertools::Itertools; diff --git a/raphtory/src/db/api/storage/nodes/nodes.rs b/raphtory/src/db/api/storage/graph/nodes/nodes.rs similarity index 94% rename from raphtory/src/db/api/storage/nodes/nodes.rs rename to raphtory/src/db/api/storage/graph/nodes/nodes.rs index 9699297401..d545e983a6 100644 --- a/raphtory/src/db/api/storage/nodes/nodes.rs +++ b/raphtory/src/db/api/storage/graph/nodes/nodes.rs @@ -1,15 +1,15 @@ -#[cfg(feature = "storage")] -use crate::disk_graph::storage_interface::nodes::DiskNodesOwned; +use super::node_ref::NodeStorageRef; use crate::{ core::{ entities::{nodes::node_store::NodeStore, VID}, storage::ReadLockedStorage, }, - db::api::storage::nodes::nodes_ref::NodesStorageEntry, + db::api::storage::graph::nodes::nodes_ref::NodesStorageEntry, }; use std::sync::Arc; -use super::node_ref::NodeStorageRef; +#[cfg(feature = "storage")] +use crate::disk_graph::storage_interface::nodes::DiskNodesOwned; pub enum NodesStorage { Mem(Arc>), diff --git a/raphtory/src/db/api/storage/nodes/nodes_ref.rs b/raphtory/src/db/api/storage/graph/nodes/nodes_ref.rs similarity index 96% rename from raphtory/src/db/api/storage/nodes/nodes_ref.rs rename to raphtory/src/db/api/storage/graph/nodes/nodes_ref.rs index 47abbadf3f..2fe0928a85 100644 --- a/raphtory/src/db/api/storage/nodes/nodes_ref.rs +++ b/raphtory/src/db/api/storage/graph/nodes/nodes_ref.rs @@ -1,18 +1,17 @@ -#[cfg(feature = "storage")] -use crate::db::api::storage::variants::storage_variants3::StorageVariants; -#[cfg(feature = "storage")] -use crate::disk_graph::storage_interface::nodes_ref::DiskNodesRef; - -#[cfg(not(feature = "storage"))] -use either::Either; - +use super::node_ref::NodeStorageRef; use crate::core::{ entities::{nodes::node_store::NodeStore, VID}, storage::ReadLockedStorage, }; use rayon::iter::ParallelIterator; -use super::node_ref::NodeStorageRef; +#[cfg(feature = "storage")] +use crate::db::api::storage::graph::variants::storage_variants3::StorageVariants; +#[cfg(feature = "storage")] +use crate::disk_graph::storage_interface::nodes_ref::DiskNodesRef; + +#[cfg(not(feature = "storage"))] +use either::Either; #[derive(Debug)] pub enum NodesStorageEntry<'a> { diff --git a/raphtory/src/db/api/storage/storage_ops/additions.rs b/raphtory/src/db/api/storage/graph/storage_ops/additions.rs similarity index 71% rename from raphtory/src/db/api/storage/storage_ops/additions.rs rename to raphtory/src/db/api/storage/graph/storage_ops/additions.rs index 42c81a6796..4372e6ab85 100644 --- a/raphtory/src/db/api/storage/storage_ops/additions.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/additions.rs @@ -1,10 +1,3 @@ -use either::Either; -use raphtory_api::core::{ - entities::{EID, VID}, - storage::timeindex::TimeIndexEntry, -}; -use std::sync::atomic::Ordering; - use super::GraphStorage; use crate::{ core::{ @@ -18,40 +11,25 @@ use crate::{ db::api::mutation::internal::InternalAdditionOps, prelude::Prop, }; +use either::Either; +use raphtory_api::core::{ + entities::{EID, VID}, + storage::{dict_mapper::MaybeNew, timeindex::TimeIndexEntry}, +}; +use std::sync::atomic::Ordering; impl InternalAdditionOps for TemporalGraph { fn next_event_id(&self) -> Result { Ok(self.event_counter.fetch_add(1, Ordering::Relaxed)) } - fn resolve_layer(&self, layer: Option<&str>) -> Result { + fn resolve_layer(&self, layer: Option<&str>) -> Result, GraphError> { Ok(layer .map(|name| self.edge_meta.get_or_create_layer_id(name)) - .unwrap_or(0)) - } - - fn set_node_type(&self, v_id: VID, node_type: &str) -> Result<(), GraphError> { - let mut node = self.storage.get_node_mut(v_id); - if node_type == "_default" { - return Err(GraphError::NodeTypeError( - "_default type is not allowed to be used on nodes".to_string(), - )); - } - if node.node_type == 0 { - let node_type_id = self.node_meta.get_or_create_node_type_id(node_type); - node.update_node_type(node_type_id); - } else { - let new_node_type_id = self.node_meta.get_node_type_id(node_type).unwrap_or(0); - if node.node_type != new_node_type_id { - return Err(GraphError::NodeTypeError( - "Node already has a non-default type".to_string(), - )); - } - } - Ok(()) + .unwrap_or(MaybeNew::Existing(0))) } - fn resolve_node(&self, n: V) -> Result { + fn resolve_node(&self, n: V) -> Result, GraphError> { match n.as_gid_ref() { Either::Left(id) => { let ref_mut = self.logical_to_physical.get_or_init(id, || { @@ -60,7 +38,38 @@ impl InternalAdditionOps for TemporalGraph { })?; Ok(ref_mut) } - Either::Right(vid) => Ok(vid), + Either::Right(vid) => Ok(MaybeNew::Existing(vid)), + } + } + + fn resolve_node_and_type( + &self, + id: V, + node_type: &str, + ) -> Result, MaybeNew)>, GraphError> { + if node_type == "_default" { + return Err(GraphError::NodeTypeError( + "_default type is not allowed to be used on nodes".to_string(), + )); + } + let vid = self.resolve_node(id)?; + let mut node_store = self.storage.get_node_mut(vid.inner()); + if node_store.node_type == 0 { + let node_type_id = self.node_meta.get_or_create_node_type_id(node_type); + node_store.update_node_type(node_type_id.inner()); + Ok(MaybeNew::New((vid, node_type_id))) + } else { + let node_type_id = self + .node_meta + .get_node_type_id(node_type) + .ok_or_else(|| GraphError::NodeTypeError("Cannot change node type".to_string()))?; + if node_type_id == node_store.node_type { + Ok(MaybeNew::Existing((vid, MaybeNew::Existing(node_type_id)))) + } else { + Err(GraphError::NodeTypeError( + "Cannot change node type".to_string(), + )) + } } } @@ -69,7 +78,7 @@ impl InternalAdditionOps for TemporalGraph { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.graph_meta.resolve_property(prop, dtype, is_static) } @@ -78,7 +87,7 @@ impl InternalAdditionOps for TemporalGraph { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.node_meta.resolve_prop_id(prop, dtype, is_static) } @@ -87,7 +96,7 @@ impl InternalAdditionOps for TemporalGraph { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.edge_meta.resolve_prop_id(prop, dtype, is_static) } @@ -95,7 +104,7 @@ impl InternalAdditionOps for TemporalGraph { &self, t: TimeIndexEntry, v: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.update_time(t); // get the node and update the time index @@ -103,7 +112,7 @@ impl InternalAdditionOps for TemporalGraph { node.update_time(t); for (id, prop) in props { let prop = self.process_prop_value(prop); - node.add_prop(t, id, prop)?; + node.add_prop(t, *id, prop)?; } Ok(()) } @@ -113,16 +122,16 @@ impl InternalAdditionOps for TemporalGraph { t: TimeIndexEntry, src: VID, dst: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, - ) -> Result { + ) -> Result, GraphError> { self.link_nodes(src, dst, t, layer, move |edge| { edge.additions_mut(layer).insert(t); if !props.is_empty() { let edge_layer = edge.layer_mut(layer); for (prop_id, prop) in props { let prop = self.process_prop_value(prop); - edge_layer.add_prop(t, prop_id, prop)?; + edge_layer.add_prop(t, *prop_id, prop)?; } } Ok(()) @@ -133,7 +142,7 @@ impl InternalAdditionOps for TemporalGraph { &self, t: TimeIndexEntry, edge: EID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, ) -> Result<(), GraphError> { self.link_edge(edge, t, layer, |edge| { @@ -142,7 +151,7 @@ impl InternalAdditionOps for TemporalGraph { let edge_layer = edge.layer_mut(layer); for (prop_id, prop) in props { let prop = self.process_prop_value(prop); - edge_layer.add_prop(t, prop_id, prop)?; + edge_layer.add_prop(t, *prop_id, prop)?; } } Ok(()) @@ -158,23 +167,27 @@ impl InternalAdditionOps for GraphStorage { } } - fn resolve_layer(&self, layer: Option<&str>) -> Result { + fn resolve_layer(&self, layer: Option<&str>) -> Result, GraphError> { match self { GraphStorage::Unlocked(storage) => storage.resolve_layer(layer), _ => Err(GraphError::AttemptToMutateImmutableGraph), } } - fn set_node_type(&self, v_id: VID, node_type: &str) -> Result<(), GraphError> { + fn resolve_node(&self, n: V) -> Result, GraphError> { match self { - GraphStorage::Unlocked(storage) => storage.set_node_type(v_id, node_type), + GraphStorage::Unlocked(storage) => storage.resolve_node(n), _ => Err(GraphError::AttemptToMutateImmutableGraph), } } - fn resolve_node(&self, n: V) -> Result { + fn resolve_node_and_type( + &self, + id: V, + node_type: &str, + ) -> Result, MaybeNew)>, GraphError> { match self { - GraphStorage::Unlocked(storage) => storage.resolve_node(n), + GraphStorage::Unlocked(storage) => storage.resolve_node_and_type(id, node_type), _ => Err(GraphError::AttemptToMutateImmutableGraph), } } @@ -184,7 +197,7 @@ impl InternalAdditionOps for GraphStorage { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { match self { GraphStorage::Unlocked(storage) => { storage.resolve_graph_property(prop, dtype, is_static) @@ -198,7 +211,7 @@ impl InternalAdditionOps for GraphStorage { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { match self { GraphStorage::Unlocked(storage) => { storage.resolve_node_property(prop, dtype, is_static) @@ -212,7 +225,7 @@ impl InternalAdditionOps for GraphStorage { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { match self { GraphStorage::Unlocked(storage) => { storage.resolve_edge_property(prop, dtype, is_static) @@ -225,7 +238,7 @@ impl InternalAdditionOps for GraphStorage { &self, t: TimeIndexEntry, v: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => storage.internal_add_node(t, v, props), @@ -238,9 +251,9 @@ impl InternalAdditionOps for GraphStorage { t: TimeIndexEntry, src: VID, dst: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, - ) -> Result { + ) -> Result, GraphError> { match self { GraphStorage::Unlocked(storage) => storage.internal_add_edge(t, src, dst, props, layer), _ => Err(GraphError::AttemptToMutateImmutableGraph), @@ -251,7 +264,7 @@ impl InternalAdditionOps for GraphStorage { &self, t: TimeIndexEntry, edge: EID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, ) -> Result<(), GraphError> { match self { diff --git a/raphtory/src/db/api/storage/storage_ops/const_props.rs b/raphtory/src/db/api/storage/graph/storage_ops/const_props.rs similarity index 100% rename from raphtory/src/db/api/storage/storage_ops/const_props.rs rename to raphtory/src/db/api/storage/graph/storage_ops/const_props.rs diff --git a/raphtory/src/db/api/storage/storage_ops/deletions.rs b/raphtory/src/db/api/storage/graph/storage_ops/deletions.rs similarity index 90% rename from raphtory/src/db/api/storage/storage_ops/deletions.rs rename to raphtory/src/db/api/storage/graph/storage_ops/deletions.rs index f9e4ef7f93..febabaef18 100644 --- a/raphtory/src/db/api/storage/storage_ops/deletions.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/deletions.rs @@ -5,7 +5,7 @@ use crate::{ }; use raphtory_api::core::{ entities::{EID, VID}, - storage::timeindex::TimeIndexEntry, + storage::{dict_mapper::MaybeNew, timeindex::TimeIndexEntry}, }; impl InternalDeletionOps for TemporalGraph { @@ -15,12 +15,11 @@ impl InternalDeletionOps for TemporalGraph { src: VID, dst: VID, layer: usize, - ) -> Result<(), GraphError> { + ) -> Result, GraphError> { self.link_nodes(src, dst, t, layer, |new_edge| { new_edge.deletions_mut(layer).insert(t); Ok(()) - })?; - Ok(()) + }) } fn internal_delete_existing_edge( @@ -44,7 +43,7 @@ impl InternalDeletionOps for GraphStorage { src: VID, dst: VID, layer: usize, - ) -> Result<(), GraphError> { + ) -> Result, GraphError> { match self { GraphStorage::Unlocked(storage) => storage.internal_delete_edge(t, src, dst, layer), _ => Err(GraphError::AttemptToMutateImmutableGraph), diff --git a/raphtory/src/db/api/storage/storage_ops/edge_filter.rs b/raphtory/src/db/api/storage/graph/storage_ops/edge_filter.rs similarity index 81% rename from raphtory/src/db/api/storage/storage_ops/edge_filter.rs rename to raphtory/src/db/api/storage/graph/storage_ops/edge_filter.rs index 4b0221b3b6..2eaf9ab08b 100644 --- a/raphtory/src/db/api/storage/storage_ops/edge_filter.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/edge_filter.rs @@ -1,10 +1,9 @@ +use super::GraphStorage; use crate::{ core::entities::LayerIds, - db::api::{storage::edges::edge_ref::EdgeStorageRef, view::internal::EdgeFilterOps}, + db::api::{storage::graph::edges::edge_ref::EdgeStorageRef, view::internal::EdgeFilterOps}, }; -use super::GraphStorage; - impl EdgeFilterOps for GraphStorage { fn edges_filtered(&self) -> bool { false diff --git a/raphtory/src/db/api/storage/storage_ops/layer_ops.rs b/raphtory/src/db/api/storage/graph/storage_ops/layer_ops.rs similarity index 100% rename from raphtory/src/db/api/storage/storage_ops/layer_ops.rs rename to raphtory/src/db/api/storage/graph/storage_ops/layer_ops.rs diff --git a/raphtory/src/db/api/storage/storage_ops/list_ops.rs b/raphtory/src/db/api/storage/graph/storage_ops/list_ops.rs similarity index 100% rename from raphtory/src/db/api/storage/storage_ops/list_ops.rs rename to raphtory/src/db/api/storage/graph/storage_ops/list_ops.rs diff --git a/raphtory/src/db/api/storage/storage_ops/materialize.rs b/raphtory/src/db/api/storage/graph/storage_ops/materialize.rs similarity index 100% rename from raphtory/src/db/api/storage/storage_ops/materialize.rs rename to raphtory/src/db/api/storage/graph/storage_ops/materialize.rs diff --git a/raphtory/src/db/api/storage/storage_ops/mod.rs b/raphtory/src/db/api/storage/graph/storage_ops/mod.rs similarity index 98% rename from raphtory/src/db/api/storage/storage_ops/mod.rs rename to raphtory/src/db/api/storage/graph/storage_ops/mod.rs index 693e2c6b3a..668686efbe 100644 --- a/raphtory/src/db/api/storage/storage_ops/mod.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/mod.rs @@ -1,5 +1,9 @@ -#[cfg(feature = "storage")] -use crate::disk_graph::{storage_interface::edge::DiskOwnedEdge, DiskGraphStorage}; +use super::{ + edges::{ + edge_entry::EdgeStorageEntry, edge_owned_entry::EdgeOwnedEntry, unlocked::UnlockedEdges, + }, + nodes::node_entry::NodeStorageEntry, +}; use crate::{ core::{ entities::{ @@ -12,7 +16,7 @@ use crate::{ Direction, }, db::api::{ - storage::{ + storage::graph::{ edges::{ edge_ref::EdgeStorageRef, edge_storage_ops::EdgeStorageOps, @@ -39,21 +43,18 @@ use std::{iter, sync::Arc}; #[cfg(feature = "storage")] use crate::{ - db::api::storage::variants::storage_variants::StorageVariants, - disk_graph::storage_interface::{ - edges::DiskEdges, - edges_ref::DiskEdgesRef, - node::{DiskNode, DiskOwnedNode}, - nodes::DiskNodesOwned, - nodes_ref::DiskNodesRef, - }, -}; - -use super::{ - edges::{ - edge_entry::EdgeStorageEntry, edge_owned_entry::EdgeOwnedEntry, unlocked::UnlockedEdges, + db::api::storage::graph::variants::storage_variants::StorageVariants, + disk_graph::{ + storage_interface::{ + edge::DiskOwnedEdge, + edges::DiskEdges, + edges_ref::DiskEdgesRef, + node::{DiskNode, DiskOwnedNode}, + nodes::DiskNodesOwned, + nodes_ref::DiskNodesRef, + }, + DiskGraphStorage, }, - nodes::node_entry::NodeStorageEntry, }; pub mod additions; diff --git a/raphtory/src/db/api/storage/storage_ops/node_filter.rs b/raphtory/src/db/api/storage/graph/storage_ops/node_filter.rs similarity index 79% rename from raphtory/src/db/api/storage/storage_ops/node_filter.rs rename to raphtory/src/db/api/storage/graph/storage_ops/node_filter.rs index 8e9f240b73..c7e7ca3a21 100644 --- a/raphtory/src/db/api/storage/storage_ops/node_filter.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/node_filter.rs @@ -1,10 +1,9 @@ +use super::GraphStorage; use crate::{ core::entities::LayerIds, - db::api::{storage::nodes::node_ref::NodeStorageRef, view::internal::NodeFilterOps}, + db::api::{storage::graph::nodes::node_ref::NodeStorageRef, view::internal::NodeFilterOps}, }; -use super::GraphStorage; - impl NodeFilterOps for GraphStorage { #[inline] fn node_list_trusted(&self) -> bool { diff --git a/raphtory/src/db/api/storage/storage_ops/prop_add.rs b/raphtory/src/db/api/storage/graph/storage_ops/prop_add.rs similarity index 72% rename from raphtory/src/db/api/storage/storage_ops/prop_add.rs rename to raphtory/src/db/api/storage/graph/storage_ops/prop_add.rs index d59cfb52dd..c042e14593 100644 --- a/raphtory/src/db/api/storage/storage_ops/prop_add.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/prop_add.rs @@ -14,33 +14,30 @@ impl InternalPropertyAdditionOps for TemporalGraph { fn internal_add_properties( &self, t: TimeIndexEntry, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { for (prop_id, prop) in props { let prop = self.process_prop_value(prop); - self.graph_meta.add_prop(t, prop_id, prop)?; + self.graph_meta.add_prop(t, *prop_id, prop)?; } Ok(()) } - fn internal_add_constant_properties( - &self, - props: Vec<(usize, Prop)>, - ) -> Result<(), GraphError> { + fn internal_add_constant_properties(&self, props: &[(usize, Prop)]) -> Result<(), GraphError> { for (id, prop) in props { let prop = self.process_prop_value(prop); - self.graph_meta.add_constant_prop(id, prop)?; + self.graph_meta.add_constant_prop(*id, prop)?; } Ok(()) } fn internal_update_constant_properties( &self, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { for (id, prop) in props { let prop = self.process_prop_value(prop); - self.graph_meta.update_constant_prop(id, prop)?; + self.graph_meta.update_constant_prop(*id, prop)?; } Ok(()) } @@ -48,13 +45,13 @@ impl InternalPropertyAdditionOps for TemporalGraph { fn internal_add_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { let mut node = self.storage.get_node_mut(vid); for (prop_id, prop) in props { let prop = self.process_prop_value(prop); - node.add_constant_prop(prop_id, prop).map_err(|err| { - let name = self.node_meta.get_prop_name(prop_id, true); + node.add_constant_prop(*prop_id, prop).map_err(|err| { + let name = self.node_meta.get_prop_name(*prop_id, true); GraphError::ConstantPropertyMutationError { name, new: err.new_value.expect("new value exists"), @@ -70,12 +67,12 @@ impl InternalPropertyAdditionOps for TemporalGraph { fn internal_update_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { let mut node = self.storage.get_node_mut(vid); for (prop_id, prop) in props { let prop = self.process_prop_value(prop); - node.update_constant_prop(prop_id, prop)?; + node.update_constant_prop(*prop_id, prop)?; } Ok(()) } @@ -84,22 +81,24 @@ impl InternalPropertyAdditionOps for TemporalGraph { &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { let mut edge = self.storage.get_edge_mut(eid); let edge_layer = edge.layer_mut(layer); for (prop_id, prop) in props { let prop = self.process_prop_value(prop); - edge_layer.add_constant_prop(prop_id, prop).map_err(|err| { - let name = self.edge_meta.get_prop_name(prop_id, true); - GraphError::ConstantPropertyMutationError { - name, - new: err.new_value.expect("new value exists"), - old: err - .previous_value - .expect("previous value exists if set failed"), - } - })?; + edge_layer + .add_constant_prop(*prop_id, prop) + .map_err(|err| { + let name = self.edge_meta.get_prop_name(*prop_id, true); + GraphError::ConstantPropertyMutationError { + name, + new: err.new_value.expect("new value exists"), + old: err + .previous_value + .expect("previous value exists if set failed"), + } + })?; } Ok(()) } @@ -108,13 +107,13 @@ impl InternalPropertyAdditionOps for TemporalGraph { &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { let mut edge = self.storage.get_edge_mut(eid); let edge_layer = edge.layer_mut(layer); for (prop_id, prop) in props { let prop = self.process_prop_value(prop); - edge_layer.update_constant_prop(prop_id, prop)?; + edge_layer.update_constant_prop(*prop_id, prop)?; } Ok(()) } @@ -124,7 +123,7 @@ impl InternalPropertyAdditionOps for GraphStorage { fn internal_add_properties( &self, t: TimeIndexEntry, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => storage.internal_add_properties(t, props), @@ -132,10 +131,7 @@ impl InternalPropertyAdditionOps for GraphStorage { } } - fn internal_add_constant_properties( - &self, - props: Vec<(usize, Prop)>, - ) -> Result<(), GraphError> { + fn internal_add_constant_properties(&self, props: &[(usize, Prop)]) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => storage.internal_add_constant_properties(props), _ => Err(GraphError::AttemptToMutateImmutableGraph), @@ -144,7 +140,7 @@ impl InternalPropertyAdditionOps for GraphStorage { fn internal_update_constant_properties( &self, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => storage.internal_update_constant_properties(props), @@ -155,7 +151,7 @@ impl InternalPropertyAdditionOps for GraphStorage { fn internal_add_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => { @@ -168,7 +164,7 @@ impl InternalPropertyAdditionOps for GraphStorage { fn internal_update_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => { @@ -182,7 +178,7 @@ impl InternalPropertyAdditionOps for GraphStorage { &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => { @@ -196,16 +192,11 @@ impl InternalPropertyAdditionOps for GraphStorage { &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => { - let mut edge = storage.storage.get_edge_mut(eid); - let edge_layer = edge.layer_mut(layer); - for (prop_id, value) in props { - edge_layer.update_constant_prop(prop_id, value)?; - } - Ok(()) + storage.internal_update_constant_edge_properties(eid, layer, props) } _ => Err(GraphError::AttemptToMutateImmutableGraph), } diff --git a/raphtory/src/db/api/storage/storage_ops/time_props.rs b/raphtory/src/db/api/storage/graph/storage_ops/time_props.rs similarity index 97% rename from raphtory/src/db/api/storage/storage_ops/time_props.rs rename to raphtory/src/db/api/storage/graph/storage_ops/time_props.rs index f62d8bfa23..1a05f74d43 100644 --- a/raphtory/src/db/api/storage/storage_ops/time_props.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/time_props.rs @@ -5,7 +5,7 @@ use raphtory_api::core::storage::{arc_str::ArcStr, timeindex::AsTime}; use crate::{ db::api::{ properties::internal::{TemporalPropertiesOps, TemporalPropertyViewOps}, - storage::tprop_storage_ops::TPropOps, + storage::graph::tprop_storage_ops::TPropOps, }, prelude::Prop, }; diff --git a/raphtory/src/db/api/storage/storage_ops/time_semantics.rs b/raphtory/src/db/api/storage/graph/storage_ops/time_semantics.rs similarity index 99% rename from raphtory/src/db/api/storage/storage_ops/time_semantics.rs rename to raphtory/src/db/api/storage/graph/storage_ops/time_semantics.rs index e9252959cf..00fc9d0032 100644 --- a/raphtory/src/db/api/storage/storage_ops/time_semantics.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/time_semantics.rs @@ -13,7 +13,7 @@ use crate::{ storage::timeindex::{TimeIndexIntoOps, TimeIndexOps}, }, db::api::{ - storage::{ + storage::graph::{ edges::{ edge_ref::EdgeStorageRef, edge_storage_ops::{EdgeStorageIntoOps, EdgeStorageOps}, diff --git a/raphtory/src/db/api/storage/tprop_storage_ops.rs b/raphtory/src/db/api/storage/graph/tprop_storage_ops.rs similarity index 97% rename from raphtory/src/db/api/storage/tprop_storage_ops.rs rename to raphtory/src/db/api/storage/graph/tprop_storage_ops.rs index 3e98686ff2..1082bbf0ba 100644 --- a/raphtory/src/db/api/storage/tprop_storage_ops.rs +++ b/raphtory/src/db/api/storage/graph/tprop_storage_ops.rs @@ -1,6 +1,6 @@ use crate::core::{entities::properties::tprop::TProp, storage::timeindex::AsTime, Prop}; #[cfg(feature = "storage")] -use crate::db::api::storage::variants::storage_variants::StorageVariants; +use crate::db::api::storage::graph::variants::storage_variants::StorageVariants; #[cfg(feature = "storage")] use pometry_storage::tprops::DiskTProp; use raphtory_api::core::storage::timeindex::TimeIndexEntry; diff --git a/raphtory/src/db/api/storage/variants/direction_variants.rs b/raphtory/src/db/api/storage/graph/variants/direction_variants.rs similarity index 100% rename from raphtory/src/db/api/storage/variants/direction_variants.rs rename to raphtory/src/db/api/storage/graph/variants/direction_variants.rs diff --git a/raphtory/src/db/api/storage/variants/filter_variants.rs b/raphtory/src/db/api/storage/graph/variants/filter_variants.rs similarity index 100% rename from raphtory/src/db/api/storage/variants/filter_variants.rs rename to raphtory/src/db/api/storage/graph/variants/filter_variants.rs diff --git a/raphtory/src/db/api/storage/variants/layer_variants.rs b/raphtory/src/db/api/storage/graph/variants/layer_variants.rs similarity index 100% rename from raphtory/src/db/api/storage/variants/layer_variants.rs rename to raphtory/src/db/api/storage/graph/variants/layer_variants.rs diff --git a/raphtory/src/db/api/storage/variants/mod.rs b/raphtory/src/db/api/storage/graph/variants/mod.rs similarity index 100% rename from raphtory/src/db/api/storage/variants/mod.rs rename to raphtory/src/db/api/storage/graph/variants/mod.rs diff --git a/raphtory/src/db/api/storage/variants/storage_variants.rs b/raphtory/src/db/api/storage/graph/variants/storage_variants.rs similarity index 98% rename from raphtory/src/db/api/storage/variants/storage_variants.rs rename to raphtory/src/db/api/storage/graph/variants/storage_variants.rs index dfb41a855b..c63f6713a5 100644 --- a/raphtory/src/db/api/storage/variants/storage_variants.rs +++ b/raphtory/src/db/api/storage/graph/variants/storage_variants.rs @@ -1,4 +1,4 @@ -use crate::{core::Prop, db::api::storage::tprop_storage_ops::TPropOps}; +use crate::{core::Prop, db::api::storage::graph::tprop_storage_ops::TPropOps}; use raphtory_api::core::storage::timeindex::TimeIndexEntry; use rayon::iter::{ plumbing::{Consumer, ProducerCallback, UnindexedConsumer}, diff --git a/raphtory/src/db/api/storage/variants/storage_variants3.rs b/raphtory/src/db/api/storage/graph/variants/storage_variants3.rs similarity index 99% rename from raphtory/src/db/api/storage/variants/storage_variants3.rs rename to raphtory/src/db/api/storage/graph/variants/storage_variants3.rs index b2bfcb92b1..5e706a2f0f 100644 --- a/raphtory/src/db/api/storage/variants/storage_variants3.rs +++ b/raphtory/src/db/api/storage/graph/variants/storage_variants3.rs @@ -1,4 +1,4 @@ -use crate::{core::Prop, db::api::storage::tprop_storage_ops::TPropOps}; +use crate::{core::Prop, db::api::storage::graph::tprop_storage_ops::TPropOps}; use raphtory_api::core::storage::timeindex::TimeIndexEntry; use rayon::iter::{ plumbing::{Consumer, ProducerCallback, UnindexedConsumer}, diff --git a/raphtory/src/db/api/storage/mod.rs b/raphtory/src/db/api/storage/mod.rs index 7f8f256ae9..f05d171044 100644 --- a/raphtory/src/db/api/storage/mod.rs +++ b/raphtory/src/db/api/storage/mod.rs @@ -1,6 +1,2 @@ -pub mod edges; -pub mod locked; -pub mod nodes; -pub mod storage_ops; -pub mod tprop_storage_ops; -pub mod variants; +pub mod graph; +pub mod storage; diff --git a/raphtory/src/db/api/storage/storage.rs b/raphtory/src/db/api/storage/storage.rs new file mode 100644 index 0000000000..8c289f5827 --- /dev/null +++ b/raphtory/src/db/api/storage/storage.rs @@ -0,0 +1,345 @@ +use crate::{ + core::{ + entities::{ + graph::tgraph::TemporalGraph, + nodes::node_ref::{AsNodeRef, NodeRef}, + }, + utils::errors::GraphError, + Prop, PropType, + }, + db::api::{ + mutation::internal::{ + InternalAdditionOps, InternalDeletionOps, InternalPropertyAdditionOps, + }, + storage::graph::{nodes::node_storage_ops::NodeStorageOps, storage_ops::GraphStorage}, + view::{Base, InheritViewOps}, + }, +}; +use once_cell::sync::OnceCell; +use raphtory_api::core::{ + entities::{EID, VID}, + storage::{dict_mapper::MaybeNew, timeindex::TimeIndexEntry}, +}; +use serde::{Deserialize, Serialize}; +use std::{ + fmt::{Display, Formatter}, + sync::Arc, +}; + +#[cfg(feature = "proto")] +use crate::serialise::incremental::GraphWriter; + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct Storage { + graph: GraphStorage, + #[cfg(feature = "proto")] + #[serde(skip)] + pub(crate) cache: OnceCell, +} + +impl Display for Storage { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + Display::fmt(&self.graph, f) + } +} + +impl Base for Storage { + type Base = GraphStorage; + + #[inline] + fn base(&self) -> &Self::Base { + &self.graph + } +} + +impl Storage { + pub(crate) fn new(num_locks: usize) -> Self { + Self { + graph: GraphStorage::Unlocked(Arc::new(TemporalGraph::new(num_locks))), + #[cfg(feature = "proto")] + cache: OnceCell::new(), + } + } + + pub(crate) fn from_inner(graph: GraphStorage) -> Self { + Self { + graph, + #[cfg(feature = "proto")] + cache: OnceCell::new(), + } + } + + #[cfg(feature = "proto")] + #[inline] + fn if_cache(&self, map_fn: impl FnOnce(&GraphWriter)) { + if let Some(cache) = self.cache.get() { + map_fn(cache) + } + } +} +impl InheritViewOps for Storage {} + +impl InternalAdditionOps for Storage { + #[inline] + fn next_event_id(&self) -> Result { + self.graph.next_event_id() + } + + fn resolve_layer(&self, layer: Option<&str>) -> Result, GraphError> { + let id = self.graph.resolve_layer(layer)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.resolve_layer(layer, id)); + + Ok(id) + } + + fn resolve_node(&self, id: V) -> Result, GraphError> { + match id.as_node_ref() { + NodeRef::Internal(id) => Ok(MaybeNew::Existing(id)), + NodeRef::External(gid) => { + let id = self.graph.resolve_node(gid)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.resolve_node(id, gid)); + + Ok(id) + } + } + } + + fn resolve_node_and_type( + &self, + id: V, + node_type: &str, + ) -> Result, MaybeNew)>, GraphError> { + let node_and_type = self.graph.resolve_node_and_type(id, node_type)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| { + let (vid, _) = node_and_type.inner(); + let node_entry = self.graph.node_entry(vid.inner()); + cache.resolve_node_and_type(node_and_type, node_type, node_entry.id()) + }); + + Ok(node_and_type) + } + + fn resolve_graph_property( + &self, + prop: &str, + dtype: PropType, + is_static: bool, + ) -> Result, GraphError> { + let id = self.graph.resolve_graph_property(prop, dtype, is_static)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.resolve_graph_property(prop, id, dtype, is_static)); + + Ok(id) + } + + fn resolve_node_property( + &self, + prop: &str, + dtype: PropType, + is_static: bool, + ) -> Result, GraphError> { + let id = self.graph.resolve_node_property(prop, dtype, is_static)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.resolve_node_property(prop, id, dtype, is_static)); + + Ok(id) + } + + fn resolve_edge_property( + &self, + prop: &str, + dtype: PropType, + is_static: bool, + ) -> Result, GraphError> { + let id = self.graph.resolve_edge_property(prop, dtype, is_static)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.resolve_edge_property(prop, id, dtype, is_static)); + + Ok(id) + } + + fn internal_add_node( + &self, + t: TimeIndexEntry, + v: VID, + props: &[(usize, Prop)], + ) -> Result<(), GraphError> { + self.graph.internal_add_node(t, v, props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_node_update(t, v, props)); + + Ok(()) + } + + fn internal_add_edge( + &self, + t: TimeIndexEntry, + src: VID, + dst: VID, + props: &[(usize, Prop)], + layer: usize, + ) -> Result, GraphError> { + let id = self.graph.internal_add_edge(t, src, dst, props, layer)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| { + cache.resolve_edge(id, src, dst); + cache.add_edge_update(t, id.inner(), props, layer); + }); + + Ok(id) + } + + fn internal_add_edge_update( + &self, + t: TimeIndexEntry, + edge: EID, + props: &[(usize, Prop)], + layer: usize, + ) -> Result<(), GraphError> { + self.graph.internal_add_edge_update(t, edge, props, layer)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_edge_update(t, edge, props, layer)); + + Ok(()) + } +} + +impl InternalPropertyAdditionOps for Storage { + fn internal_add_properties( + &self, + t: TimeIndexEntry, + props: &[(usize, Prop)], + ) -> Result<(), GraphError> { + self.graph.internal_add_properties(t, props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_graph_tprops(t, props)); + + Ok(()) + } + + fn internal_add_constant_properties(&self, props: &[(usize, Prop)]) -> Result<(), GraphError> { + self.graph.internal_add_constant_properties(props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_graph_cprops(props)); + + Ok(()) + } + + fn internal_update_constant_properties( + &self, + props: &[(usize, Prop)], + ) -> Result<(), GraphError> { + self.graph.internal_update_constant_properties(props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_graph_cprops(props)); + + Ok(()) + } + + fn internal_add_constant_node_properties( + &self, + vid: VID, + props: &[(usize, Prop)], + ) -> Result<(), GraphError> { + self.graph + .internal_add_constant_node_properties(vid, props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_node_cprops(vid, props)); + + Ok(()) + } + + fn internal_update_constant_node_properties( + &self, + vid: VID, + props: &[(usize, Prop)], + ) -> Result<(), GraphError> { + self.graph + .internal_update_constant_node_properties(vid, props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_node_cprops(vid, props)); + + Ok(()) + } + + fn internal_add_constant_edge_properties( + &self, + eid: EID, + layer: usize, + props: &[(usize, Prop)], + ) -> Result<(), GraphError> { + self.graph + .internal_add_constant_edge_properties(eid, layer, props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_edge_cprops(eid, layer, props)); + + Ok(()) + } + + fn internal_update_constant_edge_properties( + &self, + eid: EID, + layer: usize, + props: &[(usize, Prop)], + ) -> Result<(), GraphError> { + self.graph + .internal_update_constant_edge_properties(eid, layer, props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_edge_cprops(eid, layer, props)); + + Ok(()) + } +} + +impl InternalDeletionOps for Storage { + fn internal_delete_edge( + &self, + t: TimeIndexEntry, + src: VID, + dst: VID, + layer: usize, + ) -> Result, GraphError> { + let eid = self.graph.internal_delete_edge(t, src, dst, layer)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| { + cache.resolve_edge(eid, src, dst); + cache.delete_edge(eid.inner(), t, layer); + }); + + Ok(eid) + } + + fn internal_delete_existing_edge( + &self, + t: TimeIndexEntry, + eid: EID, + layer: usize, + ) -> Result<(), GraphError> { + self.graph.internal_delete_existing_edge(t, eid, layer)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.delete_edge(eid, t, layer)); + + Ok(()) + } +} diff --git a/raphtory/src/db/api/view/graph.rs b/raphtory/src/db/api/view/graph.rs index 577c143869..a77c3373d7 100644 --- a/raphtory/src/db/api/view/graph.rs +++ b/raphtory/src/db/api/view/graph.rs @@ -8,7 +8,7 @@ use crate::{ api::{ mutation::{internal::InternalAdditionOps, AdditionOps, PropertyAdditionOps}, properties::Properties, - storage::{ + storage::graph::{ edges::edge_storage_ops::EdgeStorageOps, nodes::node_storage_ops::NodeStorageOps, storage_ops::GraphStorage, }, diff --git a/raphtory/src/db/api/view/internal/core_ops.rs b/raphtory/src/db/api/view/internal/core_ops.rs index 0cc13c88fc..c8f7496476 100644 --- a/raphtory/src/db/api/view/internal/core_ops.rs +++ b/raphtory/src/db/api/view/internal/core_ops.rs @@ -13,7 +13,7 @@ use crate::{ Prop, }, db::api::{ - storage::{ + storage::graph::{ edges::{ edge_entry::EdgeStorageEntry, edge_owned_entry::EdgeOwnedEntry, edges::EdgesStorage, }, @@ -27,11 +27,11 @@ use crate::{ }, }; use enum_dispatch::enum_dispatch; +use raphtory_api::core::{entities::GID, storage::arc_str::ArcStr}; use std::{iter, ops::Range}; #[cfg(feature = "storage")] use pometry_storage::timestamps::TimeStamps; -use raphtory_api::core::{entities::GID, storage::arc_str::ArcStr}; #[cfg(feature = "storage")] use rayon::prelude::*; diff --git a/raphtory/src/db/api/view/internal/edge_filter_ops.rs b/raphtory/src/db/api/view/internal/edge_filter_ops.rs index d55750f325..ed7dd36811 100644 --- a/raphtory/src/db/api/view/internal/edge_filter_ops.rs +++ b/raphtory/src/db/api/view/internal/edge_filter_ops.rs @@ -1,6 +1,6 @@ use crate::{ core::entities::LayerIds, - db::api::{storage::edges::edge_ref::EdgeStorageRef, view::internal::Base}, + db::api::{storage::graph::edges::edge_ref::EdgeStorageRef, view::internal::Base}, }; use enum_dispatch::enum_dispatch; diff --git a/raphtory/src/db/api/view/internal/materialize.rs b/raphtory/src/db/api/view/internal/materialize.rs index 0a383f3003..77f98b142d 100644 --- a/raphtory/src/db/api/view/internal/materialize.rs +++ b/raphtory/src/db/api/view/internal/materialize.rs @@ -18,7 +18,7 @@ use crate::{ properties::internal::{ ConstPropertiesOps, TemporalPropertiesOps, TemporalPropertyViewOps, }, - storage::{ + storage::graph::{ edges::{ edge_entry::EdgeStorageEntry, edge_owned_entry::EdgeOwnedEntry, edge_ref::EdgeStorageRef, edges::EdgesStorage, @@ -34,13 +34,11 @@ use crate::{ graph::{graph::Graph, views::deletion_graph::PersistentGraph}, }, prelude::*, - BINCODE_VERSION, }; use chrono::{DateTime, Utc}; use enum_dispatch::enum_dispatch; -use raphtory_api::core::storage::arc_str::ArcStr; -use serde::{de::Error, Deserialize, Deserializer, Serialize}; -use std::{fs, io, path::Path}; +use raphtory_api::core::storage::{arc_str::ArcStr, dict_mapper::MaybeNew}; +use serde::{Deserialize, Serialize}; #[enum_dispatch(CoreGraphOps)] #[enum_dispatch(InternalLayerOps)] @@ -65,27 +63,6 @@ pub enum GraphType { PersistentGraph, } -fn version_deserialize<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let version = u32::deserialize(deserializer)?; - if version != BINCODE_VERSION { - return Err(Error::custom(GraphError::BincodeVersionError( - version, - BINCODE_VERSION, - ))); - }; - Ok(version) -} - -#[derive(Serialize, Deserialize)] -struct VersionedGraph { - #[serde(deserialize_with = "version_deserialize")] - version: u32, - graph: T, -} - impl Static for MaterializedGraph {} impl MaterializedGraph { @@ -108,60 +85,6 @@ impl MaterializedGraph { MaterializedGraph::PersistentGraph(g) => Some(g), } } - - pub fn load_from_file>(path: P, force: bool) -> Result { - let f = std::fs::File::open(path)?; - let mut reader = std::io::BufReader::new(f); - if force { - let _: String = bincode::deserialize_from(&mut reader)?; - let data: Self = bincode::deserialize_from(&mut reader)?; - Ok(data) - } else { - let version: u32 = bincode::deserialize_from(&mut reader)?; - if version != BINCODE_VERSION { - return Err(GraphError::BincodeVersionError(version, BINCODE_VERSION)); - } - let data: Self = bincode::deserialize_from(&mut reader)?; - Ok(data) - } - } - - pub fn save_to_file>(&self, path: P) -> Result<(), GraphError> { - let f = fs::File::create(path)?; - let mut writer = io::BufWriter::new(f); - let versioned_data = VersionedGraph { - version: BINCODE_VERSION, - graph: self.clone(), - }; - Ok(bincode::serialize_into(&mut writer, &versioned_data)?) - } - - pub fn save_to_path(&self, path: &Path) -> Result<(), GraphError> { - match self { - MaterializedGraph::EventGraph(g) => g.save_to_file(&path)?, - MaterializedGraph::PersistentGraph(g) => g.save_to_file(&path)?, - }; - - Ok(()) - } - - pub fn bincode(&self) -> Result, GraphError> { - let versioned_data = VersionedGraph { - version: BINCODE_VERSION, - graph: self.clone(), - }; - let encoded = bincode::serialize(&versioned_data)?; - Ok(encoded) - } - - pub fn from_bincode(b: &[u8]) -> Result { - let version: u32 = bincode::deserialize(b)?; - if version != BINCODE_VERSION { - return Err(GraphError::BincodeVersionError(version, BINCODE_VERSION)); - } - let g: VersionedGraph = bincode::deserialize(b)?; - Ok(g.graph) - } } impl InternalDeletionOps for MaterializedGraph { @@ -171,7 +94,7 @@ impl InternalDeletionOps for MaterializedGraph { src: VID, dst: VID, layer: usize, - ) -> Result<(), GraphError> { + ) -> Result, GraphError> { match self { MaterializedGraph::EventGraph(_) => Err(EventGraphDeletionsNotSupported), MaterializedGraph::PersistentGraph(g) => g.internal_delete_edge(t, src, dst, layer), @@ -191,6 +114,8 @@ impl InternalDeletionOps for MaterializedGraph { } } +impl DeletionOps for MaterializedGraph {} + #[enum_dispatch] pub trait InternalMaterialize { fn new_base_graph(&self, graph: GraphStorage) -> MaterializedGraph { diff --git a/raphtory/src/db/api/view/internal/node_filter_ops.rs b/raphtory/src/db/api/view/internal/node_filter_ops.rs index a6d27346e4..505418ac05 100644 --- a/raphtory/src/db/api/view/internal/node_filter_ops.rs +++ b/raphtory/src/db/api/view/internal/node_filter_ops.rs @@ -1,7 +1,7 @@ use crate::{ core::entities::LayerIds, db::api::{ - storage::nodes::node_ref::NodeStorageRef, + storage::graph::nodes::node_ref::NodeStorageRef, view::{Base, MaterializedGraph}, }, }; diff --git a/raphtory/src/db/api/view/internal/time_semantics.rs b/raphtory/src/db/api/view/internal/time_semantics.rs index 27a5ab9509..be398c5b20 100644 --- a/raphtory/src/db/api/view/internal/time_semantics.rs +++ b/raphtory/src/db/api/view/internal/time_semantics.rs @@ -4,7 +4,7 @@ use crate::{ Prop, }, db::api::{ - storage::{edges::edge_ref::EdgeStorageRef, nodes::node_ref::NodeStorageRef}, + storage::graph::{edges::edge_ref::EdgeStorageRef, nodes::node_ref::NodeStorageRef}, view::{internal::Base, BoxedIter, MaterializedGraph}, }, }; diff --git a/raphtory/src/db/api/view/mod.rs b/raphtory/src/db/api/view/mod.rs index e560d9c99d..8bb2bf5cd2 100644 --- a/raphtory/src/db/api/view/mod.rs +++ b/raphtory/src/db/api/view/mod.rs @@ -6,8 +6,6 @@ pub mod internal; mod layer; pub(crate) mod node; mod reset_filter; -#[cfg(feature = "proto")] -pub mod serialise; pub(crate) mod time; pub(crate) use edge::BaseEdgeViewOps; diff --git a/raphtory/src/db/api/view/node.rs b/raphtory/src/db/api/view/node.rs index 79fa63289b..11e9ba0cf2 100644 --- a/raphtory/src/db/api/view/node.rs +++ b/raphtory/src/db/api/view/node.rs @@ -6,7 +6,7 @@ use crate::{ }, db::api::{ properties::{internal::PropertiesOps, Properties}, - storage::{nodes::node_storage_ops::NodeStorageOps, storage_ops::GraphStorage}, + storage::graph::{nodes::node_storage_ops::NodeStorageOps, storage_ops::GraphStorage}, view::{ internal::{CoreGraphOps, OneHopFilter, TimeSemantics}, reset_filter::ResetFilter, diff --git a/raphtory/src/db/graph/edge.rs b/raphtory/src/db/graph/edge.rs index b39995d07f..150b0df4dd 100644 --- a/raphtory/src/db/graph/edge.rs +++ b/raphtory/src/db/graph/edge.rs @@ -23,7 +23,7 @@ use crate::{ internal::{ConstPropertiesOps, TemporalPropertiesOps, TemporalPropertyViewOps}, Properties, }, - storage::edges::edge_storage_ops::EdgeStorageOps, + storage::graph::edges::edge_storage_ops::EdgeStorageOps, view::{ internal::{OneHopFilter, Static}, BaseEdgeViewOps, IntoDynBoxed, StaticGraphViewOps, @@ -183,7 +183,7 @@ impl }), None => { if create { - self.graph.resolve_layer(layer) + Ok(self.graph.resolve_layer(layer)?.inner()) } else { self.graph .get_layer_id(name) @@ -200,8 +200,6 @@ impl /// Add constant properties for the edge /// - /// Returns a person with the name given them - /// /// # Arguments /// /// * `props` - Property key-value pairs to add @@ -228,13 +226,13 @@ impl }); } let properties: Vec<(usize, Prop)> = props.collect_properties(|name, dtype| { - self.graph.resolve_edge_property(name, dtype, true) + Ok(self.graph.resolve_edge_property(name, dtype, true)?.inner()) })?; self.graph.internal_add_constant_edge_properties( self.edge.pid(), input_layer_id, - properties, + &properties, ) } @@ -245,13 +243,13 @@ impl ) -> Result<(), GraphError> { let input_layer_id = self.resolve_layer(layer, false)?; let properties: Vec<(usize, Prop)> = props.collect_properties(|name, dtype| { - self.graph.resolve_edge_property(name, dtype, true) + Ok(self.graph.resolve_edge_property(name, dtype, true)?.inner()) })?; self.graph.internal_update_constant_edge_properties( self.edge.pid(), input_layer_id, - properties, + &properties, ) } @@ -264,11 +262,14 @@ impl let t = time_from_input(&self.graph, time)?; let layer_id = self.resolve_layer(layer, true)?; let properties: Vec<(usize, Prop)> = props.collect_properties(|name, dtype| { - self.graph.resolve_edge_property(name, dtype, false) + Ok(self + .graph + .resolve_edge_property(name, dtype, false)? + .inner()) })?; self.graph - .internal_add_edge_update(t, self.edge.pid(), properties, layer_id)?; + .internal_add_edge_update(t, self.edge.pid(), &properties, layer_id)?; Ok(()) } } diff --git a/raphtory/src/db/graph/graph.rs b/raphtory/src/db/graph/graph.rs index 06efa25a18..20c7c699db 100644 --- a/raphtory/src/db/graph/graph.rs +++ b/raphtory/src/db/graph/graph.rs @@ -18,11 +18,10 @@ use super::views::deletion_graph::PersistentGraph; use crate::{ - core::{entities::graph::tgraph::TemporalGraph, utils::errors::GraphError}, db::api::{ mutation::internal::InheritMutationOps, - storage::storage_ops::GraphStorage, - view::internal::{Base, InheritViewOps, MaterializedGraph, Static}, + storage::{graph::storage_ops::GraphStorage, storage::Storage}, + view::internal::{Base, InheritViewOps, Static}, }, prelude::*, }; @@ -31,14 +30,13 @@ use rayon::prelude::*; use serde::{Deserialize, Serialize}; use std::{ fmt::{Display, Formatter}, - path::Path, sync::Arc, }; #[repr(transparent)] #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct Graph { - inner: GraphStorage, + pub(crate) inner: Arc, } impl Static for Graph {} @@ -150,7 +148,7 @@ where } impl Base for Graph { - type Base = GraphStorage; + type Base = Storage; #[inline(always)] fn base(&self) -> &Self::Base { @@ -177,7 +175,7 @@ impl Graph { /// ``` pub fn new() -> Self { Self { - inner: GraphStorage::Unlocked(Arc::new(TemporalGraph::default())), + inner: Arc::new(Storage::default()), } } @@ -188,45 +186,22 @@ impl Graph { /// A raphtory graph pub fn new_with_shards(num_shards: usize) -> Self { Self { - inner: GraphStorage::Unlocked(Arc::new(TemporalGraph::new(num_shards))), + inner: Arc::new(Storage::new(num_shards)), } } - pub(crate) fn from_internal_graph(internal_graph: GraphStorage) -> Self { - Self { - inner: internal_graph, - } - } - - /// Load a graph from a directory - /// - /// # Arguments - /// - /// * `path` - The path to the directory - /// - /// Returns: - /// - /// A raphtory graph - /// - /// # Example - /// - /// ```no_run - /// use raphtory::prelude::Graph; - /// let g = Graph::load_from_file("path/to/graph", false); - /// ``` - pub fn load_from_file>(path: P, force: bool) -> Result { - let g = MaterializedGraph::load_from_file(path, force)?; - g.into_events().ok_or(GraphError::GraphLoadError) + pub(crate) fn from_storage(inner: Arc) -> Self { + Self { inner } } - /// Save a graph to a directory - pub fn save_to_file>(&self, path: P) -> Result<(), GraphError> { - MaterializedGraph::from(self.clone()).save_to_file(path) + pub(crate) fn from_internal_graph(graph_storage: GraphStorage) -> Self { + let inner = Arc::new(Storage::from_inner(graph_storage)); + Self { inner } } /// Get persistent graph pub fn persistent_graph(&self) -> PersistentGraph { - PersistentGraph::from_internal_graph(self.inner.clone()) + PersistentGraph::from_storage(self.inner.clone()) } } @@ -236,7 +211,10 @@ mod db_tests { use crate::{ algorithms::components::weakly_connected_components, core::{ - utils::time::{error::ParseTimeError, TryIntoTime}, + utils::{ + errors::GraphError, + time::{error::ParseTimeError, TryIntoTime}, + }, Prop, }, db::{ @@ -551,6 +529,7 @@ mod db_tests { } #[test] + #[cfg(feature = "proto")] fn graph_save_to_load_from_file() { let vs = vec![ (1, 1, 2), @@ -570,10 +549,10 @@ mod db_tests { let tmp_raphtory_path: TempDir = TempDir::new().expect("Failed to create tempdir"); let graph_path = format!("{}/graph.bin", tmp_raphtory_path.path().display()); - g.save_to_file(&graph_path).expect("Failed to save graph"); + g.encode(&graph_path).expect("Failed to save graph"); // Load from files - let g2 = Graph::load_from_file(&graph_path, false).expect("Failed to load graph"); + let g2 = Graph::decode(&graph_path).expect("Failed to load graph"); assert_eq!(g, g2); @@ -2326,12 +2305,15 @@ mod db_tests { } #[test] + #[cfg(feature = "proto")] fn save_load_serial() { let g = Graph::new(); g.add_edge(0, 0, 1, NO_PROPS, None).unwrap(); let dir = tempfile::tempdir().unwrap(); let file_path = dir.path().join("abcd11"); - g.save_to_file(file_path).unwrap(); + g.encode(&file_path).unwrap(); + let gg = Graph::decode(file_path).unwrap(); + assert_graph_equal(&g, &gg); } #[test] diff --git a/raphtory/src/db/graph/node.rs b/raphtory/src/db/graph/node.rs index 423a621092..dc7d9f708c 100644 --- a/raphtory/src/db/graph/node.rs +++ b/raphtory/src/db/graph/node.rs @@ -27,7 +27,7 @@ use crate::{ use crate::{ core::{entities::nodes::node_ref::AsNodeRef, storage::timeindex::AsTime}, - db::{api::storage::storage_ops::GraphStorage, graph::edges::Edges}, + db::{api::storage::graph::storage_ops::GraphStorage, graph::edges::Edges}, }; use chrono::{DateTime, Utc}; use raphtory_api::core::storage::arc_str::ArcStr; @@ -329,14 +329,15 @@ impl props: C, ) -> Result<(), GraphError> { let properties: Vec<(usize, Prop)> = props.collect_properties(|name, dtype| { - self.graph.resolve_node_property(name, dtype, true) + Ok(self.graph.resolve_node_property(name, dtype, true)?.inner()) })?; self.graph - .internal_add_constant_node_properties(self.node, properties) + .internal_add_constant_node_properties(self.node, &properties) } pub fn set_node_type(&self, new_type: &str) -> Result<(), GraphError> { - self.graph.set_node_type(self.node, new_type) + self.graph.resolve_node_and_type(self.node, new_type)?; + Ok(()) } pub fn update_constant_properties( @@ -344,10 +345,10 @@ impl props: C, ) -> Result<(), GraphError> { let properties: Vec<(usize, Prop)> = props.collect_properties(|name, dtype| { - self.graph.resolve_node_property(name, dtype, true) + Ok(self.graph.resolve_node_property(name, dtype, true)?.inner()) })?; self.graph - .internal_update_constant_node_properties(self.node, properties) + .internal_update_constant_node_properties(self.node, &properties) } pub fn add_updates( @@ -357,9 +358,12 @@ impl ) -> Result<(), GraphError> { let t = time_from_input(&self.graph, time)?; let properties: Vec<(usize, Prop)> = props.collect_properties(|name, dtype| { - self.graph.resolve_node_property(name, dtype, false) + Ok(self + .graph + .resolve_node_property(name, dtype, false)? + .inner()) })?; - self.graph.internal_add_node(t, self.node, properties) + self.graph.internal_add_node(t, self.node, &properties) } } diff --git a/raphtory/src/db/graph/nodes.rs b/raphtory/src/db/graph/nodes.rs index 749738113f..cda9a41c2d 100644 --- a/raphtory/src/db/graph/nodes.rs +++ b/raphtory/src/db/graph/nodes.rs @@ -4,7 +4,7 @@ use crate::{ api::{ properties::Properties, state::LazyNodeState, - storage::storage_ops::GraphStorage, + storage::graph::storage_ops::GraphStorage, view::{ internal::{OneHopFilter, Static}, BaseNodeViewOps, BoxedLIter, DynamicGraph, IntoDynBoxed, IntoDynamic, diff --git a/raphtory/src/db/graph/path.rs b/raphtory/src/db/graph/path.rs index ce74e80330..0393f3f0be 100644 --- a/raphtory/src/db/graph/path.rs +++ b/raphtory/src/db/graph/path.rs @@ -3,7 +3,7 @@ use crate::{ db::{ api::{ properties::Properties, - storage::storage_ops::GraphStorage, + storage::graph::storage_ops::GraphStorage, view::{ internal::OneHopFilter, BaseNodeViewOps, BoxedLIter, DynamicGraph, IntoDynBoxed, }, diff --git a/raphtory/src/db/graph/views/deletion_graph.rs b/raphtory/src/db/graph/views/deletion_graph.rs index 6458465bfd..b220f384b2 100644 --- a/raphtory/src/db/graph/views/deletion_graph.rs +++ b/raphtory/src/db/graph/views/deletion_graph.rs @@ -2,7 +2,6 @@ use crate::{ core::{ entities::{edges::edge_ref::EdgeRef, LayerIds, VID}, storage::timeindex::{AsTime, TimeIndexEntry, TimeIndexIntoOps, TimeIndexOps}, - utils::errors::GraphError, Prop, }, db::{ @@ -10,10 +9,13 @@ use crate::{ mutation::internal::InheritMutationOps, properties::internal::InheritPropertiesOps, storage::{ - edges::{edge_ref::EdgeStorageRef, edge_storage_ops::EdgeStorageOps}, - nodes::{node_ref::NodeStorageRef, node_storage_ops::NodeStorageOps}, - storage_ops::GraphStorage, - tprop_storage_ops::TPropOps, + graph::{ + edges::{edge_ref::EdgeStorageRef, edge_storage_ops::EdgeStorageOps}, + nodes::{node_ref::NodeStorageRef, node_storage_ops::NodeStorageOps}, + storage_ops::GraphStorage, + tprop_storage_ops::TPropOps, + }, + storage::Storage, }, view::{internal::*, BoxedIter, IntoDynBoxed}, }, @@ -29,7 +31,7 @@ use std::{ fmt::{Display, Formatter}, iter, ops::Range, - path::Path, + sync::Arc, }; /// A graph view where an edge remains active from the time it is added until it is explicitly marked as deleted. @@ -37,14 +39,14 @@ use std::{ /// Note that the graph will give you access to all edges that were added at any point in time, even those that are marked as deleted. /// The deletion only has an effect on the exploded edge view that are returned. An edge is included in a windowed view of the graph if /// it is considered active at any point in the window. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct PersistentGraph(pub GraphStorage); +#[derive(Clone, Debug, Serialize, Deserialize, Default)] +pub struct PersistentGraph(pub(crate) Arc); impl Static for PersistentGraph {} impl From for PersistentGraph { fn from(value: GraphStorage) -> Self { - Self(value) + Self(Arc::new(Storage::from_inner(value))) } } @@ -107,68 +109,22 @@ fn edge_alive_at_start(e: EdgeStorageRef, t: i64, layer_ids: &LayerIds) -> bool .any(|(_, additions, deletions)| alive_at(&additions, &deletions, t)) } -impl Default for PersistentGraph { - fn default() -> Self { - Self::new() - } -} - impl PersistentGraph { pub fn new() -> Self { - Self(GraphStorage::default()) + Self::default() + } + + pub fn from_storage(storage: Arc) -> Self { + Self(storage) } pub fn from_internal_graph(internal_graph: GraphStorage) -> Self { - Self(internal_graph) - } - - /// Save a graph to a directory - /// - /// # Arguments - /// - /// * `path` - The path to the directory - /// - /// Returns: - /// - /// A raphtory graph - /// - /// # Example - /// - /// ```no_run - /// use std::fs::File; - /// use raphtory::prelude::*; - /// let g = Graph::new(); - /// g.add_node(1, 1, NO_PROPS, None).unwrap(); - /// g.save_to_file("path_str").expect("failed to save file"); - /// ``` - pub fn save_to_file>(&self, path: P) -> Result<(), GraphError> { - MaterializedGraph::from(self.clone()).save_to_file(path) - } - - /// Load a graph from a directory - /// - /// # Arguments - /// - /// * `path` - The path to the directory - /// - /// Returns: - /// - /// A raphtory graph - /// - /// # Example - /// - /// ```no_run - /// use raphtory::prelude::*; - /// let g = Graph::load_from_file("path/to/graph", false); - /// ``` - pub fn load_from_file>(path: P, force: bool) -> Result { - let g = MaterializedGraph::load_from_file(path, force)?; - g.into_persistent().ok_or(GraphError::GraphLoadError) + Self(Arc::new(Storage::from_inner(internal_graph))) } /// Get event graph pub fn event_graph(&self) -> Graph { - Graph::from_internal_graph(self.0.clone()) + Graph::from_storage(self.0.clone()) } } @@ -179,7 +135,7 @@ impl<'graph, G: GraphViewOps<'graph>> PartialEq for PersistentGraph { } impl Base for PersistentGraph { - type Base = GraphStorage; + type Base = Storage; #[inline(always)] fn base(&self) -> &Self::Base { &self.0 diff --git a/raphtory/src/db/graph/views/layer_graph.rs b/raphtory/src/db/graph/views/layer_graph.rs index dc08caefce..572dd95ed3 100644 --- a/raphtory/src/db/graph/views/layer_graph.rs +++ b/raphtory/src/db/graph/views/layer_graph.rs @@ -6,7 +6,7 @@ use crate::{ core::{entities::LayerIds, utils::errors::GraphError}, db::api::{ properties::internal::InheritPropertiesOps, - storage::edges::{edge_ref::EdgeStorageRef, edge_storage_ops::EdgeStorageOps}, + storage::graph::edges::{edge_ref::EdgeStorageRef, edge_storage_ops::EdgeStorageOps}, view::{ internal::{ Base, EdgeFilterOps, Immutable, InheritCoreOps, InheritListOps, InheritMaterialize, diff --git a/raphtory/src/db/graph/views/node_subgraph.rs b/raphtory/src/db/graph/views/node_subgraph.rs index 19c5c07788..e0adb05f70 100644 --- a/raphtory/src/db/graph/views/node_subgraph.rs +++ b/raphtory/src/db/graph/views/node_subgraph.rs @@ -2,7 +2,7 @@ use crate::{ core::entities::{LayerIds, VID}, db::api::{ properties::internal::InheritPropertiesOps, - storage::{ + storage::graph::{ edges::{edge_ref::EdgeStorageRef, edge_storage_ops::EdgeStorageOps}, nodes::{node_ref::NodeStorageRef, node_storage_ops::NodeStorageOps}, }, diff --git a/raphtory/src/db/graph/views/node_type_filtered_subgraph.rs b/raphtory/src/db/graph/views/node_type_filtered_subgraph.rs index 950ea627b4..1e2fe73447 100644 --- a/raphtory/src/db/graph/views/node_type_filtered_subgraph.rs +++ b/raphtory/src/db/graph/views/node_type_filtered_subgraph.rs @@ -2,7 +2,7 @@ use crate::{ core::entities::LayerIds, db::api::{ properties::internal::InheritPropertiesOps, - storage::nodes::{node_ref::NodeStorageRef, node_storage_ops::NodeStorageOps}, + storage::graph::nodes::{node_ref::NodeStorageRef, node_storage_ops::NodeStorageOps}, view::internal::{ Base, Immutable, InheritCoreOps, InheritEdgeFilterOps, InheritLayerOps, InheritListOps, InheritMaterialize, InheritTimeSemantics, NodeFilterOps, Static, diff --git a/raphtory/src/db/graph/views/window_graph.rs b/raphtory/src/db/graph/views/window_graph.rs index 2b863cbf45..2f7f44ea65 100644 --- a/raphtory/src/db/graph/views/window_graph.rs +++ b/raphtory/src/db/graph/views/window_graph.rs @@ -48,7 +48,7 @@ use crate::{ properties::internal::{ InheritStaticPropertiesOps, TemporalPropertiesOps, TemporalPropertyViewOps, }, - storage::{edges::edge_ref::EdgeStorageRef, nodes::node_ref::NodeStorageRef}, + storage::graph::{edges::edge_ref::EdgeStorageRef, nodes::node_ref::NodeStorageRef}, view::{ internal::{ Base, EdgeFilterOps, Immutable, InheritCoreOps, InheritLayerOps, diff --git a/raphtory/src/db/mod.rs b/raphtory/src/db/mod.rs index 22537f5b97..63e711afda 100644 --- a/raphtory/src/db/mod.rs +++ b/raphtory/src/db/mod.rs @@ -1,4 +1,3 @@ pub mod api; pub mod graph; pub mod task; -pub mod utils; diff --git a/raphtory/src/db/task/edge/eval_edge.rs b/raphtory/src/db/task/edge/eval_edge.rs index 29b022aadf..d16a5e9b40 100644 --- a/raphtory/src/db/task/edge/eval_edge.rs +++ b/raphtory/src/db/task/edge/eval_edge.rs @@ -18,7 +18,7 @@ use crate::{ use crate::db::task::edge::eval_edges::EvalEdges; -use crate::db::{api::storage::storage_ops::GraphStorage, task::eval_graph::EvalGraph}; +use crate::db::{api::storage::graph::storage_ops::GraphStorage, task::eval_graph::EvalGraph}; use std::{cell::RefCell, rc::Rc}; pub struct EvalEdgeView<'graph, 'a, G, GH, CS: Clone, S> { diff --git a/raphtory/src/db/task/edge/eval_edges.rs b/raphtory/src/db/task/edge/eval_edges.rs index f2d725303b..abbfe2b868 100644 --- a/raphtory/src/db/task/edge/eval_edges.rs +++ b/raphtory/src/db/task/edge/eval_edges.rs @@ -6,7 +6,7 @@ use crate::{ db::{ api::{ properties::Properties, - storage::storage_ops::GraphStorage, + storage::graph::storage_ops::GraphStorage, view::{internal::OneHopFilter, BaseEdgeViewOps, BoxedLIter}, }, graph::edges::Edges, diff --git a/raphtory/src/db/task/eval_graph.rs b/raphtory/src/db/task/eval_graph.rs index 421d3c9379..18b35de24b 100644 --- a/raphtory/src/db/task/eval_graph.rs +++ b/raphtory/src/db/task/eval_graph.rs @@ -4,7 +4,7 @@ use crate::{ state::compute_state::{ComputeState, ComputeStateVec}, }, db::{ - api::storage::storage_ops::GraphStorage, + api::storage::graph::storage_ops::GraphStorage, task::{ edge::eval_edge::EvalEdgeView, node::{eval_node::EvalNodeView, eval_node_state::EVState}, diff --git a/raphtory/src/db/task/node/eval_node.rs b/raphtory/src/db/task/node/eval_node.rs index c62d5f8da1..5e94799308 100644 --- a/raphtory/src/db/task/node/eval_node.rs +++ b/raphtory/src/db/task/node/eval_node.rs @@ -11,7 +11,7 @@ use crate::{ db::{ api::{ properties::Properties, - storage::storage_ops::GraphStorage, + storage::graph::storage_ops::GraphStorage, view::{internal::OneHopFilter, BaseNodeViewOps, BoxedLIter, IntoDynBoxed}, }, graph::{create_node_type_filter, edges::Edges, node::NodeView, path::PathFromNode}, diff --git a/raphtory/src/db/task/task_runner.rs b/raphtory/src/db/task/task_runner.rs index 38e205475a..ee7db33f0e 100644 --- a/raphtory/src/db/task/task_runner.rs +++ b/raphtory/src/db/task/task_runner.rs @@ -14,7 +14,7 @@ use crate::{ }, }, db::{ - api::{storage::storage_ops::GraphStorage, view::StaticGraphViewOps}, + api::{storage::graph::storage_ops::GraphStorage, view::StaticGraphViewOps}, task::{ eval_graph::EvalGraph, node::{eval_node::EvalNodeView, eval_node_state::EVState}, diff --git a/raphtory/src/db/utils/doc_strings.rs b/raphtory/src/db/utils/doc_strings.rs deleted file mode 100644 index 6d0cbf3210..0000000000 --- a/raphtory/src/db/utils/doc_strings.rs +++ /dev/null @@ -1,49 +0,0 @@ -#[macro_export] -macro_rules! default_layer_doc_string { - () => { - " -Create a view including all the edges in the default layer - -Returns: - a view including all the edges in the default layer" - }; -} - -#[macro_export] -macro_rules! layers_doc_string { - () => { - " -Create a view including all the edges in the layers `names` - -Arguments: - name (str) : the name of the layers to include - -Returns: - a view including all the edges in the layers `names`" - }; -} - -#[macro_export] -macro_rules! layers_name_doc_string { - () => { - " -Create a view including all the edges in the layers `name` - -Arguments: - name (str) : the name of the layer to include - -Returns: - a view including all the edges in the layer `name`" - }; -} - -#[macro_export] -macro_rules! window_size_doc_string { - () => { - " -Returns the size of the window covered by this view - -Returns: - int: the size of the window" - }; -} diff --git a/raphtory/src/db/utils/mod.rs b/raphtory/src/db/utils/mod.rs deleted file mode 100644 index aefeca739a..0000000000 --- a/raphtory/src/db/utils/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod doc_strings; diff --git a/raphtory/src/disk_graph/graph_impl/edge_storage_ops.rs b/raphtory/src/disk_graph/graph_impl/edge_storage_ops.rs index cde9636dc2..88201f3fc1 100644 --- a/raphtory/src/disk_graph/graph_impl/edge_storage_ops.rs +++ b/raphtory/src/disk_graph/graph_impl/edge_storage_ops.rs @@ -4,7 +4,7 @@ use crate::{ storage::timeindex::{TimeIndex, TimeIndexOps}, Prop, }, - db::api::storage::{ + db::api::storage::graph::{ edges::edge_storage_ops::{EdgeStorageOps, TimeIndexRef}, tprop_storage_ops::TPropOps, }, diff --git a/raphtory/src/disk_graph/graph_impl/interop.rs b/raphtory/src/disk_graph/graph_impl/interop.rs index 86be7390a1..19b2761ea5 100644 --- a/raphtory/src/disk_graph/graph_impl/interop.rs +++ b/raphtory/src/disk_graph/graph_impl/interop.rs @@ -5,7 +5,7 @@ use crate::{ Direction, }, db::api::{ - storage::{ + storage::graph::{ edges::edge_storage_ops::EdgeStorageOps, nodes::node_storage_ops::NodeStorageOps, tprop_storage_ops::TPropOps, }, diff --git a/raphtory/src/disk_graph/graph_impl/mod.rs b/raphtory/src/disk_graph/graph_impl/mod.rs index 4db0c7a36c..eb9f80b7ca 100644 --- a/raphtory/src/disk_graph/graph_impl/mod.rs +++ b/raphtory/src/disk_graph/graph_impl/mod.rs @@ -1,9 +1,6 @@ use std::path::Path; -use crate::{ - disk_graph::{DiskGraphError, DiskGraphStorage}, - prelude::Graph, -}; +use crate::{core::utils::errors::GraphError, disk_graph::DiskGraphStorage, prelude::Graph}; mod edge_storage_ops; mod interop; @@ -24,7 +21,7 @@ impl Graph { pub fn persist_as_disk_graph( &self, graph_dir: impl AsRef, - ) -> Result { + ) -> Result { DiskGraphStorage::from_graph(self, graph_dir) } } @@ -44,7 +41,7 @@ mod test { use pometry_storage::{graph::TemporalGraph, properties::Properties}; use crate::{ - db::api::{storage::storage_ops::GraphStorage, view::StaticGraphViewOps}, + db::api::{storage::graph::storage_ops::GraphStorage, view::StaticGraphViewOps}, disk_graph::Time, prelude::*, }; diff --git a/raphtory/src/disk_graph/graph_impl/prop_conversion.rs b/raphtory/src/disk_graph/graph_impl/prop_conversion.rs index 4a19bb9243..b09dc6a8ee 100644 --- a/raphtory/src/disk_graph/graph_impl/prop_conversion.rs +++ b/raphtory/src/disk_graph/graph_impl/prop_conversion.rs @@ -10,7 +10,7 @@ use crate::{ PropType, }, db::api::{ - storage::{nodes::node_storage_ops::NodeStorageOps, tprop_storage_ops::TPropOps}, + storage::graph::{nodes::node_storage_ops::NodeStorageOps, tprop_storage_ops::TPropOps}, view::internal::CoreGraphOps, }, prelude::{Graph, Prop, PropUnwrap}, diff --git a/raphtory/src/disk_graph/graph_impl/tprops.rs b/raphtory/src/disk_graph/graph_impl/tprops.rs index ab9d0dcb4f..e6de5a58e6 100644 --- a/raphtory/src/disk_graph/graph_impl/tprops.rs +++ b/raphtory/src/disk_graph/graph_impl/tprops.rs @@ -4,7 +4,7 @@ use crate::{ types::{NativeType, Offset}, }, core::storage::timeindex::TimeIndexIntoOps, - db::api::{storage::tprop_storage_ops::TPropOps, view::IntoDynBoxed}, + db::api::{storage::graph::tprop_storage_ops::TPropOps, view::IntoDynBoxed}, prelude::Prop, }; use pometry_storage::{ diff --git a/raphtory/src/disk_graph/mod.rs b/raphtory/src/disk_graph/mod.rs index c82698dceb..0504307ead 100644 --- a/raphtory/src/disk_graph/mod.rs +++ b/raphtory/src/disk_graph/mod.rs @@ -12,7 +12,7 @@ use crate::{ }, utils::errors::GraphError, }, - db::{api::storage::storage_ops, graph::views::deletion_graph::PersistentGraph}, + db::{api::storage::graph::storage_ops, graph::views::deletion_graph::PersistentGraph}, disk_graph::graph_impl::{prop_conversion::make_node_properties_from_graph, ParquetLayerCols}, prelude::{Graph, Layer}, }; @@ -37,12 +37,6 @@ pub mod prelude { pub use pometry_storage::chunked_array::array_ops::*; } -#[derive(thiserror::Error, Debug)] -pub enum DiskGraphError { - #[error("Raphtory Arrow Error: {0}")] - RAError(#[from] RAError), -} - #[derive(Clone, Debug)] pub struct DiskGraphStorage { pub(crate) inner: Arc, @@ -214,9 +208,9 @@ impl DiskGraphStorage { chunk_size, t_props_chunk_size, graph_dir.as_ref(), + 2, 0, 1, - 2, ) .expect("failed to create graph") } @@ -227,7 +221,7 @@ impl DiskGraphStorage { &self, other: &DiskGraphStorage, new_graph_dir: impl AsRef, - ) -> Result { + ) -> Result { let graph_dir = new_graph_dir.as_ref(); let inner = merge_graphs(graph_dir, &self.inner, &other.inner)?; Ok(DiskGraphStorage::new(inner)) @@ -256,7 +250,7 @@ impl DiskGraphStorage { let resolved_id = edge_meta .resolve_prop_id(prop_name, data_type.into(), false) .expect("Arrow data types should without failing"); - if id != resolved_id { + if resolved_id != id { println!("Warning: Layers with different edge properties are not supported by the high-level apis on top of the disk_graph graph yet, edge properties will not be available to high-level apis"); edge_meta = Meta::new(); break; @@ -294,7 +288,7 @@ impl DiskGraphStorage { } } - pub fn from_graph(graph: &Graph, graph_dir: impl AsRef) -> Result { + pub fn from_graph(graph: &Graph, graph_dir: impl AsRef) -> Result { let inner_graph = TemporalGraph::from_graph(graph, graph_dir.as_ref(), || { make_node_properties_from_graph(graph, graph_dir.as_ref()) })?; @@ -306,9 +300,9 @@ impl DiskGraphStorage { chunk_size: usize, t_props_chunk_size: usize, graph_dir: impl AsRef + Sync, + time_col_idx: usize, src_col_idx: usize, dst_col_idx: usize, - time_col_idx: usize, ) -> Result { let inner = TemporalGraph::from_sorted_edge_list( graph_dir, diff --git a/raphtory/src/disk_graph/storage_interface/edge.rs b/raphtory/src/disk_graph/storage_interface/edge.rs index 7616864d25..2f99189cf8 100644 --- a/raphtory/src/disk_graph/storage_interface/edge.rs +++ b/raphtory/src/disk_graph/storage_interface/edge.rs @@ -3,7 +3,7 @@ use crate::{ entities::{edges::edge_ref::EdgeRef, LayerIds, EID, ELID}, storage::timeindex::TimeIndexOps, }, - db::api::storage::edges::edge_storage_ops::EdgeStorageIntoOps, + db::api::storage::graph::edges::edge_storage_ops::EdgeStorageIntoOps, }; use pometry_storage::{edge::Edge, edges::Edges, graph::TemporalGraph, timestamps::TimeStamps}; use raphtory_api::core::storage::timeindex::TimeIndexEntry; diff --git a/raphtory/src/disk_graph/storage_interface/edges.rs b/raphtory/src/disk_graph/storage_interface/edges.rs index 443e199bab..771639fc2a 100644 --- a/raphtory/src/disk_graph/storage_interface/edges.rs +++ b/raphtory/src/disk_graph/storage_interface/edges.rs @@ -1,6 +1,6 @@ use crate::{ core::entities::{LayerIds, EID}, - db::api::storage::variants::layer_variants::LayerVariants, + db::api::storage::graph::variants::layer_variants::LayerVariants, disk_graph::storage_interface::{edge::DiskEdge, edges_ref::DiskEdgesRef}, }; use pometry_storage::{graph::TemporalGraph, graph_fragment::TempColGraphFragment}; diff --git a/raphtory/src/disk_graph/storage_interface/edges_ref.rs b/raphtory/src/disk_graph/storage_interface/edges_ref.rs index 631baaaf51..d3fcf4f4ed 100644 --- a/raphtory/src/disk_graph/storage_interface/edges_ref.rs +++ b/raphtory/src/disk_graph/storage_interface/edges_ref.rs @@ -1,6 +1,6 @@ use crate::{ core::entities::{LayerIds, EID}, - db::api::storage::variants::layer_variants::LayerVariants, + db::api::storage::graph::variants::layer_variants::LayerVariants, disk_graph::storage_interface::edge::DiskEdge, }; use pometry_storage::{graph::TemporalGraph, graph_fragment::TempColGraphFragment}; diff --git a/raphtory/src/disk_graph/storage_interface/node.rs b/raphtory/src/disk_graph/storage_interface/node.rs index e739ff8a02..034b9782c6 100644 --- a/raphtory/src/disk_graph/storage_interface/node.rs +++ b/raphtory/src/disk_graph/storage_interface/node.rs @@ -4,7 +4,7 @@ use crate::{ Direction, }, db::api::{ - storage::{ + storage::graph::{ nodes::node_storage_ops::{NodeStorageIntoOps, NodeStorageOps}, tprop_storage_ops::TPropOps, variants::{direction_variants::DirectionVariants, layer_variants::LayerVariants}, diff --git a/raphtory/src/graph_loader/company_house.rs b/raphtory/src/graph_loader/company_house.rs index 6167a5d76c..9231fbae59 100644 --- a/raphtory/src/graph_loader/company_house.rs +++ b/raphtory/src/graph_loader/company_house.rs @@ -28,7 +28,7 @@ pub fn company_house_graph(path: Option) -> Graph { fn restore_from_bincode(encoded_data_dir: &PathBuf) -> Option { if encoded_data_dir.exists() { let now = Instant::now(); - let g = Graph::load_from_file(encoded_data_dir.as_path(), false) + let g = Graph::decode(encoded_data_dir.as_path()) .map_err(|err| { println!( "Restoring from bincode failed with error: {}! Reloading file!", @@ -155,8 +155,7 @@ pub fn company_house_graph(path: Option) -> Graph { now.elapsed().as_secs() ); - g.save_to_file(encoded_data_dir) - .expect("Failed to save graph"); + g.encode(encoded_data_dir).expect("Failed to save graph"); g }); diff --git a/raphtory/src/graph_loader/neo4j_examples.rs b/raphtory/src/graph_loader/neo4j_examples.rs index 112aaef70c..61e08a9de0 100644 --- a/raphtory/src/graph_loader/neo4j_examples.rs +++ b/raphtory/src/graph_loader/neo4j_examples.rs @@ -37,7 +37,7 @@ fn load_movies(row: Row, graph: &rap::Graph) { actor_name, film_title, NO_PROPS, - Some(relation_type.as_str()), + Some(relation_type), ) .unwrap(); } diff --git a/raphtory/src/graph_loader/stable_coins.rs b/raphtory/src/graph_loader/stable_coins.rs index ccb15899bc..1f31ff82c2 100644 --- a/raphtory/src/graph_loader/stable_coins.rs +++ b/raphtory/src/graph_loader/stable_coins.rs @@ -35,10 +35,10 @@ pub fn stable_coin_graph(path: Option, subset: bool) -> Graph { unzip_file(zip_str,dir_str).expect("Failed to unzip stable coin data from https://snap.stanford.edu/data/ERC20-stablecoins.zip"); } - fn restore_from_bincode(encoded_data_dir: &PathBuf) -> Option { - if encoded_data_dir.exists() { + fn restore_from_file(encoded_data_file: &PathBuf) -> Option { + if encoded_data_file.exists() { let now = Instant::now(); - let g = Graph::load_from_file(encoded_data_dir.as_path(), false) + let g = Graph::decode(encoded_data_file.as_path()) .map_err(|err| { println!( "Restoring from bincode failed with error: {}! Reloading file!", @@ -49,7 +49,7 @@ pub fn stable_coin_graph(path: Option, subset: bool) -> Graph { println!( "Loaded graph from encoded data files {} with {} nodes, {} edges which took {} seconds", - encoded_data_dir.to_str().unwrap(), + encoded_data_file.to_str().unwrap(), g.count_nodes(), g.count_edges(), now.elapsed().as_secs() @@ -62,7 +62,7 @@ pub fn stable_coin_graph(path: Option, subset: bool) -> Graph { } let encoded_data_dir = data_dir.join("graphdb.bincode"); - let g = restore_from_bincode(&encoded_data_dir).unwrap_or_else(|| { + let g = restore_from_file(&encoded_data_dir).unwrap_or_else(|| { let g = Graph::new(); let now = Instant::now(); @@ -106,8 +106,7 @@ pub fn stable_coin_graph(path: Option, subset: bool) -> Graph { now.elapsed().as_secs() ); - g.save_to_file(encoded_data_dir) - .expect("Failed to save graph"); + g.encode(encoded_data_dir).expect("Failed to save graph"); g }); diff --git a/raphtory/src/io/arrow/dataframe.rs b/raphtory/src/io/arrow/dataframe.rs index 2b8173f5e1..76b5a6ed0b 100644 --- a/raphtory/src/io/arrow/dataframe.rs +++ b/raphtory/src/io/arrow/dataframe.rs @@ -10,20 +10,16 @@ use polars_arrow::{ use itertools::Itertools; -#[derive(Debug)] -pub(crate) struct DFView { - pub(crate) names: Vec, - pub(crate) arrays: Vec>>, +pub(crate) struct DFView { + pub names: Vec, + pub(crate) chunks: I, + pub num_rows: usize, } -impl DFView { - pub(crate) fn get_inner_size(&self) -> usize { - if self.arrays.is_empty() || self.arrays[0].is_empty() { - return 0; - } - self.arrays[0][0].len() - } - +impl DFView +where + I: Iterator>, +{ pub fn check_cols_exist(&self, cols: &[&str]) -> Result<(), GraphError> { let non_cols: Vec<&&str> = cols .iter() @@ -36,66 +32,58 @@ impl DFView { Ok(()) } + pub(crate) fn get_index(&self, name: &str) -> Result { + self.names + .iter() + .position(|n| n == name) + .ok_or_else(|| GraphError::ColumnDoesNotExist(name.to_string())) + } +} + +#[derive(Clone)] +pub(crate) struct DFChunk { + pub(crate) chunk: Vec>, +} + +impl DFChunk { pub(crate) fn iter_col( &self, - name: &str, + idx: usize, ) -> Option> + '_> { - let idx = self.names.iter().position(|n| n == name)?; - - let _ = (&self.arrays[0])[idx] + let col_arr = (&self.chunk)[idx] .as_any() .downcast_ref::>()?; - - let iter = self.arrays.iter().flat_map(move |arr| { - let arr = &arr[idx]; - let arr = arr.as_any().downcast_ref::>().unwrap(); - arr.iter() - }); - - Some(iter) + Some(col_arr.iter()) } - pub fn utf8(&self, name: &str) -> Option> + '_> { - let idx = self.names.iter().position(|n| n == name)?; + pub fn utf8(&self, idx: usize) -> Option> + '_> { // test that it's actually a utf8 array - let _ = (&self.arrays[0])[idx] - .as_any() - .downcast_ref::>()?; + let col_arr = (&self.chunk)[idx].as_any().downcast_ref::>()?; - let iter = self.arrays.iter().flat_map(move |arr| { - let arr = &arr[idx]; - let arr = arr.as_any().downcast_ref::>().unwrap(); - arr.iter() - }); - - Some(iter) + Some(col_arr.iter()) } - pub fn time_iter_col(&self, name: &str) -> Option> + '_> { - let idx = self.names.iter().position(|n| n == name)?; - - let _ = (&self.arrays[0])[idx] + pub fn time_iter_col(&self, idx: usize) -> Option> + '_> { + let col_arr = (&self.chunk)[idx] .as_any() .downcast_ref::>()?; - let iter = self.arrays.iter().flat_map(move |arr| { - let arr = &arr[idx]; - let arr = if let DataType::Timestamp(_, _) = arr.data_type() { - let array = cast::cast( - &*arr.clone(), - &DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".to_string())), - CastOptions::default(), - ) - .unwrap(); - array - } else { - arr.clone() - }; - - let arr = arr.as_any().downcast_ref::>().unwrap(); - arr.clone().into_iter() - }); - - Some(iter) + let arr = if let DataType::Timestamp(_, _) = col_arr.data_type() { + let array = cast::cast( + col_arr, + &DataType::Timestamp(TimeUnit::Millisecond, Some("UTC".to_string())), + CastOptions::default(), + ) + .unwrap(); + array + .as_any() + .downcast_ref::>() + .unwrap() + .clone() + } else { + col_arr.clone() + }; + + Some(arr.into_iter()) } } diff --git a/raphtory/src/io/arrow/df_loaders.rs b/raphtory/src/io/arrow/df_loaders.rs index b05806af28..a7aeeca5ca 100644 --- a/raphtory/src/io/arrow/df_loaders.rs +++ b/raphtory/src/io/arrow/df_loaders.rs @@ -4,352 +4,407 @@ use crate::{ mutation::{internal::*, AdditionOps}, view::StaticGraphViewOps, }, - io::arrow::{dataframe::DFView, prop_handler::*}, + io::arrow::{ + dataframe::{DFChunk, DFView}, + prop_handler::*, + }, prelude::*, }; -#[cfg(feature = "python")] -use kdam::tqdm; + +use kdam::{Bar, BarBuilder, BarExt}; use std::{collections::HashMap, iter}; -#[cfg(feature = "python")] -macro_rules! maybe_tqdm { - ($iter:expr, $size:expr, $desc:literal) => { - tqdm!( - $iter, - desc = "Loading nodes", - total = $size, - animation = kdam::Animation::FillUp, - unit_scale = true - ) - }; +fn build_progress_bar(des: String, num_rows: usize) -> Result { + BarBuilder::default() + .desc(des) + .animation(kdam::Animation::FillUp) + .total(num_rows) + .unit_scale(true) + .build() + .map_err(|_| GraphError::TqdmError) } - -#[cfg(not(feature = "python"))] -macro_rules! maybe_tqdm { - ($iter:expr, $size:expr, $desc:literal) => { - $iter - }; +fn extract_out_default_type(n_t: Option<&str>) -> Option<&str> { + if n_t == Some("_default") { + None + } else { + n_t + } } pub(crate) fn load_nodes_from_df< 'a, G: StaticGraphViewOps + InternalPropertyAdditionOps + InternalAdditionOps, >( - df: &'a DFView, - size: usize, - node_id: &str, + df_view: DFView>>, time: &str, - properties: Option>, - const_properties: Option>, - shared_const_properties: Option>, + node_id: &str, + properties: Option<&[&str]>, + constant_properties: Option<&[&str]>, + shared_constant_properties: Option<&HashMap>, node_type: Option<&str>, - node_type_in_df: bool, + node_type_col: Option<&str>, graph: &G, ) -> Result<(), GraphError> { - let (prop_iter, const_prop_iter) = get_prop_rows(df, properties, const_properties)?; - - let node_type: Box>> = match node_type { - Some(node_type) => { - if node_type_in_df { - let iter_res: Result>>, GraphError> = - if let Some(node_types) = df.utf8::(node_type) { - Ok(Box::new(node_types)) - } else if let Some(node_types) = df.utf8::(node_type) { - Ok(Box::new(node_types)) - } else { - Err(GraphError::LoadFailure( - "Unable to convert / find node_type column in dataframe.".to_string(), - )) - }; - iter_res? - } else { - Box::new(iter::repeat(Some(node_type))) - } - } - None => Box::new(iter::repeat(None)), + let properties = properties.unwrap_or(&[]); + let constant_properties = constant_properties.unwrap_or(&[]); + + let properties_indices = properties + .iter() + .map(|name| df_view.get_index(name)) + .collect::, GraphError>>()?; + let constant_properties_indices = constant_properties + .iter() + .map(|name| df_view.get_index(name)) + .collect::, GraphError>>()?; + + let node_type_index = if let Some(node_type_col) = node_type_col { + Some(df_view.get_index(node_type_col.as_ref())) + } else { + None }; - - if let (Some(node_id), Some(time)) = (df.iter_col::(node_id), df.time_iter_col(time)) { - let iter = node_id - .map(|i| i.copied()) - .zip(time) - .zip(node_type) - .map(|((node_id, time), n_t)| (node_id, time, n_t)); - load_nodes_from_num_iter( - graph, - size, - iter, - prop_iter, - const_prop_iter, - shared_const_properties, - )?; - } else if let (Some(node_id), Some(time)) = - (df.iter_col::(node_id), df.time_iter_col(time)) - { - let iter = node_id.map(i64_opt_into_u64_opt).zip(time); - let iter = iter - .zip(node_type) - .map(|((node_id, time), n_t)| (node_id, time, n_t)); - - load_nodes_from_num_iter( - graph, - size, - iter, - prop_iter, - const_prop_iter, - shared_const_properties, - )?; - } else if let (Some(node_id), Some(time)) = (df.utf8::(node_id), df.time_iter_col(time)) { - let iter = node_id.into_iter().zip(time); - let iter = iter - .zip(node_type) - .map(|((node_id, time), n_t)| (node_id, time, n_t)); - - let iter = maybe_tqdm!( - iter.zip(prop_iter).zip(const_prop_iter), - size, - "Loading nodes" - ); - - for (((node_id, time, n_t), props), const_props) in iter { - if let (Some(node_id), Some(time), n_t) = (node_id, time, n_t) { - let actual_type = extract_out_default_type(n_t); - let v = graph.add_node(time, node_id, props, actual_type)?; - v.add_constant_properties(const_props)?; - if let Some(shared_const_props) = &shared_const_properties { - v.add_constant_properties(shared_const_props.iter())?; + let node_type_index = node_type_index.transpose()?; + + let node_id_index = df_view.get_index(node_id)?; + let time_index = df_view.get_index(time)?; + let mut pb = build_progress_bar("Loading nodes".to_string(), df_view.num_rows)?; + + for chunk in df_view.chunks { + let df = chunk?; + let prop_iter = combine_properties(properties, &properties_indices, &df)?; + let const_prop_iter = + combine_properties(constant_properties, &constant_properties_indices, &df)?; + + let node_type: Result>>, GraphError> = + match (node_type, node_type_index) { + (None, None) => Ok(Box::new(iter::repeat(None))), + (Some(node_type), None) => Ok(Box::new(iter::repeat(Some(node_type)))), + (None, Some(node_type_index)) => { + let iter_res: Result>>, GraphError> = + if let Some(node_types) = df.utf8::(node_type_index) { + Ok(Box::new(node_types)) + } else if let Some(node_types) = df.utf8::(node_type_index) { + Ok(Box::new(node_types)) + } else { + Err(GraphError::LoadFailure( + "Unable to convert / find node_type column in dataframe." + .to_string(), + )) + }; + iter_res } + _ => Err(GraphError::WrongNumOfArgs( + "node_type".to_string(), + "node_type_col".to_string(), + )), + }; + let node_type = node_type?; + + if let (Some(node_id), Some(time)) = ( + df.iter_col::(node_id_index), + df.time_iter_col(time_index), + ) { + let iter = node_id + .map(|i| i.copied()) + .zip(time) + .zip(node_type) + .map(|((node_id, time), n_t)| (node_id, time, n_t)); + load_nodes_from_num_iter( + graph, + &mut pb, + iter, + prop_iter, + const_prop_iter, + shared_constant_properties, + )?; + } else if let (Some(node_id), Some(time)) = ( + df.iter_col::(node_id_index), + df.time_iter_col(time_index), + ) { + let iter = node_id.map(i64_opt_into_u64_opt).zip(time); + let iter = iter + .zip(node_type) + .map(|((node_id, time), n_t)| (node_id, time, n_t)); + + load_nodes_from_num_iter( + graph, + &mut pb, + iter, + prop_iter, + const_prop_iter, + shared_constant_properties, + )?; + } else if let (Some(node_id), Some(time)) = + (df.utf8::(node_id_index), df.time_iter_col(time_index)) + { + let iter = node_id.into_iter().zip(time); + let iter = iter + .zip(node_type) + .map(|((node_id, time), n_t)| (node_id, time, n_t)); + + for (((node_id, time, n_t), props), const_props) in + iter.zip(prop_iter).zip(const_prop_iter) + { + if let (Some(node_id), Some(time), n_t) = (node_id, time, n_t) { + let actual_type = extract_out_default_type(n_t); + let v = graph.add_node(time, node_id, props, actual_type)?; + v.add_constant_properties(const_props)?; + if let Some(shared_const_props) = &shared_constant_properties { + v.add_constant_properties(shared_const_props.iter())?; + } + } + let _ = pb.update(1); } - } - } else if let (Some(node_id), Some(time)) = (df.utf8::(node_id), df.time_iter_col(time)) { - let iter = node_id.into_iter().zip(time); - let iter = iter - .zip(node_type) - .map(|((node_id, time), n_t)| (node_id, time, n_t)); - - let iter = maybe_tqdm!( - iter.zip(prop_iter).zip(const_prop_iter), - size, - "Loading nodes" - ); - - for (((node_id, time, n_t), props), const_props) in iter { - let actual_type = extract_out_default_type(n_t); - if let (Some(node_id), Some(time), n_t) = (node_id, time, actual_type) { - let v = graph.add_node(time, node_id, props, n_t)?; - v.add_constant_properties(const_props)?; - if let Some(shared_const_props) = &shared_const_properties { - v.add_constant_properties(shared_const_props)?; + } else if let (Some(node_id), Some(time)) = + (df.utf8::(node_id_index), df.time_iter_col(time_index)) + { + let iter = node_id.into_iter().zip(time); + let iter = iter + .zip(node_type) + .map(|((node_id, time), n_t)| (node_id, time, n_t)); + + for (((node_id, time, n_t), props), const_props) in + iter.zip(prop_iter).zip(const_prop_iter) + { + let actual_type = extract_out_default_type(n_t); + if let (Some(node_id), Some(time), n_t) = (node_id, time, actual_type) { + let v = graph.add_node(time, node_id, props, n_t)?; + v.add_constant_properties(const_props)?; + if let Some(shared_const_props) = shared_constant_properties { + v.add_constant_properties(shared_const_props)?; + } } + let _ = pb.update(1); } - } - } else { - return Err(GraphError::LoadFailure( - "node id column must be either u64 or text, time column must be i64. Ensure these contain no NaN, Null or None values.".to_string(), - )); + } else { + return Err(GraphError::LoadFailure( + "node id column must be either u64 or text, time column must be i64. Ensure these contain no NaN, Null or None values.".to_string(), + )); + }; } - Ok(()) } -fn extract_out_default_type(n_t: Option<&str>) -> Option<&str> { - if n_t == Some("_default") { - None - } else { - n_t - } -} - pub(crate) fn load_edges_from_df< 'a, - S: AsRef, G: StaticGraphViewOps + InternalPropertyAdditionOps + InternalAdditionOps, >( - df: &'a DFView, - size: usize, + df_view: DFView>>, + time: &str, src: &str, dst: &str, - time: &str, - properties: Option>, - const_properties: Option>, - shared_const_properties: Option>, - layer: Option, - layer_in_df: bool, + properties: Option<&[&str]>, + constant_properties: Option<&[&str]>, + shared_constant_properties: Option<&HashMap>, + layer: Option<&str>, + layer_col: Option<&str>, graph: &G, ) -> Result<(), GraphError> { - let (prop_iter, const_prop_iter) = get_prop_rows(df, properties, const_properties)?; - let layer = lift_layer(layer, layer_in_df, df); - - if let (Some(src), Some(dst), Some(time)) = ( - df.iter_col::(src), - df.iter_col::(dst), - df.time_iter_col(time), - ) { - let triplets = src - .map(|i| i.copied()) - .zip(dst.map(|i| i.copied())) - .zip(time); - load_edges_from_num_iter( - graph, - size, - triplets, - prop_iter, - const_prop_iter, - shared_const_properties, - layer, - )?; - } else if let (Some(src), Some(dst), Some(time)) = ( - df.iter_col::(src), - df.iter_col::(dst), - df.time_iter_col(time), - ) { - let triplets = src - .map(i64_opt_into_u64_opt) - .zip(dst.map(i64_opt_into_u64_opt)) - .zip(time); - load_edges_from_num_iter( - graph, - size, - triplets, - prop_iter, - const_prop_iter, - shared_const_properties, - layer, - )?; - } else if let (Some(src), Some(dst), Some(time)) = ( - df.utf8::(src), - df.utf8::(dst), - df.time_iter_col(time), - ) { - let triplets = src.into_iter().zip(dst.into_iter()).zip(time.into_iter()); - - let iter = maybe_tqdm!( - triplets.zip(prop_iter).zip(const_prop_iter).zip(layer), - size, - "Loading edges" - ); - - for (((((src, dst), time), props), const_props), layer) in iter { - if let (Some(src), Some(dst), Some(time)) = (src, dst, time) { - let e = graph.add_edge(time, src, dst, props, layer.as_deref())?; - e.add_constant_properties(const_props, layer.as_deref())?; - if let Some(shared_const_props) = &shared_const_properties { - e.add_constant_properties(shared_const_props.iter(), layer.as_deref())?; + let properties = properties.unwrap_or(&[]); + let constant_properties = constant_properties.unwrap_or(&[]); + + let properties_indices = properties + .iter() + .map(|name| df_view.get_index(name)) + .collect::, GraphError>>()?; + let constant_properties_indices = constant_properties + .iter() + .map(|name| df_view.get_index(name)) + .collect::, GraphError>>()?; + + let src_index = df_view.get_index(src)?; + let dst_index = df_view.get_index(dst)?; + let time_index = df_view.get_index(time)?; + let layer_index = if let Some(layer_col) = layer_col { + Some(df_view.get_index(layer_col.as_ref())) + } else { + None + }; + let layer_index = layer_index.transpose()?; + let mut pb = build_progress_bar("Loading edges".to_string(), df_view.num_rows)?; + + for chunk in df_view.chunks { + let df = chunk?; + let prop_iter = combine_properties(properties, &properties_indices, &df)?; + let const_prop_iter = + combine_properties(constant_properties, &constant_properties_indices, &df)?; + + let layer = lift_layer(layer, layer_index, &df)?; + + if let (Some(src), Some(dst), Some(time)) = ( + df.iter_col::(src_index), + df.iter_col::(dst_index), + df.time_iter_col(time_index), + ) { + let triplets = src + .map(|i| i.copied()) + .zip(dst.map(|i| i.copied())) + .zip(time); + load_edges_from_num_iter( + graph, + &mut pb, + triplets, + prop_iter, + const_prop_iter, + shared_constant_properties, + layer, + )?; + } else if let (Some(src), Some(dst), Some(time)) = ( + df.iter_col::(src_index), + df.iter_col::(dst_index), + df.time_iter_col(time_index), + ) { + let triplets = src + .map(i64_opt_into_u64_opt) + .zip(dst.map(i64_opt_into_u64_opt)) + .zip(time); + load_edges_from_num_iter( + graph, + &mut pb, + triplets, + prop_iter, + const_prop_iter, + shared_constant_properties, + layer, + )?; + } else if let (Some(src), Some(dst), Some(time)) = ( + df.utf8::(src_index), + df.utf8::(dst_index), + df.time_iter_col(time_index), + ) { + let triplets = src.into_iter().zip(dst.into_iter()).zip(time.into_iter()); + + for (((((src, dst), time), props), const_props), layer) in + triplets.zip(prop_iter).zip(const_prop_iter).zip(layer) + { + if let (Some(src), Some(dst), Some(time)) = (src, dst, time) { + let e = graph.add_edge(time, src, dst, props, layer.as_deref())?; + e.add_constant_properties(const_props, layer.as_deref())?; + if let Some(shared_const_props) = &shared_constant_properties { + e.add_constant_properties(shared_const_props.iter(), layer.as_deref())?; + } } + let _ = pb.update(1); } - } - } else if let (Some(src), Some(dst), Some(time)) = ( - df.utf8::(src), - df.utf8::(dst), - df.time_iter_col(time), - ) { - let triplets = src.into_iter().zip(dst.into_iter()).zip(time.into_iter()); - let iter = maybe_tqdm!( - triplets.zip(prop_iter).zip(const_prop_iter).zip(layer), - size, - "Loading edges" - ); - - for (((((src, dst), time), props), const_props), layer) in iter { - if let (Some(src), Some(dst), Some(time)) = (src, dst, time) { - let e = graph.add_edge(time, src, dst, props, layer.as_deref())?; - e.add_constant_properties(const_props, layer.as_deref())?; - if let Some(shared_const_props) = &shared_const_properties { - e.add_constant_properties(shared_const_props.iter(), layer.as_deref())?; + } else if let (Some(src), Some(dst), Some(time)) = ( + df.utf8::(src_index), + df.utf8::(dst_index), + df.time_iter_col(time_index), + ) { + let triplets = src.into_iter().zip(dst.into_iter()).zip(time.into_iter()); + for (((((src, dst), time), props), const_props), layer) in + triplets.zip(prop_iter).zip(const_prop_iter).zip(layer) + { + if let (Some(src), Some(dst), Some(time)) = (src, dst, time) { + let e = graph.add_edge(time, src, dst, props, layer.as_deref())?; + e.add_constant_properties(const_props, layer.as_deref())?; + if let Some(shared_const_props) = &shared_constant_properties { + e.add_constant_properties(shared_const_props.iter(), layer.as_deref())?; + } } + let _ = pb.update(1); } - } - } else { - return Err(GraphError::LoadFailure( - "Source and Target columns must be either u64 or text, Time column must be i64. Ensure these contain no NaN, Null or None values." - .to_string(), - )); + } else { + return Err(GraphError::LoadFailure( + "Source and Target columns must be either u64 or text, Time column must be i64. Ensure these contain no NaN, Null or None values." + .to_string(), + )); + }; } Ok(()) } -pub(crate) fn load_edges_deletions_from_df< +pub(crate) fn load_edge_deletions_from_df< 'a, - S: AsRef, G: StaticGraphViewOps + InternalPropertyAdditionOps + InternalAdditionOps + DeletionOps, >( - df: &'a DFView, - size: usize, + df_view: DFView>>, + time: &str, src: &str, dst: &str, - time: &str, - layer: Option, - layer_in_df: bool, + layer: Option<&str>, + layer_col: Option<&str>, graph: &G, ) -> Result<(), GraphError> { - let layer = lift_layer(layer, layer_in_df, df); - - if let (Some(src), Some(dst), Some(time)) = ( - df.iter_col::(src), - df.iter_col::(dst), - df.time_iter_col(time), - ) { - let triplets = src - .map(|i| i.copied()) - .zip(dst.map(|i| i.copied())) - .zip(time); - - let iter = maybe_tqdm!(triplets.zip(layer), size, "Loading edges"); - - for (((src, dst), time), layer) in iter { - if let (Some(src), Some(dst), Some(time)) = (src, dst, time) { - graph.delete_edge(time, src, dst, layer.as_deref())?; + let src_index = df_view.get_index(src)?; + let dst_index = df_view.get_index(dst)?; + let time_index = df_view.get_index(time)?; + let layer_index = if let Some(layer_col) = layer_col { + Some(df_view.get_index(layer_col.as_ref())) + } else { + None + }; + let layer_index = layer_index.transpose()?; + let mut pb = build_progress_bar("Loading edge deletions".to_string(), df_view.num_rows)?; + + for chunk in df_view.chunks { + let df = chunk?; + let layer = lift_layer(layer, layer_index, &df)?; + + if let (Some(src), Some(dst), Some(time)) = ( + df.iter_col::(src_index), + df.iter_col::(dst_index), + df.time_iter_col(time_index), + ) { + let triplets = src + .map(|i| i.copied()) + .zip(dst.map(|i| i.copied())) + .zip(time); + + for (((src, dst), time), layer) in triplets.zip(layer) { + if let (Some(src), Some(dst), Some(time)) = (src, dst, time) { + graph.delete_edge(time, src, dst, layer.as_deref())?; + } + let _ = pb.update(1); } - } - } else if let (Some(src), Some(dst), Some(time)) = ( - df.iter_col::(src), - df.iter_col::(dst), - df.time_iter_col(time), - ) { - let triplets = src - .map(i64_opt_into_u64_opt) - .zip(dst.map(i64_opt_into_u64_opt)) - .zip(time); - - let iter = maybe_tqdm!(triplets.zip(layer), size, "Loading edges"); - - for (((src, dst), time), layer) in iter { - if let (Some(src), Some(dst), Some(time)) = (src, dst, time) { - graph.delete_edge(time, src, dst, layer.as_deref())?; + } else if let (Some(src), Some(dst), Some(time)) = ( + df.iter_col::(src_index), + df.iter_col::(dst_index), + df.time_iter_col(time_index), + ) { + let triplets = src + .map(i64_opt_into_u64_opt) + .zip(dst.map(i64_opt_into_u64_opt)) + .zip(time); + + for (((src, dst), time), layer) in triplets.zip(layer) { + if let (Some(src), Some(dst), Some(time)) = (src, dst, time) { + graph.delete_edge(time, src, dst, layer.as_deref())?; + } + let _ = pb.update(1); } - } - } else if let (Some(src), Some(dst), Some(time)) = ( - df.utf8::(src), - df.utf8::(dst), - df.time_iter_col(time), - ) { - let triplets = src.into_iter().zip(dst.into_iter()).zip(time.into_iter()); - let iter = maybe_tqdm!(triplets.zip(layer), size, "Loading edges"); - - for (((src, dst), time), layer) in iter { - if let (Some(src), Some(dst), Some(time)) = (src, dst, time) { - graph.delete_edge(time, src, dst, layer.as_deref())?; + } else if let (Some(src), Some(dst), Some(time)) = ( + df.utf8::(src_index), + df.utf8::(dst_index), + df.time_iter_col(time_index), + ) { + let triplets = src.into_iter().zip(dst.into_iter()).zip(time.into_iter()); + for (((src, dst), time), layer) in triplets.zip(layer) { + if let (Some(src), Some(dst), Some(time)) = (src, dst, time) { + graph.delete_edge(time, src, dst, layer.as_deref())?; + } + let _ = pb.update(1); } - } - } else if let (Some(src), Some(dst), Some(time)) = ( - df.utf8::(src), - df.utf8::(dst), - df.time_iter_col(time), - ) { - let triplets = src.into_iter().zip(dst.into_iter()).zip(time.into_iter()); - let iter = maybe_tqdm!(triplets.zip(layer), size, "Loading edges"); - - for (((src, dst), time), layer) in iter { - if let (Some(src), Some(dst), Some(time)) = (src, dst, time) { - graph.delete_edge(time, src, dst, layer.as_deref())?; + } else if let (Some(src), Some(dst), Some(time)) = ( + df.utf8::(src_index), + df.utf8::(dst_index), + df.time_iter_col(time_index), + ) { + let triplets = src.into_iter().zip(dst.into_iter()).zip(time.into_iter()); + + for (((src, dst), time), layer) in triplets.zip(layer) { + if let (Some(src), Some(dst), Some(time)) = (src, dst, time) { + graph.delete_edge(time, src, dst, layer.as_deref())?; + } + let _ = pb.update(1); } - } - } else { - return Err(GraphError::LoadFailure( - "Source and Target columns must be either u64 or text, Time column must be i64. Ensure these contain no NaN, Null or None values." - .to_string(), - )); + } else { + return Err(GraphError::LoadFailure( + "Source and Target columns must be either u64 or text, Time column must be i64. Ensure these contain no NaN, Null or None values." + .to_string(), + )); + }; } + Ok(()) } @@ -357,190 +412,250 @@ pub(crate) fn load_node_props_from_df< 'a, G: StaticGraphViewOps + InternalPropertyAdditionOps + InternalAdditionOps, >( - df: &'a DFView, - size: usize, + df_view: DFView>>, node_id: &str, - const_properties: Option>, - shared_const_properties: Option>, + node_type: Option<&str>, + node_type_col: Option<&str>, + constant_properties: Option<&[&str]>, + shared_constant_properties: Option<&HashMap>, graph: &G, ) -> Result<(), GraphError> { - let (_, const_prop_iter) = get_prop_rows(df, None, const_properties)?; - - if let Some(node_id) = df.iter_col::(node_id) { - let iter = node_id.map(|i| i.copied()); - let iter = maybe_tqdm!(iter.zip(const_prop_iter), size, "Loading node properties"); - - for (node_id, const_props) in iter { - if let Some(node_id) = node_id { - let v = graph - .node(node_id) - .ok_or(GraphError::NodeIdError(node_id))?; - v.add_constant_properties(const_props)?; - if let Some(shared_const_props) = &shared_const_properties { - v.add_constant_properties(shared_const_props.iter())?; + let constant_properties = constant_properties.unwrap_or(&[]); + let constant_properties_indices = constant_properties + .iter() + .map(|name| df_view.get_index(name)) + .collect::, GraphError>>()?; + let node_id_index = df_view.get_index(node_id)?; + let node_type_index = if let Some(node_type_col) = node_type_col { + Some(df_view.get_index(node_type_col.as_ref())) + } else { + None + }; + let node_type_index = node_type_index.transpose()?; + let mut pb = build_progress_bar("Loading node properties".to_string(), df_view.num_rows)?; + for chunk in df_view.chunks { + let df = chunk?; + let const_prop_iter = + combine_properties(constant_properties, &constant_properties_indices, &df)?; + + let node_type: Result>>, GraphError> = + match (node_type, node_type_index) { + (None, None) => Ok(Box::new(iter::repeat(None))), + (Some(node_type), None) => Ok(Box::new(iter::repeat(Some(node_type)))), + (None, Some(node_type_index)) => { + let iter_res: Result>>, GraphError> = + if let Some(node_types) = df.utf8::(node_type_index) { + Ok(Box::new(node_types)) + } else if let Some(node_types) = df.utf8::(node_type_index) { + Ok(Box::new(node_types)) + } else { + Err(GraphError::LoadFailure( + "Unable to convert / find node_type column in dataframe." + .to_string(), + )) + }; + iter_res } + _ => Err(GraphError::WrongNumOfArgs( + "node_type".to_string(), + "node_type_col".to_string(), + )), + }; + let node_type = node_type?; + + if let Some(node_id) = df.iter_col::(node_id_index) { + let iter = node_id.map(|i| i.copied()); + for ((node_id, const_props), node_type) in iter.zip(const_prop_iter).zip(node_type) { + if let Some(node_id) = node_id { + let v = graph + .node(node_id) + .ok_or(GraphError::NodeIdError(node_id))?; + v.add_constant_properties(const_props)?; + if let Some(shared_const_props) = &shared_constant_properties { + v.add_constant_properties(shared_const_props.iter())?; + } + if let Some(node_type) = node_type { + v.set_node_type(node_type)?; + } + } + let _ = pb.update(1); } - } - } else if let Some(node_id) = df.iter_col::(node_id) { - let iter = node_id.map(i64_opt_into_u64_opt); - let iter = maybe_tqdm!(iter.zip(const_prop_iter), size, "Loading node properties"); - - for (node_id, const_props) in iter { - if let Some(node_id) = node_id { - let v = graph - .node(node_id) - .ok_or(GraphError::NodeIdError(node_id))?; - v.add_constant_properties(const_props)?; - if let Some(shared_const_props) = &shared_const_properties { - v.add_constant_properties(shared_const_props.iter())?; + } else if let Some(node_id) = df.iter_col::(node_id_index) { + let iter = node_id.map(i64_opt_into_u64_opt); + for ((node_id, const_props), node_type) in iter.zip(const_prop_iter).zip(node_type) { + if let Some(node_id) = node_id { + let v = graph + .node(node_id) + .ok_or(GraphError::NodeIdError(node_id))?; + v.add_constant_properties(const_props)?; + if let Some(shared_const_props) = &shared_constant_properties { + v.add_constant_properties(shared_const_props.iter())?; + } + if let Some(node_type) = node_type { + v.set_node_type(node_type)?; + } } + let _ = pb.update(1); } - } - } else if let Some(node_id) = df.utf8::(node_id) { - let iter = node_id.into_iter(); - let iter = maybe_tqdm!(iter.zip(const_prop_iter), size, "Loading node properties"); - - for (node_id, const_props) in iter { - if let Some(node_id) = node_id { - let v = graph - .node(node_id) - .ok_or_else(|| GraphError::NodeNameError(node_id.to_owned()))?; - v.add_constant_properties(const_props)?; - if let Some(shared_const_props) = &shared_const_properties { - v.add_constant_properties(shared_const_props.iter())?; + } else if let Some(node_id) = df.utf8::(node_id_index) { + let iter = node_id.into_iter(); + for ((node_id, const_props), node_type) in iter.zip(const_prop_iter).zip(node_type) { + if let Some(node_id) = node_id { + let v = graph + .node(node_id) + .ok_or_else(|| GraphError::NodeNameError(node_id.to_owned()))?; + v.add_constant_properties(const_props)?; + if let Some(shared_const_props) = &shared_constant_properties { + v.add_constant_properties(shared_const_props.iter())?; + } + if let Some(node_type) = node_type { + v.set_node_type(node_type)?; + } } + let _ = pb.update(1); } - } - } else if let Some(node_id) = df.utf8::(node_id) { - let iter = node_id.into_iter(); - let iter = maybe_tqdm!(iter.zip(const_prop_iter), size, "Loading node properties"); - - for (node_id, const_props) in iter { - if let Some(node_id) = node_id { - let v = graph - .node(node_id) - .ok_or_else(|| GraphError::NodeNameError(node_id.to_owned()))?; - v.add_constant_properties(const_props)?; - if let Some(shared_const_props) = &shared_const_properties { - v.add_constant_properties(shared_const_props.iter())?; + } else if let Some(node_id) = df.utf8::(node_id_index) { + let iter = node_id.into_iter(); + for ((node_id, const_props), node_type) in iter.zip(const_prop_iter).zip(node_type) { + if let Some(node_id) = node_id { + let v = graph + .node(node_id) + .ok_or_else(|| GraphError::NodeNameError(node_id.to_owned()))?; + v.add_constant_properties(const_props)?; + if let Some(shared_const_props) = &shared_constant_properties { + v.add_constant_properties(shared_const_props.iter())?; + } + if let Some(node_type) = node_type { + v.set_node_type(node_type)?; + } } + let _ = pb.update(1); } - } - } else { - return Err(GraphError::LoadFailure( - "node id column must be either u64 or text, time column must be i64. Ensure these contain no NaN, Null or None values.".to_string(), - )); + } else { + return Err(GraphError::LoadFailure( + "node id column must be either u64 or text, time column must be i64. Ensure these contain no NaN, Null or None values.".to_string(), + )); + }; } Ok(()) } pub(crate) fn load_edges_props_from_df< 'a, - S: AsRef, G: StaticGraphViewOps + InternalPropertyAdditionOps + InternalAdditionOps, >( - df: &'a DFView, - size: usize, + df_view: DFView>>, src: &str, dst: &str, - const_properties: Option>, - shared_const_properties: Option>, - layer: Option, - layer_in_df: bool, + constant_properties: Option<&[&str]>, + shared_constant_properties: Option<&HashMap>, + layer: Option<&str>, + layer_col: Option<&str>, graph: &G, ) -> Result<(), GraphError> { - let (_, const_prop_iter) = get_prop_rows(df, None, const_properties)?; - let layer = lift_layer(layer, layer_in_df, df); - - if let (Some(src), Some(dst)) = (df.iter_col::(src), df.iter_col::(dst)) { - let triplets = src.map(|i| i.copied()).zip(dst.map(|i| i.copied())); - let iter = maybe_tqdm!( - triplets.zip(const_prop_iter).zip(layer), - size, - "Loading edge properties" - ); - - for (((src, dst), const_props), layer) in iter { - if let (Some(src), Some(dst)) = (src, dst) { - let e = graph - .edge(src, dst) - .ok_or(GraphError::EdgeIdError { src, dst })?; - e.add_constant_properties(const_props, layer.as_deref())?; - if let Some(shared_const_props) = &shared_const_properties { - e.add_constant_properties(shared_const_props.iter(), layer.as_deref())?; + let constant_properties = constant_properties.unwrap_or(&[]); + let constant_properties_indices = constant_properties + .iter() + .map(|name| df_view.get_index(name)) + .collect::, GraphError>>()?; + let src_index = df_view.get_index(src)?; + let dst_index = df_view.get_index(dst)?; + let layer_index = if let Some(layer_col) = layer_col { + Some(df_view.get_index(layer_col.as_ref())) + } else { + None + }; + let layer_index = layer_index.transpose()?; + let mut pb = build_progress_bar("Loading edge properties".to_string(), df_view.num_rows)?; + + for chunk in df_view.chunks { + let df = chunk?; + let const_prop_iter = + combine_properties(constant_properties, &constant_properties_indices, &df)?; + + let layer = lift_layer(layer, layer_index, &df)?; + + if let (Some(src), Some(dst)) = + (df.iter_col::(src_index), df.iter_col::(dst_index)) + { + let triplets = src.map(|i| i.copied()).zip(dst.map(|i| i.copied())); + + for (((src, dst), const_props), layer) in triplets.zip(const_prop_iter).zip(layer) { + if let (Some(src), Some(dst)) = (src, dst) { + let e = graph + .edge(src, dst) + .ok_or(GraphError::EdgeIdError { src, dst })?; + e.add_constant_properties(const_props, layer.as_deref())?; + if let Some(shared_const_props) = &shared_constant_properties { + e.add_constant_properties(shared_const_props.iter(), layer.as_deref())?; + } } + let _ = pb.update(1); } - } - } else if let (Some(src), Some(dst)) = (df.iter_col::(src), df.iter_col::(dst)) { - let triplets = src - .map(i64_opt_into_u64_opt) - .zip(dst.map(i64_opt_into_u64_opt)); - let iter = maybe_tqdm!( - triplets.zip(const_prop_iter).zip(layer), - size, - "Loading edge properties" - ); - - for (((src, dst), const_props), layer) in iter { - if let (Some(src), Some(dst)) = (src, dst) { - let e = graph - .edge(src, dst) - .ok_or(GraphError::EdgeIdError { src, dst })?; - e.add_constant_properties(const_props, layer.as_deref())?; - if let Some(shared_const_props) = &shared_const_properties { - e.add_constant_properties(shared_const_props.iter(), layer.as_deref())?; + } else if let (Some(src), Some(dst)) = + (df.iter_col::(src_index), df.iter_col::(dst_index)) + { + let triplets = src + .map(i64_opt_into_u64_opt) + .zip(dst.map(i64_opt_into_u64_opt)); + + for (((src, dst), const_props), layer) in triplets.zip(const_prop_iter).zip(layer) { + if let (Some(src), Some(dst)) = (src, dst) { + let e = graph + .edge(src, dst) + .ok_or(GraphError::EdgeIdError { src, dst })?; + e.add_constant_properties(const_props, layer.as_deref())?; + if let Some(shared_const_props) = &shared_constant_properties { + e.add_constant_properties(shared_const_props.iter(), layer.as_deref())?; + } } + let _ = pb.update(1); } - } - } else if let (Some(src), Some(dst)) = (df.utf8::(src), df.utf8::(dst)) { - let triplets = src.into_iter().zip(dst.into_iter()); - let iter = maybe_tqdm!( - triplets.zip(const_prop_iter).zip(layer), - size, - "Loading edge properties" - ); - - for (((src, dst), const_props), layer) in iter { - if let (Some(src), Some(dst)) = (src, dst) { - let e = graph - .edge(src, dst) - .ok_or_else(|| GraphError::EdgeNameError { - src: src.to_owned(), - dst: dst.to_owned(), - })?; - e.add_constant_properties(const_props, layer.as_deref())?; - if let Some(shared_const_props) = &shared_const_properties { - e.add_constant_properties(shared_const_props.iter(), layer.as_deref())?; + } else if let (Some(src), Some(dst)) = + (df.utf8::(src_index), df.utf8::(dst_index)) + { + let triplets = src.into_iter().zip(dst.into_iter()); + for (((src, dst), const_props), layer) in triplets.zip(const_prop_iter).zip(layer) { + if let (Some(src), Some(dst)) = (src, dst) { + let e = graph + .edge(src, dst) + .ok_or_else(|| GraphError::EdgeNameError { + src: src.to_owned(), + dst: dst.to_owned(), + })?; + e.add_constant_properties(const_props, layer.as_deref())?; + if let Some(shared_const_props) = &shared_constant_properties { + e.add_constant_properties(shared_const_props.iter(), layer.as_deref())?; + } } + let _ = pb.update(1); } - } - } else if let (Some(src), Some(dst)) = (df.utf8::(src), df.utf8::(dst)) { - let triplets = src.into_iter().zip(dst.into_iter()); - let iter = maybe_tqdm!( - triplets.zip(const_prop_iter).zip(layer), - size, - "Loading edge properties" - ); - - for (((src, dst), const_props), layer) in iter { - if let (Some(src), Some(dst)) = (src, dst) { - let e = graph - .edge(src, dst) - .ok_or_else(|| GraphError::EdgeNameError { - src: src.to_owned(), - dst: dst.to_owned(), - })?; - e.add_constant_properties(const_props, layer.as_deref())?; - if let Some(shared_const_props) = &shared_const_properties { - e.add_constant_properties(shared_const_props.iter(), layer.as_deref())?; + } else if let (Some(src), Some(dst)) = + (df.utf8::(src_index), df.utf8::(dst_index)) + { + let triplets = src.into_iter().zip(dst.into_iter()); + + for (((src, dst), const_props), layer) in triplets.zip(const_prop_iter).zip(layer) { + if let (Some(src), Some(dst)) = (src, dst) { + let e = graph + .edge(src, dst) + .ok_or_else(|| GraphError::EdgeNameError { + src: src.to_owned(), + dst: dst.to_owned(), + })?; + e.add_constant_properties(const_props, layer.as_deref())?; + if let Some(shared_const_props) = &shared_constant_properties { + e.add_constant_properties(shared_const_props.iter(), layer.as_deref())?; + } } + let _ = pb.update(1); } - } - } else { - return Err(GraphError::LoadFailure( - "Source and Target columns must be either u64 or text, Time column must be i64. Ensure these contain no NaN, Null or None values." - .to_string(), - )); + } else { + return Err(GraphError::LoadFailure( + "Source and Target columns must be either u64 or text, Time column must be i64. Ensure these contain no NaN, Null or None values." + .to_string(), + )); + }; } Ok(()) } @@ -558,26 +673,24 @@ fn load_edges_from_num_iter< G: StaticGraphViewOps + InternalPropertyAdditionOps + InternalAdditionOps, >( graph: &G, - size: usize, + pb: &mut Bar, edges: I, properties: PI, - const_properties: PI, - shared_const_properties: Option>, + constant_properties: PI, + shared_constant_properties: Option<&HashMap>, layer: IL, ) -> Result<(), GraphError> { - let iter = maybe_tqdm!( - edges.zip(properties).zip(const_properties).zip(layer), - size, - "Loading edges" - ); - for (((((src, dst), time), edge_props), const_props), layer) in iter { + for (((((src, dst), time), edge_props), const_props), layer) in + edges.zip(properties).zip(constant_properties).zip(layer) + { if let (Some(src), Some(dst), Some(time)) = (src, dst, time) { let e = graph.add_edge(time, src, dst, edge_props, layer.as_deref())?; e.add_constant_properties(const_props, layer.as_deref())?; - if let Some(shared_const_props) = &shared_const_properties { + if let Some(shared_const_props) = &shared_constant_properties { e.add_constant_properties(shared_const_props.iter(), layer.as_deref())?; } } + let _ = pb.update(1); } Ok(()) } @@ -590,18 +703,15 @@ fn load_nodes_from_num_iter< G: StaticGraphViewOps + InternalPropertyAdditionOps + InternalAdditionOps, >( graph: &G, - size: usize, + pb: &mut Bar, nodes: I, properties: PI, - const_properties: PI, - shared_const_properties: Option>, + constant_properties: PI, + shared_constant_properties: Option<&HashMap>, ) -> Result<(), GraphError> { - let iter = maybe_tqdm!( - nodes.zip(properties).zip(const_properties), - size, - "Loading nodes" - ); - for (((node, time, node_type), props), const_props) in iter { + for (((node, time, node_type), props), const_props) in + nodes.zip(properties).zip(constant_properties) + { if let (Some(v), Some(t), n_t, props, const_props) = (node, time, node_type, props, const_props) { @@ -609,9 +719,10 @@ fn load_nodes_from_num_iter< let v = graph.add_node(t, v, props, actual_node_type)?; v.add_constant_properties(const_props)?; - if let Some(shared_const_props) = &shared_const_properties { + if let Some(shared_const_props) = &shared_constant_properties { v.add_constant_properties(shared_const_props.iter())?; } + let _ = pb.update(1); } } Ok(()) diff --git a/raphtory/src/io/arrow/mod.rs b/raphtory/src/io/arrow/mod.rs index 5ba8b90548..64a1a95e9f 100644 --- a/raphtory/src/io/arrow/mod.rs +++ b/raphtory/src/io/arrow/mod.rs @@ -5,7 +5,10 @@ mod prop_handler; #[cfg(test)] mod test { use crate::{ - io::arrow::{dataframe::DFView, df_loaders::*}, + io::arrow::{ + dataframe::{DFChunk, DFView}, + df_loaders::*, + }, prelude::*, }; use polars_arrow::array::{PrimitiveArray, Utf8Array}; @@ -18,37 +21,42 @@ mod test { .iter() .map(|s| s.to_string()) .collect(), - arrays: vec![ - vec![ - Box::new(PrimitiveArray::::from(vec![Some(1)])), - Box::new(PrimitiveArray::::from(vec![Some(2)])), - Box::new(PrimitiveArray::::from(vec![Some(1)])), - Box::new(PrimitiveArray::::from(vec![Some(1.0)])), - Box::new(Utf8Array::::from(vec![Some("a")])), - ], - vec![ - Box::new(PrimitiveArray::::from(vec![Some(2), Some(3)])), - Box::new(PrimitiveArray::::from(vec![Some(3), Some(4)])), - Box::new(PrimitiveArray::::from(vec![Some(2), Some(3)])), - Box::new(PrimitiveArray::::from(vec![Some(2.0), Some(3.0)])), - Box::new(Utf8Array::::from(vec![Some("b"), Some("c")])), - ], - ], + chunks: vec![ + Ok(DFChunk { + chunk: vec![ + Box::new(PrimitiveArray::::from(vec![Some(1)])), + Box::new(PrimitiveArray::::from(vec![Some(2)])), + Box::new(PrimitiveArray::::from(vec![Some(1)])), + Box::new(PrimitiveArray::::from(vec![Some(1.0)])), + Box::new(Utf8Array::::from(vec![Some("a")])), + ], + }), + Ok(DFChunk { + chunk: vec![ + Box::new(PrimitiveArray::::from(vec![Some(2), Some(3)])), + Box::new(PrimitiveArray::::from(vec![Some(3), Some(4)])), + Box::new(PrimitiveArray::::from(vec![Some(2), Some(3)])), + Box::new(PrimitiveArray::::from(vec![Some(2.0), Some(3.0)])), + Box::new(Utf8Array::::from(vec![Some("b"), Some("c")])), + ], + }), + ] + .into_iter(), + num_rows: 3, }; let graph = Graph::new(); - let layer: Option<&str> = None; - let layer_in_df: bool = true; + let layer_name: Option<&str> = None; + let layer_col: Option<&str> = None; load_edges_from_df( - &df, - 5, + df, + "time", "src", "dst", - "time", - Some(vec!["prop1", "prop2"]), + Some(&*vec!["prop1", "prop2"]), None, None, - layer, - layer_in_df, + layer_name, + layer_col, &graph, ) .expect("failed to load edges from pretend df"); @@ -108,33 +116,38 @@ mod test { .iter() .map(|s| s.to_string()) .collect(), - arrays: vec![ - vec![ - Box::new(PrimitiveArray::::from(vec![Some(1)])), - Box::new(Utf8Array::::from(vec![Some("a")])), - Box::new(PrimitiveArray::::from(vec![Some(1)])), - Box::new(Utf8Array::::from(vec![Some("atype")])), - ], - vec![ - Box::new(PrimitiveArray::::from(vec![Some(2)])), - Box::new(Utf8Array::::from(vec![Some("b")])), - Box::new(PrimitiveArray::::from(vec![Some(2)])), - Box::new(Utf8Array::::from(vec![Some("btype")])), - ], - ], + chunks: vec![ + Ok(DFChunk { + chunk: vec![ + Box::new(PrimitiveArray::::from(vec![Some(1)])), + Box::new(Utf8Array::::from(vec![Some("a")])), + Box::new(PrimitiveArray::::from(vec![Some(1)])), + Box::new(Utf8Array::::from(vec![Some("atype")])), + ], + }), + Ok(DFChunk { + chunk: vec![ + Box::new(PrimitiveArray::::from(vec![Some(2)])), + Box::new(Utf8Array::::from(vec![Some("b")])), + Box::new(PrimitiveArray::::from(vec![Some(2)])), + Box::new(Utf8Array::::from(vec![Some("btype")])), + ], + }), + ] + .into_iter(), + num_rows: 2, }; let graph = Graph::new(); load_nodes_from_df( - &df, - 3, - "id", + df, "time", - Some(vec!["name"]), + "id", + Some(&*vec!["name"]), None, None, Some("node_type"), - false, + None, &graph, ) .expect("failed to load nodes from pretend df"); diff --git a/raphtory/src/io/arrow/prop_handler.rs b/raphtory/src/io/arrow/prop_handler.rs index c3d07979c4..2d91f6e542 100644 --- a/raphtory/src/io/arrow/prop_handler.rs +++ b/raphtory/src/io/arrow/prop_handler.rs @@ -6,51 +6,45 @@ use polars_arrow::{ use crate::{ core::{utils::errors::GraphError, IntoPropList}, - io::arrow::dataframe::DFView, + io::arrow::dataframe::DFChunk, prelude::Prop, }; pub struct PropIter<'a> { - inner: Box> + 'a>, + inner: Vec> + 'a>>, } impl<'a> Iterator for PropIter<'a> { type Item = Vec<(&'a str, Prop)>; fn next(&mut self) -> Option { - self.inner.next() + self.inner + .iter_mut() + .map(|v| v.next()) + .filter_map(|r| match r { + Some(r1) => match r1 { + Some(r2) => Some(Some(r2)), + None => None, + }, + None => Some(None), + }) + .collect() } } -pub(crate) fn get_prop_rows<'a>( - df: &'a DFView, - props: Option>, - const_props: Option>, -) -> Result<(PropIter<'a>, PropIter<'a>), GraphError> { - let prop_iter = combine_properties(props, df)?; - let const_prop_iter = combine_properties(const_props, df)?; - Ok((prop_iter, const_prop_iter)) -} - -fn combine_properties<'a>( - props: Option>, - df: &'a DFView, +pub(crate) fn combine_properties<'a>( + props: &'a [&str], + indices: &'a [usize], + df: &'a DFChunk, ) -> Result, GraphError> { - let iter = props - .unwrap_or_default() - .into_iter() - .map(|name| lift_property(name, df)) - .reduce(|i1, i2| { - let i1 = i1?; - let i2 = i2?; - Ok(Box::new(i1.zip(i2).map(|(mut v1, v2)| { - v1.extend(v2); - v1 - }))) - }) - .unwrap_or_else(|| Ok(Box::new(std::iter::repeat(vec![])))); - - Ok(PropIter { inner: iter? }) + for idx in indices { + is_data_type_supported(df.chunk[*idx].data_type())?; + } + let zipped = props.iter().zip(indices.iter()); + let iter = zipped.map(|(name, idx)| lift_property(*idx, name, df)); + Ok(PropIter { + inner: iter.collect(), + }) } fn arr_as_prop(arr: Box) -> Prop { @@ -124,7 +118,7 @@ fn arr_as_prop(arr: Box) -> Prop { } } -fn validate_data_types(dt: &DataType) -> Result<(), GraphError> { +fn is_data_type_supported(dt: &DataType) -> Result<(), GraphError> { match dt { DataType::Boolean => {} DataType::Int32 => {} @@ -137,9 +131,9 @@ fn validate_data_types(dt: &DataType) -> Result<(), GraphError> { DataType::Float64 => {} DataType::Utf8 => {} DataType::LargeUtf8 => {} - DataType::List(v) => validate_data_types(v.data_type())?, - DataType::FixedSizeList(v, _) => validate_data_types(v.data_type())?, - DataType::LargeList(v) => validate_data_types(v.data_type())?, + DataType::List(v) => is_data_type_supported(v.data_type())?, + DataType::FixedSizeList(v, _) => is_data_type_supported(v.data_type())?, + DataType::LargeList(v) => is_data_type_supported(v.data_type())?, DataType::Timestamp(_, _) => {} _ => Err(GraphError::UnsupportedDataType)?, } @@ -147,279 +141,241 @@ fn validate_data_types(dt: &DataType) -> Result<(), GraphError> { } pub(crate) fn lift_property<'a: 'b, 'b>( + idx: usize, name: &'a str, - df: &'b DFView, -) -> Result> + 'b>, GraphError> { - let idx = df - .names - .iter() - .position(|n| n == name) - .ok_or_else(|| GraphError::ColumnDoesNotExist(name.to_string()))?; - - if let Some(first_chunk) = df.arrays.get(0) { - validate_data_types(first_chunk[idx].data_type())?; - } - - let r = df.arrays.iter().flat_map(move |arr| { - let arr: &Box = &arr[idx]; - match arr.data_type() { - DataType::Boolean => { - let arr = arr.as_any().downcast_ref::().unwrap(); - iter_as_prop(name, arr.iter()) - } - DataType::Int32 => { - let arr = arr.as_any().downcast_ref::>().unwrap(); - iter_as_prop(name, arr.iter().map(|i| i.copied())) - } - DataType::Int64 => { - let arr = arr.as_any().downcast_ref::>().unwrap(); - iter_as_prop(name, arr.iter().map(|i| i.copied())) - } - DataType::UInt8 => { - let arr = arr.as_any().downcast_ref::>().unwrap(); - iter_as_prop(name, arr.iter().map(|i| i.copied())) - } - DataType::UInt16 => { - let arr = arr.as_any().downcast_ref::>().unwrap(); - iter_as_prop(name, arr.iter().map(|i| i.copied())) - } - DataType::UInt32 => { - let arr = arr.as_any().downcast_ref::>().unwrap(); - iter_as_prop(name, arr.iter().map(|i| i.copied())) - } - DataType::UInt64 => { - let arr = arr.as_any().downcast_ref::>().unwrap(); - iter_as_prop(name, arr.iter().map(|i| i.copied())) - } - DataType::Float32 => { - let arr = arr.as_any().downcast_ref::>().unwrap(); - iter_as_prop(name, arr.iter().map(|i| i.copied())) - } - DataType::Float64 => { - let arr = arr.as_any().downcast_ref::>().unwrap(); - iter_as_prop(name, arr.iter().map(|i| i.copied())) - } - DataType::Utf8 => { - let arr = arr.as_any().downcast_ref::>().unwrap(); - iter_as_prop(name, arr.iter()) - } - DataType::LargeUtf8 => { - let arr = arr.as_any().downcast_ref::>().unwrap(); - iter_as_prop(name, arr.iter()) - } - DataType::List(_) => { - let arr = arr.as_any().downcast_ref::>().unwrap(); - iter_as_arr_prop(name, arr.iter()) - } - DataType::FixedSizeList(_, _) => { - let arr = arr.as_any().downcast_ref::().unwrap(); - iter_as_arr_prop(name, arr.iter()) - } - DataType::LargeList(_) => { - let arr = arr.as_any().downcast_ref::>().unwrap(); - iter_as_arr_prop(name, arr.iter()) - } - DataType::Timestamp(timeunit, timezone) => { - let arr = arr.as_any().downcast_ref::>().unwrap(); - match timezone { - Some(_) => match timeunit { - TimeUnit::Second => { - println!("Timestamp(Second, Some({:?})); ", timezone); - let r: Box> + 'b> = - Box::new(arr.iter().map(move |val| { - val.into_iter() - .map(|v| { - ( - name, - Prop::DTime( - DateTime::::from_timestamp(*v, 0) - .expect("DateTime conversion failed"), - ), - ) - }) - .collect::>() - })); - r - } - TimeUnit::Millisecond => { - println!("Timestamp(Millisecond, Some({:?})); ", timezone); - let r: Box> + 'b> = - Box::new(arr.iter().map(move |val| { - val.into_iter() - .map(|v| { - ( - name, - Prop::DTime( - DateTime::::from_timestamp_millis(*v) - .expect("DateTime conversion failed"), - ), - ) - }) - .collect::>() - })); - r - } - TimeUnit::Microsecond => { - println!("Timestamp(Microsecond, Some({:?})); ", timezone); - let r: Box> + 'b> = - Box::new(arr.iter().map(move |val| { - val.into_iter() - .map(|v| { - ( - name, - Prop::DTime( - DateTime::::from_timestamp_micros(*v) - .expect("DateTime conversion failed"), - ), - ) - }) - .collect::>() - })); - r - } - TimeUnit::Nanosecond => { - println!("Timestamp(Nanosecond, Some({:?})); ", timezone); - let r: Box> + 'b> = - Box::new(arr.iter().map(move |val| { - val.into_iter() - .map(|v| { - ( - name, - Prop::DTime(DateTime::::from_timestamp_nanos( - *v, - )), - ) - }) - .collect::>() - })); - r - } - }, - None => match timeunit { - TimeUnit::Second => { - println!("Timestamp(Second, None); "); - let r: Box> + 'b> = - Box::new(arr.iter().map(move |val| { - val.into_iter() - .map(|v| { - ( - name, - Prop::NDTime( - DateTime::from_timestamp(*v, 0) - .expect("DateTime conversion failed") - .naive_utc(), - ), - ) - }) - .collect::>() - })); - r - } - TimeUnit::Millisecond => { - println!("Timestamp(Millisecond, None); "); - let r: Box> + 'b> = - Box::new(arr.iter().map(move |val| { - val.into_iter() - .map(|v| { - ( - name, - Prop::NDTime( - DateTime::from_timestamp_millis(*v) - .expect("DateTime conversion failed") - .naive_utc(), - ), - ) - }) - .collect::>() - })); - r - } - TimeUnit::Microsecond => { - println!("Timestamp(Microsecond, None); "); - let r: Box> + 'b> = - Box::new(arr.iter().map(move |val| { - val.into_iter() - .map(|v| { - ( - name, - Prop::NDTime( - DateTime::from_timestamp_micros(*v) - .expect("DateTime conversion failed") - .naive_utc(), - ), - ) - }) - .collect::>() - })); - r - } - TimeUnit::Nanosecond => { - println!("Timestamp(Nanosecond, None); "); - let r: Box> + 'b> = - Box::new(arr.iter().map(move |val| { - val.into_iter() - .map(|v| { - ( - name, - Prop::NDTime( - DateTime::from_timestamp_nanos(*v).naive_utc(), - ), - ) - }) - .collect::>() - })); - r - } - }, - } + df: &'b DFChunk, +) -> Box> + 'b> { + let arr = &df.chunk[idx]; + let r = match arr.data_type() { + DataType::Boolean => { + let arr = arr.as_any().downcast_ref::().unwrap(); + iter_as_prop(name, arr.iter()) + } + DataType::Int32 => { + let arr = arr.as_any().downcast_ref::>().unwrap(); + iter_as_prop(name, arr.iter().map(|i| i.copied())) + } + DataType::Int64 => { + let arr = arr.as_any().downcast_ref::>().unwrap(); + iter_as_prop(name, arr.iter().map(|i| i.copied())) + } + DataType::UInt8 => { + let arr = arr.as_any().downcast_ref::>().unwrap(); + iter_as_prop(name, arr.iter().map(|i| i.copied())) + } + DataType::UInt16 => { + let arr = arr.as_any().downcast_ref::>().unwrap(); + iter_as_prop(name, arr.iter().map(|i| i.copied())) + } + DataType::UInt32 => { + let arr = arr.as_any().downcast_ref::>().unwrap(); + iter_as_prop(name, arr.iter().map(|i| i.copied())) + } + DataType::UInt64 => { + let arr = arr.as_any().downcast_ref::>().unwrap(); + iter_as_prop(name, arr.iter().map(|i| i.copied())) + } + DataType::Float32 => { + let arr = arr.as_any().downcast_ref::>().unwrap(); + iter_as_prop(name, arr.iter().map(|i| i.copied())) + } + DataType::Float64 => { + let arr = arr.as_any().downcast_ref::>().unwrap(); + iter_as_prop(name, arr.iter().map(|i| i.copied())) + } + DataType::Utf8 => { + let arr = arr.as_any().downcast_ref::>().unwrap(); + iter_as_prop(name, arr.iter()) + } + DataType::LargeUtf8 => { + let arr = arr.as_any().downcast_ref::>().unwrap(); + iter_as_prop(name, arr.iter()) + } + DataType::List(_) => { + let arr = arr.as_any().downcast_ref::>().unwrap(); + iter_as_arr_prop(name, arr.iter()) + } + DataType::FixedSizeList(_, _) => { + let arr = arr.as_any().downcast_ref::().unwrap(); + iter_as_arr_prop(name, arr.iter()) + } + DataType::LargeList(_) => { + let arr = arr.as_any().downcast_ref::>().unwrap(); + iter_as_arr_prop(name, arr.iter()) + } + DataType::Timestamp(timeunit, timezone) => { + let arr = arr.as_any().downcast_ref::>().unwrap(); + match timezone { + Some(_) => match timeunit { + TimeUnit::Second => { + println!("Timestamp(Second, Some({:?})); ", timezone); + let r: Box> + 'b> = + Box::new(arr.iter().map(move |val| { + val.map(|v| { + ( + name, + Prop::DTime( + DateTime::::from_timestamp(*v, 0) + .expect("DateTime conversion failed"), + ), + ) + }) + })); + r + } + TimeUnit::Millisecond => { + println!("Timestamp(Millisecond, Some({:?})); ", timezone); + let r: Box> + 'b> = + Box::new(arr.iter().map(move |val| { + val.map(|v| { + ( + name, + Prop::DTime( + DateTime::::from_timestamp_millis(*v) + .expect("DateTime conversion failed"), + ), + ) + }) + })); + r + } + TimeUnit::Microsecond => { + println!("Timestamp(Microsecond, Some({:?})); ", timezone); + let r: Box> + 'b> = + Box::new(arr.iter().map(move |val| { + val.map(|v| { + ( + name, + Prop::DTime( + DateTime::::from_timestamp_micros(*v) + .expect("DateTime conversion failed"), + ), + ) + }) + })); + r + } + TimeUnit::Nanosecond => { + println!("Timestamp(Nanosecond, Some({:?})); ", timezone); + let r: Box> + 'b> = + Box::new(arr.iter().map(move |val| { + val.map(|v| { + (name, Prop::DTime(DateTime::::from_timestamp_nanos(*v))) + }) + })); + r + } + }, + None => match timeunit { + TimeUnit::Second => { + println!("Timestamp(Second, None); "); + let r: Box> + 'b> = + Box::new(arr.iter().map(move |val| { + val.map(|v| { + ( + name, + Prop::NDTime( + DateTime::from_timestamp(*v, 0) + .expect("DateTime conversion failed") + .naive_utc(), + ), + ) + }) + })); + r + } + TimeUnit::Millisecond => { + println!("Timestamp(Millisecond, None); "); + let r: Box> + 'b> = + Box::new(arr.iter().map(move |val| { + val.map(|v| { + ( + name, + Prop::NDTime( + DateTime::from_timestamp_millis(*v) + .expect("DateTime conversion failed") + .naive_utc(), + ), + ) + }) + })); + r + } + TimeUnit::Microsecond => { + println!("Timestamp(Microsecond, None); "); + let r: Box> + 'b> = + Box::new(arr.iter().map(move |val| { + val.map(|v| { + ( + name, + Prop::NDTime( + DateTime::from_timestamp_micros(*v) + .expect("DateTime conversion failed") + .naive_utc(), + ), + ) + }) + })); + r + } + TimeUnit::Nanosecond => { + println!("Timestamp(Nanosecond, None); "); + let r: Box> + 'b> = + Box::new(arr.iter().map(move |val| { + val.map(|v| { + ( + name, + Prop::NDTime( + DateTime::from_timestamp_nanos(*v).naive_utc(), + ), + ) + }) + })); + r + } + }, } - unsupported => panic!("Data type not supported: {:?}", unsupported), } - }); + unsupported => panic!("Data type not supported: {:?}", unsupported), + }; - Ok(Box::new(r)) + r } -pub(crate) fn lift_layer<'a, S: AsRef>( - layer: Option, - layer_in_df: bool, - df: &'a DFView, -) -> Box> + 'a> { - if let Some(layer) = layer { - if layer_in_df { - if let Some(col) = df.utf8::(layer.as_ref()) { - Box::new(col.map(|v| v.map(|v| v.to_string()))) - } else if let Some(col) = df.utf8::(layer.as_ref()) { - Box::new(col.map(|v| v.map(|v| v.to_string()))) +pub(crate) fn lift_layer<'a>( + layer_name: Option<&str>, + layer_index: Option, + df: &'a DFChunk, +) -> Result> + 'a>, GraphError> { + match (layer_name, layer_index) { + (None, None) => Ok(Box::new(std::iter::repeat(None))), + (Some(layer_name), None) => Ok(Box::new(std::iter::repeat(Some(layer_name.to_string())))), + (None, Some(layer_index)) => { + if let Some(col) = df.utf8::(layer_index) { + Ok(Box::new(col.map(|v| v.map(|v| v.to_string())))) + } else if let Some(col) = df.utf8::(layer_index) { + Ok(Box::new(col.map(|v| v.map(|v| v.to_string())))) } else { - Box::new(std::iter::repeat(None)) + Ok(Box::new(std::iter::repeat(None))) } - } else { - Box::new(std::iter::repeat(Some(layer.as_ref().to_string()))) } - } else { - Box::new(std::iter::repeat(None)) + _ => Err(GraphError::WrongNumOfArgs( + "layer_name".to_string(), + "layer_col".to_string(), + )), } } fn iter_as_prop<'a, T: Into + 'a, I: Iterator> + 'a>( name: &'a str, is: I, -) -> Box> + 'a> { - Box::new(is.map(move |val| { - val.into_iter() - .map(|v| (name, (v).into())) - .collect::>() - })) +) -> Box> + 'a> { + Box::new(is.map(move |val| val.map(|v| (name, v.into())))) } fn iter_as_arr_prop<'a, I: Iterator>> + 'a>( name: &'a str, is: I, -) -> Box> + 'a> { - Box::new(is.map(move |val| { - val.into_iter() - .map(|v| (name, arr_as_prop(v))) - .collect::>() - })) +) -> Box> + 'a> { + Box::new(is.map(move |val| val.map(|v| (name, arr_as_prop(v))))) } diff --git a/raphtory/src/io/neo4j_loader.rs b/raphtory/src/io/neo4j_loader.rs index 882d771afd..3013c76841 100644 --- a/raphtory/src/io/neo4j_loader.rs +++ b/raphtory/src/io/neo4j_loader.rs @@ -1,5 +1,6 @@ use crate::db::graph::graph as rap; use neo4rs::*; + /// A struct that defines the Neo4J loader with configurable options. pub struct Neo4JConnection { // The created graph object given the arguments @@ -23,21 +24,13 @@ impl Neo4JConnection { Ok(Self { neo_graph: graph }) } - pub async fn run(&self, query: Query) -> Result<()> { - self.neo_graph.run(query).await - } - - pub async fn execute(&self, query: Query) -> Result { - self.neo_graph.execute(query).await - } - pub async fn load_query_into_graph( &self, g: &rap::Graph, query: Query, loader: fn(Row, &rap::Graph), ) -> Result<()> { - let mut result = self.neo_graph.execute(query).await.unwrap(); + let mut result = self.neo_graph.execute(query).await?; while let Ok(Some(row)) = result.next().await { loader(row, g); @@ -87,7 +80,7 @@ mod neo_loader_test { actor_name, film_title, NO_PROPS, - Some(relation_type.as_str()), + Some(relation_type), ) .unwrap(); } @@ -128,11 +121,13 @@ mod neo_loader_test { .await .unwrap(); - neo.run(query("CREATE (p:Person {id: $id})").param("id", 3)) + neo.neo_graph + .run(query("CREATE (p:Person {id: $id})").param("id", 3)) .await .unwrap(); let mut result = neo + .neo_graph .execute(query("MATCH (p:Person {id: $id}) RETURN p").param("id", 3)) .await .unwrap(); diff --git a/raphtory/src/io/parquet_loaders.rs b/raphtory/src/io/parquet_loaders.rs index ff27f45da3..ee5a828eda 100644 --- a/raphtory/src/io/parquet_loaders.rs +++ b/raphtory/src/io/parquet_loaders.rs @@ -11,12 +11,7 @@ use crate::{ prelude::DeletionOps, }; use itertools::Itertools; -use polars_arrow::{ - array::Array, - datatypes::{ArrowDataType as DataType, ArrowSchema, Field}, - legacy::error, - record_batch::RecordBatch as Chunk, -}; +use polars_arrow::datatypes::{ArrowDataType as DataType, ArrowSchema, Field}; use polars_parquet::{ read, read::{read_metadata, FileMetaData, FileReader}, @@ -24,6 +19,7 @@ use polars_parquet::{ use std::{ collections::HashMap, fs, + fs::File, path::{Path, PathBuf}, }; @@ -32,37 +28,33 @@ pub fn load_nodes_from_parquet< >( graph: &G, parquet_path: &Path, - id: &str, time: &str, + id: &str, node_type: Option<&str>, - node_type_in_df: Option, - properties: Option>, - const_properties: Option>, - shared_const_properties: Option>, + node_type_col: Option<&str>, + properties: Option<&[&str]>, + constant_properties: Option<&[&str]>, + shared_constant_properties: Option<&HashMap>, ) -> Result<(), GraphError> { let mut cols_to_check = vec![id, time]; - cols_to_check.extend(properties.as_ref().unwrap_or(&Vec::new())); - cols_to_check.extend(const_properties.as_ref().unwrap_or(&Vec::new())); - if node_type_in_df.unwrap_or(true) { - if let Some(ref node_type) = node_type { - cols_to_check.push(node_type.as_ref()); - } + cols_to_check.extend(properties.unwrap_or(&Vec::new())); + cols_to_check.extend(constant_properties.unwrap_or(&Vec::new())); + if let Some(ref node_type_col) = node_type_col { + cols_to_check.push(node_type_col.as_ref()); } for path in get_parquet_file_paths(parquet_path)? { - let df = process_parquet_file_to_df(path.as_path(), cols_to_check.clone())?; - df.check_cols_exist(&cols_to_check)?; - let size = df.get_inner_size(); + let df_view = process_parquet_file_to_df(path.as_path(), &cols_to_check)?; + df_view.check_cols_exist(&cols_to_check)?; load_nodes_from_df( - &df, - size, - id, + df_view, time, - properties.clone(), - const_properties.clone(), - shared_const_properties.clone(), + id, + properties, + constant_properties, + shared_constant_properties, node_type, - node_type_in_df.unwrap_or(true), + node_type_col, graph, ) .map_err(|e| GraphError::LoadFailure(format!("Failed to load graph {e:?}")))?; @@ -76,40 +68,36 @@ pub fn load_edges_from_parquet< >( graph: &G, parquet_path: impl AsRef, + time: &str, src: &str, dst: &str, - time: &str, - properties: Option>, - const_properties: Option>, - shared_const_properties: Option>, + properties: Option<&[&str]>, + constant_properties: Option<&[&str]>, + shared_constant_properties: Option<&HashMap>, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { let parquet_path = parquet_path.as_ref(); let mut cols_to_check = vec![src, dst, time]; - cols_to_check.extend(properties.as_ref().unwrap_or(&Vec::new())); - cols_to_check.extend(const_properties.as_ref().unwrap_or(&Vec::new())); - if layer_in_df.unwrap_or(false) { - if let Some(ref layer) = layer { - cols_to_check.push(layer.as_ref()); - } + cols_to_check.extend(properties.unwrap_or(&Vec::new())); + cols_to_check.extend(constant_properties.unwrap_or(&Vec::new())); + if let Some(ref layer_col) = layer_col { + cols_to_check.push(layer_col.as_ref()); } for path in get_parquet_file_paths(parquet_path)? { - let df = process_parquet_file_to_df(path.as_path(), cols_to_check.clone())?; - df.check_cols_exist(&cols_to_check)?; - let size = cols_to_check.len(); + let df_view = process_parquet_file_to_df(path.as_path(), &cols_to_check)?; + df_view.check_cols_exist(&cols_to_check)?; load_edges_from_df( - &df, - size, + df_view, + time, src, dst, - time, - properties.clone(), - const_properties.clone(), - shared_const_properties.clone(), + properties, + constant_properties, + shared_constant_properties, layer, - layer_in_df.unwrap_or(true), + layer_col, graph, ) .map_err(|e| GraphError::LoadFailure(format!("Failed to load graph {e:?}")))?; @@ -124,22 +112,28 @@ pub fn load_node_props_from_parquet< graph: &G, parquet_path: &Path, id: &str, - const_properties: Option>, - shared_const_properties: Option>, + node_type: Option<&str>, + node_type_col: Option<&str>, + constant_properties: Option<&[&str]>, + shared_constant_properties: Option<&HashMap>, ) -> Result<(), GraphError> { let mut cols_to_check = vec![id]; - cols_to_check.extend(const_properties.as_ref().unwrap_or(&Vec::new())); + cols_to_check.extend(constant_properties.unwrap_or(&Vec::new())); + if let Some(ref node_type_col) = node_type_col { + cols_to_check.push(node_type_col.as_ref()); + } for path in get_parquet_file_paths(parquet_path)? { - let df = process_parquet_file_to_df(path.as_path(), cols_to_check.clone())?; - df.check_cols_exist(&cols_to_check)?; - let size = cols_to_check.len(); + let df_view = process_parquet_file_to_df(path.as_path(), &cols_to_check)?; + df_view.check_cols_exist(&cols_to_check)?; + load_node_props_from_df( - &df, - size, + df_view, id, - const_properties.clone(), - shared_const_properties.clone(), + node_type, + node_type_col, + constant_properties, + shared_constant_properties, graph, ) .map_err(|e| GraphError::LoadFailure(format!("Failed to load graph {e:?}")))?; @@ -155,32 +149,28 @@ pub fn load_edge_props_from_parquet< parquet_path: &Path, src: &str, dst: &str, - const_properties: Option>, - shared_const_properties: Option>, + constant_properties: Option<&[&str]>, + shared_const_properties: Option<&HashMap>, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { let mut cols_to_check = vec![src, dst]; - if layer_in_df.unwrap_or(false) { - if let Some(ref layer) = layer { - cols_to_check.push(layer.as_ref()); - } + if let Some(ref layer_col) = layer_col { + cols_to_check.push(layer_col.as_ref()); } - cols_to_check.extend(const_properties.as_ref().unwrap_or(&Vec::new())); + cols_to_check.extend(constant_properties.unwrap_or(&Vec::new())); for path in get_parquet_file_paths(parquet_path)? { - let df = process_parquet_file_to_df(path.as_path(), cols_to_check.clone())?; - df.check_cols_exist(&cols_to_check)?; - let size = cols_to_check.len(); + let df_view = process_parquet_file_to_df(path.as_path(), &cols_to_check)?; + df_view.check_cols_exist(&cols_to_check)?; load_edges_props_from_df( - &df, - size, + df_view, src, dst, - const_properties.clone(), - shared_const_properties.clone(), + constant_properties, + shared_const_properties, layer, - layer_in_df.unwrap_or(true), + layer_col, graph.core_graph(), ) .map_err(|e| GraphError::LoadFailure(format!("Failed to load graph {e:?}")))?; @@ -189,72 +179,64 @@ pub fn load_edge_props_from_parquet< Ok(()) } -pub fn load_edges_deletions_from_parquet< +pub fn load_edge_deletions_from_parquet< G: StaticGraphViewOps + InternalPropertyAdditionOps + InternalAdditionOps + DeletionOps, >( graph: &G, parquet_path: &Path, + time: &str, src: &str, dst: &str, - time: &str, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { let mut cols_to_check = vec![src, dst, time]; - if layer_in_df.unwrap_or(true) { - if let Some(ref layer) = layer { - cols_to_check.push(layer.as_ref()); - } + if let Some(ref layer_col) = layer_col { + cols_to_check.push(layer_col.as_ref()); } for path in get_parquet_file_paths(parquet_path)? { - let df = process_parquet_file_to_df(path.as_path(), cols_to_check.clone())?; - df.check_cols_exist(&cols_to_check)?; - let size = cols_to_check.len(); - load_edges_deletions_from_df( - &df, - size, - src, - dst, - time, - layer, - layer_in_df.unwrap_or(true), - graph, - ) - .map_err(|e| GraphError::LoadFailure(format!("Failed to load graph {e:?}")))?; + let df_view = process_parquet_file_to_df(path.as_path(), &cols_to_check)?; + df_view.check_cols_exist(&cols_to_check)?; + load_edge_deletions_from_df(df_view, time, src, dst, layer, layer_col, graph) + .map_err(|e| GraphError::LoadFailure(format!("Failed to load graph {e:?}")))?; } - Ok(()) } pub(crate) fn process_parquet_file_to_df( parquet_file_path: &Path, - col_names: Vec<&str>, -) -> Result { - let (names, arrays) = read_parquet_file(parquet_file_path, &col_names)?; + col_names: &[&str], +) -> Result>>, GraphError> { + let (names, chunks, num_rows) = read_parquet_file(parquet_file_path, col_names)?; - let names = names + let names: Vec = names .into_iter() .filter(|x| col_names.contains(&x.as_str())) .collect(); - let arrays = arrays - .map_ok(|r| r.into_iter().map(|boxed| boxed.clone()).collect_vec()) - .collect::, _>>()?; - Ok(DFView { names, arrays }) + let chunks = chunks.into_iter().map(move |result| { + result + .map(|r| DFChunk { + chunk: r.into_iter().map(|boxed| boxed.clone()).collect_vec(), + }) + .map_err(|e| { + GraphError::LoadFailure(format!("Failed to process Parquet file: {:?}", e)) + }) + }); + + Ok(DFView { + names, + chunks, + num_rows, + }) } fn read_parquet_file( path: impl AsRef, - col_names: &Vec<&str>, -) -> Result< - ( - Vec, - impl Iterator>, error::PolarsError>>, - ), - GraphError, -> { - let read_schema = |metadata: &FileMetaData| -> Result { + col_names: &[&str], +) -> Result<(Vec, FileReader, usize), GraphError> { + let read_schema = |metadata: &FileMetaData| -> Result<(ArrowSchema, usize), GraphError> { let schema = read::infer_schema(metadata)?; let fields = schema .fields @@ -272,20 +254,23 @@ fn read_parquet_file( }) .collect::>(); - Ok(ArrowSchema::from(fields).with_metadata(schema.metadata)) + Ok(( + ArrowSchema::from(fields).with_metadata(schema.metadata), + metadata.num_rows, + )) }; let mut file = std::fs::File::open(&path)?; let metadata = read_metadata(&mut file)?; let row_groups = metadata.clone().row_groups; - let schema = read_schema(&metadata)?; + let (schema, num_rows) = read_schema(&metadata)?; // Although fields are already filtered by col_names, we need names in the order as it appears // in the schema to create PretendDF let names = schema.fields.iter().map(|f| f.name.clone()).collect_vec(); let reader = FileReader::new(file, row_groups, schema, None, None, None); - Ok((names, reader)) + Ok((names, reader, num_rows)) } fn get_parquet_file_paths(parquet_path: &Path) -> Result, GraphError> { @@ -312,7 +297,7 @@ fn get_parquet_file_paths(parquet_path: &Path) -> Result, GraphErro #[cfg(test)] mod test { use super::*; - use polars_arrow::array::{PrimitiveArray, Utf8Array}; + use polars_arrow::array::{Array, PrimitiveArray, Utf8Array}; use std::path::PathBuf; #[test] @@ -320,28 +305,33 @@ mod test { let parquet_file_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("resources/test/test_data.parquet"); - let col_names = vec!["src", "dst", "time", "weight", "marbles"]; + let col_names: &[&str] = &["src", "dst", "time", "weight", "marbles"]; let df = process_parquet_file_to_df(parquet_file_path.as_path(), col_names).unwrap(); - let df1 = DFView { - names: vec!["src", "dst", "time", "weight", "marbles"] - .iter() - .map(|s| s.to_string()) - .collect(), - arrays: vec![vec![ - Box::new(PrimitiveArray::::from_values(vec![1, 2, 3, 4, 5])), - Box::new(PrimitiveArray::::from_values(vec![2, 3, 4, 5, 6])), - Box::new(PrimitiveArray::::from_values(vec![1, 2, 3, 4, 5])), - Box::new(PrimitiveArray::::from_values(vec![ - 1f64, 2f64, 3f64, 4f64, 5f64, - ])), - Box::new(Utf8Array::::from_iter_values( - vec!["red", "blue", "green", "yellow", "purple"].into_iter(), - )), - ]], - }; - - assert_eq!(df.names, df1.names); - assert_eq!(df.arrays, df1.arrays); + let expected_names: Vec = vec!["src", "dst", "time", "weight", "marbles"] + .iter() + .map(|s| s.to_string()) + .collect(); + let expected_chunks: Vec>> = vec![vec![ + Box::new(PrimitiveArray::::from_values(vec![1, 2, 3, 4, 5])), + Box::new(PrimitiveArray::::from_values(vec![2, 3, 4, 5, 6])), + Box::new(PrimitiveArray::::from_values(vec![1, 2, 3, 4, 5])), + Box::new(PrimitiveArray::::from_values(vec![ + 1f64, 2f64, 3f64, 4f64, 5f64, + ])), + Box::new(Utf8Array::::from_iter_values( + vec!["red", "blue", "green", "yellow", "purple"].into_iter(), + )), + ]]; + + let actual_names = df.names; + let chunks: Vec> = df.chunks.collect_vec(); + let chunks: Result, GraphError> = chunks.into_iter().collect(); + let chunks: Vec = chunks.unwrap(); + let actual_chunks: Vec>> = + chunks.into_iter().map(|c: DFChunk| c.chunk).collect_vec(); + + assert_eq!(actual_names, expected_names); + assert_eq!(actual_chunks, expected_chunks); } } diff --git a/raphtory/src/lib.rs b/raphtory/src/lib.rs index 0e48df2be3..3acb134bf7 100644 --- a/raphtory/src/lib.rs +++ b/raphtory/src/lib.rs @@ -82,7 +82,7 @@ //! raphtory is created by [Pometry](https://pometry.com). //! We are always looking for contributors to help us improve the library. //! If you are interested in contributing, please see -//! our [Github repository](https://github.com/Raphtory/raphtory) +//! our [GitHub repository](https://github.com/Raphtory/raphtory) pub mod algorithms; pub mod core; pub mod db; @@ -107,6 +107,9 @@ pub mod vectors; #[cfg(feature = "io")] pub mod io; +#[cfg(feature = "proto")] +pub mod serialise; + pub mod prelude { pub const NO_PROPS: [(&str, Prop); 0] = []; pub use crate::{ @@ -123,18 +126,14 @@ pub mod prelude { }, }; pub use raphtory_api::core::{entities::GID, input::input_node::InputNode}; + + #[cfg(feature = "proto")] + pub use crate::serialise::{CacheOps, StableDecode, StableEncode}; } -// Upgrade this version number every time you make a breaking change to Graph structure. -pub const BINCODE_VERSION: u32 = 3u32; #[cfg(feature = "storage")] pub use polars_arrow as arrow2; -#[cfg(feature = "proto")] -mod serialise { - include!(concat!(env!("OUT_DIR"), "/serialise.rs")); -} - #[cfg(test)] mod test_utils { use crate::prelude::Graph; diff --git a/raphtory/src/python/graph/disk_graph.rs b/raphtory/src/python/graph/disk_graph.rs index c5e6426b31..06f662d343 100644 --- a/raphtory/src/python/graph/disk_graph.rs +++ b/raphtory/src/python/graph/disk_graph.rs @@ -6,12 +6,10 @@ use crate::{ }, core::utils::errors::GraphError, db::graph::views::deletion_graph::PersistentGraph, - disk_graph::{graph_impl::ParquetLayerCols, DiskGraphError, DiskGraphStorage}, - io::arrow::dataframe::DFView, + disk_graph::{graph_impl::ParquetLayerCols, DiskGraphStorage}, + io::arrow::dataframe::{DFChunk, DFView}, prelude::Graph, - python::{ - graph::graph::PyGraph, types::repr::StructReprBuilder, utils::errors::adapt_err_value, - }, + python::{graph::graph::PyGraph, types::repr::StructReprBuilder}, }; use itertools::Itertools; /// A columnar temporal graph. @@ -21,12 +19,6 @@ use pyo3::{ }; use std::path::Path; -impl From for PyErr { - fn from(value: DiskGraphError) -> Self { - adapt_err_value(&value) - } -} - #[derive(Clone)] #[pyclass(name = "DiskGraphStorage")] pub struct PyDiskGraph { @@ -117,10 +109,7 @@ impl<'a> FromPyObject<'a> for ParquetLayerColsList<'a> { #[pymethods] impl PyGraph { /// save graph in disk_graph format and memory map the result - pub fn persist_as_disk_graph( - &self, - graph_dir: &str, - ) -> Result { + pub fn persist_as_disk_graph(&self, graph_dir: &str) -> Result { self.graph.persist_as_disk_graph(graph_dir) } } @@ -140,26 +129,25 @@ impl PyDiskGraph { } #[staticmethod] - #[pyo3(signature = (graph_dir, edge_df, src_col, dst_col, time_col))] + #[pyo3(signature = (graph_dir, edge_df, time_col, src_col, dst_col))] pub fn load_from_pandas( graph_dir: &str, edge_df: &PyAny, + time_col: &str, src_col: &str, dst_col: &str, - time_col: &str, ) -> Result { - let graph: Result = Python::with_gil(|py| { + let graph: Result = Python::with_gil(|py| { let cols_to_check = vec![src_col, dst_col, time_col]; let df_columns: Vec = edge_df.getattr("columns")?.extract()?; let df_columns: Vec<&str> = df_columns.iter().map(|x| x.as_str()).collect(); - let df = process_pandas_py_df(edge_df, py, df_columns)?; - - df.check_cols_exist(&cols_to_check)?; - let graph = Self::from_pandas(graph_dir, df, src_col, dst_col, time_col)?; + let df_view = process_pandas_py_df(edge_df, py, df_columns)?; + df_view.check_cols_exist(&cols_to_check)?; + let graph = Self::from_pandas(graph_dir, df_view, time_col, src_col, dst_col)?; - Ok::<_, PyErr>(graph) + Ok::<_, GraphError>(graph) }); graph.map_err(|e| { @@ -177,7 +165,9 @@ impl PyDiskGraph { } #[staticmethod] - #[pyo3(signature = (graph_dir, layer_parquet_cols, node_properties, chunk_size, t_props_chunk_size, read_chunk_size, concurrent_files, num_threads, node_type_col))] + #[pyo3( + signature = (graph_dir, layer_parquet_cols, node_properties, chunk_size, t_props_chunk_size, read_chunk_size, concurrent_files, num_threads, node_type_col) + )] fn load_from_parquets( graph_dir: &str, layer_parquet_cols: ParquetLayerColsList, @@ -211,7 +201,7 @@ impl PyDiskGraph { &self, other: &Self, graph_dir: &str, - ) -> Result { + ) -> Result { self.graph.merge_by_sorted_gids(&other.graph, graph_dir) } @@ -231,49 +221,52 @@ impl PyDiskGraph { impl PyDiskGraph { fn from_pandas( graph_dir: &str, - df: DFView, + df_view: DFView>>, + time: &str, src: &str, dst: &str, - time: &str, ) -> Result { - let src_col_idx = df.names.iter().position(|x| x == src).unwrap(); - let dst_col_idx = df.names.iter().position(|x| x == dst).unwrap(); - let time_col_idx = df.names.iter().position(|x| x == time).unwrap(); - - let chunk_size = df - .arrays - .first() - .map(|arr| arr.len()) - .ok_or_else(|| GraphError::LoadFailure("Empty pandas dataframe".to_owned()))?; - - let t_props_chunk_size = chunk_size; - - let names = df.names.clone(); - - let edge_lists = df - .arrays - .into_iter() - .map(|arr| { - let fields = arr + let src_index = df_view.get_index(src)?; + let dst_index = df_view.get_index(dst)?; + let time_index = df_view.get_index(time)?; + + let mut chunks_iter = df_view.chunks.peekable(); + let chunk_size = if let Some(result) = chunks_iter.peek() { + match result { + Ok(df) => df.chunk.len(), + Err(e) => { + return Err(GraphError::LoadFailure(format!( + "Failed to load graph {e:?}" + ))) + } + } + } else { + return Err(GraphError::LoadFailure("No chunks available".to_string())); + }; + + let edge_lists = chunks_iter + .map_ok(|df| { + let fields = df + .chunk .iter() - .zip(names.iter()) + .zip(df_view.names.iter()) .map(|(arr, col_name)| { Field::new(col_name, arr.data_type().clone(), arr.null_count() > 0) }) .collect_vec(); - let s_array = StructArray::new(DataType::Struct(fields), arr, None); + let s_array = StructArray::new(DataType::Struct(fields), df.chunk, None); s_array }) - .collect::>(); + .collect::, GraphError>>()?; DiskGraphStorage::load_from_edge_lists( &edge_lists, chunk_size, - t_props_chunk_size, + chunk_size, graph_dir, - src_col_idx, - dst_col_idx, - time_col_idx, + time_index, + src_index, + dst_index, ) .map_err(|err| GraphError::LoadFailure(format!("Failed to load graph {err:?}"))) } diff --git a/raphtory/src/python/graph/graph.rs b/raphtory/src/python/graph/graph.rs index 41fd11e1b1..45e464cb88 100644 --- a/raphtory/src/python/graph/graph.rs +++ b/raphtory/src/python/graph/graph.rs @@ -7,10 +7,7 @@ use crate::{ algorithms::components::LargestConnectedComponent, core::{entities::nodes::node_ref::NodeRef, utils::errors::GraphError}, db::{ - api::view::{ - internal::{CoreGraphOps, DynamicGraph, IntoDynamic, MaterializedGraph}, - serialise::{StableDecode, StableEncoder}, - }, + api::view::internal::{CoreGraphOps, DynamicGraph, IntoDynamic, MaterializedGraph}, graph::{edge::EdgeView, node::NodeView, views::node_subgraph::NodeSubgraph}, }, io::parquet_loaders::*, @@ -22,13 +19,14 @@ use crate::{ }, utils::PyTime, }, + serialise::{StableDecode, StableEncode}, }; -use pyo3::{prelude::*, types::PyBytes}; +use pyo3::prelude::*; use raphtory_api::core::{entities::GID, storage::arc_str::ArcStr}; use std::{ collections::HashMap, fmt::{Debug, Formatter}, - path::{Path, PathBuf}, + path::PathBuf, }; /// A temporal graph. @@ -38,6 +36,8 @@ pub struct PyGraph { pub graph: Graph, } +impl_serialise!(PyGraph, graph: Graph, "Graph"); + impl Debug for PyGraph { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.graph) @@ -151,26 +151,14 @@ impl PyGraph { } #[cfg(feature = "storage")] - pub fn to_disk_graph(&self, graph_dir: String) -> PyResult> { + pub fn to_disk_graph(&self, graph_dir: String) -> Result { + use crate::db::api::storage::graph::storage_ops::GraphStorage; use std::sync::Arc; - use crate::db::api::storage::storage_ops::GraphStorage; - let disk_graph = Graph::persist_as_disk_graph(&self.graph, graph_dir)?; let storage = GraphStorage::Disk(Arc::new(disk_graph)); let graph = Graph::from_internal_graph(storage); - - Python::with_gil(|py| { - Ok(Py::new( - py, - ( - Self { - graph: graph.clone(), - }, - PyGraphView::from(graph.clone()), - ), - )?) - }) + Ok(graph) } /// Adds a new node with the given id and properties to the graph. @@ -365,30 +353,6 @@ impl PyGraph { // Alternative constructors are tricky, see: https://gist.github.com/redshiftzero/648e4feeff3843ffd9924f13625f839c - /// Loads a graph from the given path. - /// - /// Arguments: - /// path (str): The path to the graph. - /// - /// Returns: - /// Graph: The loaded graph. - #[staticmethod] - #[pyo3(signature = (path, force = false))] - pub fn load_from_file(path: &str, force: bool) -> Result { - Graph::load_from_file(path, force) - } - - /// Saves the graph to the given path. - /// - /// Arguments: - /// path (str): The path to the graph. - /// - /// Returns: - /// None - pub fn save_to_file(&self, path: &str) -> Result<(), GraphError> { - self.graph.save_to_file(Path::new(path)) - } - /// Returns all the node types in the graph. /// /// Returns: @@ -397,19 +361,6 @@ impl PyGraph { self.graph.get_all_node_types() } - /// Get bincode encoded graph - pub fn bincode<'py>(&'py self, py: Python<'py>) -> Result<&'py PyBytes, GraphError> { - let bytes = MaterializedGraph::from(self.graph.clone()).bincode()?; - Ok(PyBytes::new(py, &bytes)) - } - - /// Creates a graph from a bincode encoded graph - #[staticmethod] - fn from_bincode(bytes: &[u8]) -> Result, GraphError> { - let graph = MaterializedGraph::from_bincode(bytes)?; - Ok(graph.into_events()) - } - /// Gives the large connected component of a graph. /// /// # Example Usage: @@ -427,193 +378,46 @@ impl PyGraph { PyPersistentGraph::py_from_db_graph(self.graph.persistent_graph()) } - /// Load a graph from a Pandas DataFrame. - /// - /// Args: - /// edge_df (pandas.DataFrame): The DataFrame containing the edges. - /// edge_src (str): The column name for the source node ids. - /// edge_dst (str): The column name for the destination node ids. - /// edge_time (str): The column name for the timestamps. - /// edge_properties (list): The column names for the temporal properties (optional) Defaults to None. - /// edge_const_properties (list): The column names for the constant properties (optional) Defaults to None. - /// edge_shared_const_properties (dict): A dictionary of constant properties that will be added to every edge (optional) Defaults to None. - /// edge_layer (str): The edge layer name (optional) Defaults to None. - /// layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the edge_df or if it should be used directly as the layer for all edges (optional) defaults to True. - /// node_df (pandas.DataFrame): The DataFrame containing the nodes (optional) Defaults to None. - /// node_id (str): The column name for the node ids (optional) Defaults to None. - /// node_time (str): The column name for the node timestamps (optional) Defaults to None. - /// node_properties (list): The column names for the node temporal properties (optional) Defaults to None. - /// node_const_properties (list): The column names for the node constant properties (optional) Defaults to None. - /// node_shared_const_properties (dict): A dictionary of constant properties that will be added to every node (optional) Defaults to None. - /// node_type (str): the column name for the node type - /// node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - /// - /// Returns: - /// Graph: The loaded Graph object. - #[staticmethod] - #[pyo3(signature = (edge_df, edge_src, edge_dst, edge_time, edge_properties = None, edge_const_properties = None, edge_shared_const_properties = None, - edge_layer = None, layer_in_df = true, node_df = None, node_id = None, node_time = None, node_properties = None, - node_const_properties = None, node_shared_const_properties = None, node_type = None, node_type_in_df = true))] - fn load_from_pandas( - edge_df: &PyAny, - edge_src: &str, - edge_dst: &str, - edge_time: &str, - edge_properties: Option>, - edge_const_properties: Option>, - edge_shared_const_properties: Option>, - edge_layer: Option<&str>, - layer_in_df: Option, - node_df: Option<&PyAny>, - node_id: Option<&str>, - node_time: Option<&str>, - node_properties: Option>, - node_const_properties: Option>, - node_shared_const_properties: Option>, - node_type: Option<&str>, - node_type_in_df: Option, - ) -> Result { - let graph = PyGraph { - graph: Graph::new(), - }; - if let (Some(node_df), Some(node_id), Some(node_time)) = (node_df, node_id, node_time) { - graph.load_nodes_from_pandas( - node_df, - node_id, - node_time, - node_type, - node_type_in_df, - node_properties, - node_const_properties, - node_shared_const_properties, - )?; - } - graph.load_edges_from_pandas( - edge_df, - edge_src, - edge_dst, - edge_time, - edge_properties, - edge_const_properties, - edge_shared_const_properties, - edge_layer, - layer_in_df, - )?; - Ok(graph.graph) - } - - /// Load a graph from Parquet file. - /// - /// Args: - /// edge_parquet_path (str): Parquet file or directory of Parquet files containing the edges. - /// edge_src (str): The column name for the source node ids. - /// edge_dst (str): The column name for the destination node ids. - /// edge_time (str): The column name for the timestamps. - /// edge_properties (list): The column names for the temporal properties (optional) Defaults to None. - /// edge_const_properties (list): The column names for the constant properties (optional) Defaults to None. - /// edge_shared_const_properties (dict): A dictionary of constant properties that will be added to every edge (optional) Defaults to None. - /// edge_layer (str): The edge layer name (optional) Defaults to None. - /// layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the edge_df or if it should be used directly as the layer for all edges (optional) defaults to True. - /// node_parquet_path (str): Parquet file or directory of Parquet files containing the nodes (optional) Defaults to None. - /// node_id (str): The column name for the node ids (optional) Defaults to None. - /// node_time (str): The column name for the node timestamps (optional) Defaults to None. - /// node_properties (list): The column names for the node temporal properties (optional) Defaults to None. - /// node_const_properties (list): The column names for the node constant properties (optional) Defaults to None. - /// node_shared_const_properties (dict): A dictionary of constant properties that will be added to every node (optional) Defaults to None. - /// node_type (str): the column name for the node type - /// node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - /// - /// Returns: - /// Graph: The loaded Graph object. - #[staticmethod] - #[pyo3(signature = (edge_parquet_path, edge_src, edge_dst, edge_time, edge_properties = None, edge_const_properties = None, edge_shared_const_properties = None, - edge_layer = None, layer_in_df = true, node_parquet_path = None, node_id = None, node_time = None, node_properties = None, - node_const_properties = None, node_shared_const_properties = None, node_type = None, node_type_in_df = true))] - fn load_from_parquet( - edge_parquet_path: PathBuf, - edge_src: &str, - edge_dst: &str, - edge_time: &str, - edge_properties: Option>, - edge_const_properties: Option>, - edge_shared_const_properties: Option>, - edge_layer: Option<&str>, - layer_in_df: Option, - node_parquet_path: Option, - node_id: Option<&str>, - node_time: Option<&str>, - node_properties: Option>, - node_const_properties: Option>, - node_shared_const_properties: Option>, - node_type: Option<&str>, - node_type_in_df: Option, - ) -> Result { - let graph = PyGraph { - graph: Graph::new(), - }; - if let (Some(node_parquet_path), Some(node_id), Some(node_time)) = - (node_parquet_path, node_id, node_time) - { - graph.load_nodes_from_parquet( - node_parquet_path, - node_id, - node_time, - node_type, - node_type_in_df, - node_properties, - node_const_properties, - node_shared_const_properties, - )?; - } - graph.load_edges_from_parquet( - edge_parquet_path, - edge_src, - edge_dst, - edge_time, - edge_properties, - edge_const_properties, - edge_shared_const_properties, - edge_layer, - layer_in_df, - )?; - Ok(graph.graph) - } - /// Load nodes from a Pandas DataFrame into the graph. /// /// Arguments: /// df (pandas.DataFrame): The Pandas DataFrame containing the nodes. - /// id (str): The column name for the node IDs. /// time (str): The column name for the timestamps. - /// node_type (str): the column name for the node type - /// node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - /// properties (List): List of node property column names. Defaults to None. (optional) - /// const_properties (List): List of constant node property column names. Defaults to None. (optional) - /// shared_const_properties (Dictionary/Hashmap of properties): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + /// id (str): The column name for the node IDs. + /// node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + /// node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + /// properties (List[str]): List of node property column names. Defaults to None. (optional) + /// constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (df, id, time, node_type = None, node_type_in_df = true, properties = None, const_properties = None, shared_const_properties = None))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3( + signature = (df,time, id, node_type = None, node_type_col = None, properties = None, constant_properties = None, shared_constant_properties = None) + )] fn load_nodes_from_pandas( &self, df: &PyAny, - id: &str, time: &str, + id: &str, node_type: Option<&str>, - node_type_in_df: Option, + node_type_col: Option<&str>, properties: Option>, - const_properties: Option>, - shared_const_properties: Option>, + constant_properties: Option>, + shared_constant_properties: Option>, ) -> Result<(), GraphError> { load_nodes_from_pandas( self.graph.core_graph(), df, - id, time, + id, node_type, - node_type_in_df, - properties, - const_properties, - shared_const_properties, + node_type_col, + properties.as_ref().map(|props| props.as_ref()), + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), ) } @@ -621,37 +425,42 @@ impl PyGraph { /// /// Arguments: /// parquet_path (str): Parquet file or directory of Parquet files containing the nodes - /// id (str): The column name for the node IDs. /// time (str): The column name for the timestamps. - /// node_type (str): the column name for the node type - /// node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - /// properties (List): List of node property column names. Defaults to None. (optional) - /// const_properties (List): List of constant node property column names. Defaults to None. (optional) - /// shared_const_properties (Dictionary/Hashmap of properties): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + /// id (str): The column name for the node IDs. + /// node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + /// node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + /// properties (List[str]): List of node property column names. Defaults to None. (optional) + /// constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (parquet_path, id, time, node_type = None, node_type_in_df = true, properties = None, const_properties = None, shared_const_properties = None))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3( + signature = (parquet_path, time, id, node_type = None, node_type_col = None, properties = None, constant_properties = None, shared_constant_properties = None) + )] fn load_nodes_from_parquet( &self, parquet_path: PathBuf, - id: &str, time: &str, + id: &str, node_type: Option<&str>, - node_type_in_df: Option, + node_type_col: Option<&str>, properties: Option>, - const_properties: Option>, - shared_const_properties: Option>, + constant_properties: Option>, + shared_constant_properties: Option>, ) -> Result<(), GraphError> { load_nodes_from_parquet( &self.graph, parquet_path.as_path(), - id, time, + id, node_type, - node_type_in_df, - properties, - const_properties, - shared_const_properties, + node_type_col, + properties.as_ref().map(|props| props.as_ref()), + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), ) } @@ -659,41 +468,45 @@ impl PyGraph { /// /// Arguments: /// df (Dataframe): The Pandas DataFrame containing the edges. + /// time (str): The column name for the update timestamps. /// src (str): The column name for the source node ids. /// dst (str): The column name for the destination node ids. - /// time (str): The column name for the update timestamps. - /// properties (List): List of edge property column names. Defaults to None. (optional) - /// const_properties (List): List of constant edge property column names. Defaults to None. (optional) - /// shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - /// layer (str): The edge layer name (optional) Defaults to None. - /// layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the dateframe or if it should be used directly as the layer for all edges (optional) defaults to True. - /// + /// properties (List[str]): List of edge property column names. Defaults to None. (optional) + /// constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + /// layer (str): A constant value to use as the layer for all edges (optional) Defaults to None. (cannot be used in combination with layer_col) + /// layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. (cannot be used in combination with layer) /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (df, src, dst, time, properties = None, const_properties = None, shared_const_properties = None, layer = None, layer_in_df = true))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3( + signature = (df, time, src, dst, properties = None, constant_properties = None, shared_constant_properties = None, layer = None, layer_col = None) + )] fn load_edges_from_pandas( &self, df: &PyAny, + time: &str, src: &str, dst: &str, - time: &str, properties: Option>, - const_properties: Option>, - shared_const_properties: Option>, + constant_properties: Option>, + shared_constant_properties: Option>, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { load_edges_from_pandas( self.graph.core_graph(), df, + time, src, dst, - time, - properties, - const_properties, - shared_const_properties, + properties.as_ref().map(|props| props.as_ref()), + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), layer, - layer_in_df, + layer_col, ) } @@ -701,41 +514,45 @@ impl PyGraph { /// /// Arguments: /// parquet_path (str): Parquet file or directory of Parquet files path containing edges + /// time (str): The column name for the update timestamps. /// src (str): The column name for the source node ids. /// dst (str): The column name for the destination node ids. - /// time (str): The column name for the update timestamps. - /// properties (List): List of edge property column names. Defaults to None. (optional) - /// const_properties (List): List of constant edge property column names. Defaults to None. (optional) - /// shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - /// layer (str): The edge layer name (optional) Defaults to None. - /// layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the dataframe or if it should be used directly as the layer for all edges (optional) defaults to True. - /// + /// properties (List[str]): List of edge property column names. Defaults to None. (optional) + /// constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + /// layer (str): A constant value to use as the layer for all edges (optional) Defaults to None. (cannot be used in combination with layer_col) + /// layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. (cannot be used in combination with layer) /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (parquet_path, src, dst, time, properties = None, const_properties = None, shared_const_properties = None, layer = None, layer_in_df = true))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3( + signature = (parquet_path, time, src, dst, properties = None, constant_properties = None, shared_constant_properties = None, layer = None, layer_col = None) + )] fn load_edges_from_parquet( &self, parquet_path: PathBuf, + time: &str, src: &str, dst: &str, - time: &str, properties: Option>, - const_properties: Option>, - shared_const_properties: Option>, + constant_properties: Option>, + shared_constant_properties: Option>, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { load_edges_from_parquet( &self.graph, parquet_path.as_path(), + time, src, dst, - time, - properties, - const_properties, - shared_const_properties, + properties.as_ref().map(|props| props.as_ref()), + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), layer, - layer_in_df, + layer_col, ) } @@ -744,25 +561,34 @@ impl PyGraph { /// Arguments: /// df (Dataframe): The Pandas DataFrame containing node information. /// id(str): The column name for the node IDs. - /// const_properties (List): List of constant node property column names. Defaults to None. (optional) - /// shared_const_properties (>): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + /// node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + /// node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + /// constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) /// /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (df, id, const_properties = None, shared_const_properties = None))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3(signature = (df, id, node_type=None, node_type_col=None, constant_properties = None, shared_constant_properties = None))] fn load_node_props_from_pandas( &self, df: &PyAny, id: &str, - const_properties: Option>, - shared_const_properties: Option>, + node_type: Option<&str>, + node_type_col: Option<&str>, + constant_properties: Option>, + shared_constant_properties: Option>, ) -> Result<(), GraphError> { load_node_props_from_pandas( self.graph.core_graph(), df, id, - const_properties, - shared_const_properties, + node_type, + node_type_col, + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), ) } @@ -771,25 +597,34 @@ impl PyGraph { /// Arguments: /// parquet_path (str): Parquet file or directory of Parquet files path containing node information. /// id(str): The column name for the node IDs. - /// const_properties (List): List of constant node property column names. Defaults to None. (optional) - /// shared_const_properties (>): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + /// node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + /// node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + /// constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) /// /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (parquet_path, id, const_properties = None, shared_const_properties = None))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3(signature = (parquet_path, id, node_type=None,node_type_col=None, constant_properties = None, shared_constant_properties = None))] fn load_node_props_from_parquet( &self, parquet_path: PathBuf, id: &str, - const_properties: Option>, - shared_const_properties: Option>, + node_type: Option<&str>, + node_type_col: Option<&str>, + constant_properties: Option>, + shared_constant_properties: Option>, ) -> Result<(), GraphError> { load_node_props_from_parquet( &self.graph, parquet_path.as_path(), id, - const_properties, - shared_const_properties, + node_type, + node_type_col, + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), ) } @@ -799,33 +634,38 @@ impl PyGraph { /// df (Dataframe): The Pandas DataFrame containing edge information. /// src (str): The column name for the source node. /// dst (str): The column name for the destination node. - /// const_properties (List): List of constant edge property column names. Defaults to None. (optional) - /// shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - /// layer (str): Layer name. Defaults to None. (optional) - /// layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the data frame or if it should be used directly as the layer for all edges (optional) defaults to True. + /// constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + /// layer (str): The edge layer name (optional) Defaults to None. + /// layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. /// /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (df, src, dst, const_properties = None, shared_const_properties = None, layer = None, layer_in_df = true))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3( + signature = (df, src, dst, constant_properties = None, shared_constant_properties = None, layer = None, layer_col = None) + )] fn load_edge_props_from_pandas( &self, df: &PyAny, src: &str, dst: &str, - const_properties: Option>, - shared_const_properties: Option>, + constant_properties: Option>, + shared_constant_properties: Option>, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { load_edge_props_from_pandas( self.graph.core_graph(), df, src, dst, - const_properties, - shared_const_properties, + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), layer, - layer_in_df, + layer_col, ) } @@ -835,33 +675,38 @@ impl PyGraph { /// parquet_path (str): Parquet file or directory of Parquet files path containing edge information. /// src (str): The column name for the source node. /// dst (str): The column name for the destination node. - /// const_properties (List): List of constant edge property column names. Defaults to None. (optional) - /// shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - /// layer (str): Layer name. Defaults to None. (optional) - /// layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the data frame or if it should be used directly as the layer for all edges (optional) defaults to True. + /// constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + /// layer (str): The edge layer name (optional) Defaults to None. + /// layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. /// /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (parquet_path, src, dst, const_properties = None, shared_const_properties = None, layer = None, layer_in_df = true))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3( + signature = (parquet_path, src, dst, constant_properties = None, shared_constant_properties = None, layer = None, layer_col = None) + )] fn load_edge_props_from_parquet( &self, parquet_path: PathBuf, src: &str, dst: &str, - const_properties: Option>, - shared_const_properties: Option>, + constant_properties: Option>, + shared_constant_properties: Option>, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { load_edge_props_from_parquet( &self.graph, parquet_path.as_path(), src, dst, - const_properties, - shared_const_properties, + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), layer, - layer_in_df, + layer_col, ) } } diff --git a/raphtory/src/python/graph/graph_with_deletions.rs b/raphtory/src/python/graph/graph_with_deletions.rs index 4ad7f2842c..3b8bc955d1 100644 --- a/raphtory/src/python/graph/graph_with_deletions.rs +++ b/raphtory/src/python/graph/graph_with_deletions.rs @@ -10,10 +10,7 @@ use crate::{ db::{ api::{ mutation::{AdditionOps, PropertyAdditionOps}, - view::{ - internal::{CoreGraphOps, MaterializedGraph}, - serialise::StableEncoder, - }, + view::internal::CoreGraphOps, }, graph::{edge::EdgeView, node::NodeView, views::deletion_graph::PersistentGraph}, }, @@ -23,19 +20,19 @@ use crate::{ utils::PyTime, }, }; -use pyo3::{prelude::*, types::PyBytes}; +use pyo3::prelude::*; use raphtory_api::core::{entities::GID, storage::arc_str::ArcStr}; use std::{ collections::HashMap, fmt::{Debug, Formatter}, - path::{Path, PathBuf}, + path::PathBuf, }; use super::{ graph::{PyGraph, PyGraphEncoder}, io::pandas_loaders::*, }; -use crate::io::parquet_loaders::*; +use crate::{io::parquet_loaders::*, serialise::StableEncode}; /// A temporal graph that allows edges and nodes to be deleted. #[derive(Clone)] @@ -44,6 +41,8 @@ pub struct PyPersistentGraph { pub(crate) graph: PersistentGraph, } +impl_serialise!(PyPersistentGraph, graph: PersistentGraph, "PersistentGraph"); + impl Debug for PyPersistentGraph { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.graph) @@ -209,14 +208,14 @@ impl PyPersistentGraph { /// layer (str): The layer of the edge. (optional) /// /// Returns: - /// None or a GraphError if the edge could not be deleted + /// The deleted edge pub fn delete_edge( &self, timestamp: PyTime, src: GID, dst: GID, layer: Option<&str>, - ) -> Result<(), GraphError> { + ) -> Result, GraphError> { self.graph.delete_edge(timestamp, src, dst, layer) } @@ -327,31 +326,6 @@ impl PyPersistentGraph { // Alternative constructors are tricky, see: https://gist.github.com/redshiftzero/648e4feeff3843ffd9924f13625f839c - /// Loads a graph from the given path. - /// - /// Arguments: - /// path (str): The path to the graph. - /// - /// Returns: - /// Graph: The loaded graph. - #[staticmethod] - #[pyo3(signature = (path, force = false))] - pub fn load_from_file(path: &str, force: bool) -> Result { - let file_path: PathBuf = [env!("CARGO_MANIFEST_DIR"), path].iter().collect(); - PersistentGraph::load_from_file(file_path, force) - } - - /// Saves the graph to the given path. - /// - /// Arguments: - /// path (str): The path to the graph. - /// - /// Returns: - /// None - pub fn save_to_file(&self, path: &str) -> Result<(), GraphError> { - self.graph.save_to_file(Path::new(path)) - } - /// Returns all the node types in the graph. /// /// Returns: @@ -360,211 +334,49 @@ impl PyPersistentGraph { self.graph.get_all_node_types() } - /// Get bincode encoded graph - pub fn bincode<'py>(&'py self, py: Python<'py>) -> Result<&'py PyBytes, GraphError> { - let bytes = MaterializedGraph::from(self.graph.clone()).bincode()?; - Ok(PyBytes::new(py, &bytes)) - } - - /// Creates a graph from a bincode encoded graph - #[staticmethod] - fn from_bincode(bytes: &[u8]) -> Result, GraphError> { - let graph = MaterializedGraph::from_bincode(bytes)?; - Ok(graph.into_persistent()) - } - /// Get event graph pub fn event_graph<'py>(&'py self) -> PyResult> { PyGraph::py_from_db_graph(self.graph.event_graph()) } - /// Load a graph from a Pandas DataFrame. - /// - /// Args: - /// edge_df (pandas.DataFrame): The DataFrame containing the edges. - /// edge_src (str): The column name for the source node ids. - /// edge_dst (str): The column name for the destination node ids. - /// edge_time (str): The column name for the timestamps. - /// edge_properties (list): The column names for the temporal properties (optional) Defaults to None. - /// edge_const_properties (list): The column names for the constant properties (optional) Defaults to None. - /// edge_shared_const_properties (dict): A dictionary of constant properties that will be added to every edge (optional) Defaults to None. - /// edge_layer (str): The edge layer name (optional) Defaults to None. - /// layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the edge_df or if it should be used directly as the layer for all edges (optional) defaults to True. - /// node_df (pandas.DataFrame): The DataFrame containing the nodes (optional) Defaults to None. - /// node_id (str): The column name for the node ids (optional) Defaults to None. - /// node_time (str): The column name for the node timestamps (optional) Defaults to None. - /// node_properties (list): The column names for the node temporal properties (optional) Defaults to None. - /// node_const_properties (list): The column names for the node constant properties (optional) Defaults to None. - /// node_shared_const_properties (dict): A dictionary of constant properties that will be added to every node (optional) Defaults to None. - /// node_type (str): the column name for the node type - /// node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - /// - /// Returns: - /// Graph: The loaded Graph object. - #[staticmethod] - #[pyo3(signature = (edge_df, edge_src, edge_dst, edge_time, edge_properties = None, edge_const_properties = None, edge_shared_const_properties = None, - edge_layer = None, layer_in_df = true, node_df = None, node_id = None, node_time = None, node_properties = None, - node_const_properties = None, node_shared_const_properties = None, node_type = None, node_type_in_df = true))] - fn load_from_pandas( - edge_df: &PyAny, - edge_src: &str, - edge_dst: &str, - edge_time: &str, - edge_properties: Option>, - edge_const_properties: Option>, - edge_shared_const_properties: Option>, - edge_layer: Option<&str>, - layer_in_df: Option, - node_df: Option<&PyAny>, - node_id: Option<&str>, - node_time: Option<&str>, - node_properties: Option>, - node_const_properties: Option>, - node_shared_const_properties: Option>, - node_type: Option<&str>, - node_type_in_df: Option, - ) -> Result { - let graph = PyPersistentGraph { - graph: PersistentGraph::new(), - }; - graph.load_edges_from_pandas( - edge_df, - edge_src, - edge_dst, - edge_time, - edge_properties, - edge_const_properties, - edge_shared_const_properties, - edge_layer, - layer_in_df, - )?; - if let (Some(node_df), Some(node_id), Some(node_time)) = (node_df, node_id, node_time) { - graph.load_nodes_from_pandas( - node_df, - node_id, - node_time, - node_type, - node_type_in_df, - node_properties, - node_const_properties, - node_shared_const_properties, - )?; - } - Ok(graph.graph) - } - - /// Load a graph from Parquet file. - /// - /// Args: - /// edge_parquet_path (str): Parquet file or directory of Parquet files containing the edges. - /// edge_src (str): The column name for the source node ids. - /// edge_dst (str): The column name for the destination node ids. - /// edge_time (str): The column name for the timestamps. - /// edge_properties (list): The column names for the temporal properties (optional) Defaults to None. - /// edge_const_properties (list): The column names for the constant properties (optional) Defaults to None. - /// edge_shared_const_properties (dict): A dictionary of constant properties that will be added to every edge (optional) Defaults to None. - /// edge_layer (str): The edge layer name (optional) Defaults to None. - /// layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the edge_df or if it should be used directly as the layer for all edges (optional) defaults to True. - /// node_parquet_path (str): Parquet file or directory of Parquet files containing the nodes (optional) Defaults to None. - /// node_id (str): The column name for the node ids (optional) Defaults to None. - /// node_time (str): The column name for the node timestamps (optional) Defaults to None. - /// node_properties (list): The column names for the node temporal properties (optional) Defaults to None. - /// node_const_properties (list): The column names for the node constant properties (optional) Defaults to None. - /// node_shared_const_properties (dict): A dictionary of constant properties that will be added to every node (optional) Defaults to None. - /// node_type (str): the column name for the node type - /// node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - /// - /// Returns: - /// Graph: The loaded Graph object. - #[staticmethod] - #[pyo3(signature = (edge_parquet_path, edge_src, edge_dst, edge_time, edge_properties = None, edge_const_properties = None, edge_shared_const_properties = None, - edge_layer = None, layer_in_df = true, node_parquet_path = None, node_id = None, node_time = None, node_properties = None, - node_const_properties = None, node_shared_const_properties = None, node_type = None, node_type_in_df = true))] - fn load_from_parquet( - edge_parquet_path: PathBuf, - edge_src: &str, - edge_dst: &str, - edge_time: &str, - edge_properties: Option>, - edge_const_properties: Option>, - edge_shared_const_properties: Option>, - edge_layer: Option<&str>, - layer_in_df: Option, - node_parquet_path: Option, - node_id: Option<&str>, - node_time: Option<&str>, - node_properties: Option>, - node_const_properties: Option>, - node_shared_const_properties: Option>, - node_type: Option<&str>, - node_type_in_df: Option, - ) -> Result { - let graph = PyPersistentGraph { - graph: PersistentGraph::new(), - }; - if let (Some(node_parquet_file_path), Some(node_id), Some(node_time)) = - (node_parquet_path, node_id, node_time) - { - graph.load_nodes_from_parquet( - node_parquet_file_path, - node_id, - node_time, - node_type, - node_type_in_df, - node_properties, - node_const_properties, - node_shared_const_properties, - )?; - } - graph.load_edges_from_parquet( - edge_parquet_path, - edge_src, - edge_dst, - edge_time, - edge_properties, - edge_const_properties, - edge_shared_const_properties, - edge_layer, - layer_in_df, - )?; - Ok(graph.graph) - } - /// Load nodes from a Pandas DataFrame into the graph. /// /// Arguments: /// df (pandas.DataFrame): The Pandas DataFrame containing the nodes. - /// id (str): The column name for the node IDs. /// time (str): The column name for the timestamps. - /// node_type (str): the column name for the node type - /// node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - /// properties (List): List of node property column names. Defaults to None. (optional) - /// const_properties (List): List of constant node property column names. Defaults to None. (optional) - /// shared_const_properties (Dictionary/Hashmap of properties): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + /// id (str): The column name for the node IDs. + /// node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + /// node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + /// properties (List[str]): List of node property column names. Defaults to None. (optional) + /// constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (df, id, time, node_type = None, node_type_in_df = true, properties = None, const_properties = None, shared_const_properties = None))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3(signature = (df,time,id, node_type = None, node_type_col = None, properties = None, constant_properties = None, shared_constant_properties = None))] fn load_nodes_from_pandas( &self, df: &PyAny, - id: &str, time: &str, + id: &str, node_type: Option<&str>, - node_type_in_df: Option, + node_type_col: Option<&str>, properties: Option>, - const_properties: Option>, - shared_const_properties: Option>, + constant_properties: Option>, + shared_constant_properties: Option>, ) -> Result<(), GraphError> { load_nodes_from_pandas( - &self.graph.0, + self.graph.core_graph(), df, - id, time, + id, node_type, - node_type_in_df, - properties, - const_properties, - shared_const_properties, + node_type_col, + properties.as_ref().map(|props| props.as_ref()), + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), ) } @@ -572,37 +384,40 @@ impl PyPersistentGraph { /// /// Arguments: /// parquet_path (str): Parquet file or directory of Parquet files containing the nodes - /// id (str): The column name for the node IDs. /// time (str): The column name for the timestamps. - /// node_type (str): the column name for the node type - /// node_type_in_df (bool): whether the node type should be used to look up the values in a column of the df or if it should be used directly as the node type - /// properties (List): List of node property column names. Defaults to None. (optional) - /// const_properties (List): List of constant node property column names. Defaults to None. (optional) - /// shared_const_properties (Dictionary/Hashmap of properties): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + /// id (str): The column name for the node IDs. + /// node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + /// node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + /// properties (List[str]): List of node property column names. Defaults to None. (optional) + /// constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (parquet_path, id, time, node_type = None, node_type_in_df = true, properties = None, const_properties = None, shared_const_properties = None))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3(signature = (parquet_path, time,id, node_type = None, node_type_col = None, properties = None, constant_properties = None, shared_constant_properties = None))] fn load_nodes_from_parquet( &self, parquet_path: PathBuf, - id: &str, time: &str, + id: &str, node_type: Option<&str>, - node_type_in_df: Option, + node_type_col: Option<&str>, properties: Option>, - const_properties: Option>, - shared_const_properties: Option>, + constant_properties: Option>, + shared_constant_properties: Option>, ) -> Result<(), GraphError> { load_nodes_from_parquet( &self.graph, parquet_path.as_path(), - id, time, + id, node_type, - node_type_in_df, - properties, - const_properties, - shared_const_properties, + node_type_col, + properties.as_ref().map(|props| props.as_ref()), + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), ) } @@ -610,41 +425,43 @@ impl PyPersistentGraph { /// /// Arguments: /// df (Dataframe): The Pandas DataFrame containing the edges. + /// time (str): The column name for the update timestamps. /// src (str): The column name for the source node ids. /// dst (str): The column name for the destination node ids. - /// time (str): The column name for the update timestamps. - /// properties (List): List of edge property column names. Defaults to None. (optional) - /// const_properties (List): List of constant edge property column names. Defaults to None. (optional) - /// shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - /// layer (str): The edge layer name (optional) Defaults to None. - /// layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the dataframe or if it should be used directly as the layer for all edges (optional) defaults to True. - /// + /// properties (List[str]): List of edge property column names. Defaults to None. (optional) + /// constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + /// layer (str): A constant value to use as the layer for all edges (optional) Defaults to None. (cannot be used in combination with layer_col) + /// layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. (cannot be used in combination with layer) /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (df, src, dst, time, properties = None, const_properties = None, shared_const_properties = None, layer = None, layer_in_df = true))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3(signature = (df, time, src, dst, properties = None, constant_properties = None, shared_constant_properties = None, layer = None, layer_col = None))] fn load_edges_from_pandas( &self, df: &PyAny, + time: &str, src: &str, dst: &str, - time: &str, properties: Option>, - const_properties: Option>, - shared_const_properties: Option>, + constant_properties: Option>, + shared_constant_properties: Option>, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { load_edges_from_pandas( - &self.graph.0, + self.graph.core_graph(), df, + time, src, dst, - time, - properties, - const_properties, - shared_const_properties, + properties.as_ref().map(|props| props.as_ref()), + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), layer, - layer_in_df, + layer_col, ) } @@ -652,41 +469,43 @@ impl PyPersistentGraph { /// /// Arguments: /// parquet_path (str): Parquet file or directory of Parquet files path containing edges + /// time (str): The column name for the update timestamps. /// src (str): The column name for the source node ids. /// dst (str): The column name for the destination node ids. - /// time (str): The column name for the update timestamps. - /// properties (List): List of edge property column names. Defaults to None. (optional) - /// const_properties (List): List of constant edge property column names. Defaults to None. (optional) - /// shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - /// layer (str): The edge layer name (optional) Defaults to None. - /// layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the dataframe or if it should be used directly as the layer for all edges (optional) defaults to True. - /// + /// properties (List[str]): List of edge property column names. Defaults to None. (optional) + /// constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + /// layer (str): A constant value to use as the layer for all edges (optional) Defaults to None. (cannot be used in combination with layer_col) + /// layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. (cannot be used in combination with layer) /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (parquet_path, src, dst, time, properties = None, const_properties = None, shared_const_properties = None, layer = None, layer_in_df = true))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3(signature = (parquet_path, time, src, dst, properties = None, constant_properties = None, shared_constant_properties = None, layer = None, layer_col = None))] fn load_edges_from_parquet( &self, parquet_path: PathBuf, + time: &str, src: &str, dst: &str, - time: &str, properties: Option>, - const_properties: Option>, - shared_const_properties: Option>, + constant_properties: Option>, + shared_constant_properties: Option>, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { load_edges_from_parquet( &self.graph, parquet_path.as_path(), + time, src, dst, - time, - properties, - const_properties, - shared_const_properties, + properties.as_ref().map(|props| props.as_ref()), + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), layer, - layer_in_df, + layer_col, ) } @@ -694,25 +513,35 @@ impl PyPersistentGraph { /// /// Arguments: /// df (Dataframe): The Pandas DataFrame containing the edges. + /// time (str): The column name for the update timestamps. /// src (str): The column name for the source node ids. /// dst (str): The column name for the destination node ids. - /// time (str): The column name for the update timestamps. - /// layer (str): The edge layer name (optional) Defaults to None. - /// layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the dataframe or if it should be used directly as the layer for all edges (optional) defaults to True. - /// + /// layer (str): A constant value to use as the layer for all edges (optional) Defaults to None. (cannot be used in combination with layer_col) + /// layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. (cannot be used in combination with layer) /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (df, src, dst, time, layer = None, layer_in_df = true))] - fn load_edges_deletions_from_pandas( + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3(signature = (df, time, src, dst, layer = None, layer_col = None))] + fn load_edge_deletions_from_pandas( &self, df: &PyAny, + time: &str, src: &str, dst: &str, - time: &str, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { - load_edges_deletions_from_pandas(&self.graph.0, df, src, dst, time, layer, layer_in_df) + load_edge_deletions_from_pandas( + self.graph.core_graph(), + df, + time, + src, + dst, + layer, + layer_col, + ) } /// Load edges deletions from a Parquet file into the graph. @@ -722,29 +551,31 @@ impl PyPersistentGraph { /// src (str): The column name for the source node ids. /// dst (str): The column name for the destination node ids. /// time (str): The column name for the update timestamps. - /// layer (str): The edge layer name (optional) Defaults to None. - /// layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the dataframe or if it should be used directly as the layer for all edges (optional) defaults to True. - /// + /// layer (str): A constant value to use as the layer for all edges (optional) Defaults to None. (cannot be used in combination with layer_col) + /// layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. (cannot be used in combination with layer) /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (parquet_path, src, dst, time, layer = None, layer_in_df = true))] - fn load_edges_deletions_from_parquet( + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3(signature = (parquet_path, time, src, dst, layer = None, layer_col = None))] + fn load_edge_deletions_from_parquet( &self, parquet_path: PathBuf, + time: &str, src: &str, dst: &str, - time: &str, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { - load_edges_deletions_from_parquet( + load_edge_deletions_from_parquet( &self.graph, parquet_path.as_path(), + time, src, dst, - time, layer, - layer_in_df, + layer_col, ) } @@ -753,25 +584,34 @@ impl PyPersistentGraph { /// Arguments: /// df (Dataframe): The Pandas DataFrame containing node information. /// id(str): The column name for the node IDs. - /// const_properties (List): List of constant node property column names. Defaults to None. (optional) - /// shared_const_properties (>): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + /// node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + /// node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + /// constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) /// /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (df, id, const_properties = None, shared_const_properties = None))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3(signature = (df, id, node_type=None, node_type_col=None, constant_properties = None, shared_constant_properties = None))] fn load_node_props_from_pandas( &self, df: &PyAny, id: &str, - const_properties: Option>, - shared_const_properties: Option>, + node_type: Option<&str>, + node_type_col: Option<&str>, + constant_properties: Option>, + shared_constant_properties: Option>, ) -> Result<(), GraphError> { load_node_props_from_pandas( - &self.graph.0, + self.graph.core_graph(), df, id, - const_properties, - shared_const_properties, + node_type, + node_type_col, + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), ) } @@ -780,25 +620,34 @@ impl PyPersistentGraph { /// Arguments: /// parquet_path (str): Parquet file or directory of Parquet files path containing node information. /// id(str): The column name for the node IDs. - /// const_properties (List): List of constant node property column names. Defaults to None. (optional) - /// shared_const_properties (>): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) + /// node_type (str): A constant value to use as the node type for all nodes (optional). Defaults to None. (cannot be used in combination with node_type_col) + /// node_type_col (str): The node type col name in dataframe (optional) Defaults to None. (cannot be used in combination with node_type) + /// constant_properties (List[str]): List of constant node property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every node. Defaults to None. (optional) /// /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (parquet_path, id, const_properties = None, shared_const_properties = None))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3(signature = (parquet_path, id, node_type = None, node_type_col=None, constant_properties = None, shared_constant_properties = None))] fn load_node_props_from_parquet( &self, parquet_path: PathBuf, id: &str, - const_properties: Option>, - shared_const_properties: Option>, + node_type: Option<&str>, + node_type_col: Option<&str>, + constant_properties: Option>, + shared_constant_properties: Option>, ) -> Result<(), GraphError> { load_node_props_from_parquet( &self.graph, parquet_path.as_path(), id, - const_properties, - shared_const_properties, + node_type, + node_type_col, + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), ) } @@ -808,33 +657,36 @@ impl PyPersistentGraph { /// df (Dataframe): The Pandas DataFrame containing edge information. /// src (str): The column name for the source node. /// dst (str): The column name for the destination node. - /// const_properties (List): List of constant edge property column names. Defaults to None. (optional) - /// shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - /// layer (str): Layer name. Defaults to None. (optional) - /// layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the data frame or if it should be used directly as the layer for all edges (optional) defaults to True. + /// constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + /// layer (str): The edge layer name (optional) Defaults to None. + /// layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. /// /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (df, src, dst, const_properties = None, shared_const_properties = None, layer = None, layer_in_df = true))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3(signature = (df, src, dst, constant_properties = None, shared_constant_properties = None, layer = None, layer_col = None))] fn load_edge_props_from_pandas( &self, df: &PyAny, src: &str, dst: &str, - const_properties: Option>, - shared_const_properties: Option>, + constant_properties: Option>, + shared_constant_properties: Option>, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { load_edge_props_from_pandas( - &self.graph.0, + self.graph.core_graph(), df, src, dst, - const_properties, - shared_const_properties, + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), layer, - layer_in_df, + layer_col, ) } @@ -844,33 +696,36 @@ impl PyPersistentGraph { /// parquet_path (str): Parquet file or directory of Parquet files path containing edge information. /// src (str): The column name for the source node. /// dst (str): The column name for the destination node. - /// const_properties (List): List of constant edge property column names. Defaults to None. (optional) - /// shared_const_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) - /// layer (str): Layer name. Defaults to None. (optional) - /// layer_in_df (bool): Whether the layer name should be used to look up the values in a column of the data frame or if it should be used directly as the layer for all edges (optional) defaults to True. + /// constant_properties (List[str]): List of constant edge property column names. Defaults to None. (optional) + /// shared_constant_properties (dict): A dictionary of constant properties that will be added to every edge. Defaults to None. (optional) + /// layer (str): The edge layer name (optional) Defaults to None. + /// layer_col (str): The edge layer col name in dataframe (optional) Defaults to None. /// /// Returns: - /// Result<(), GraphError>: Result of the operation. - #[pyo3(signature = (parquet_path, src, dst, const_properties = None, shared_const_properties = None, layer = None, layer_in_df = true))] + /// None: If the operation is successful. + /// + /// Raises: + /// GraphError: If the operation fails. + #[pyo3(signature = (parquet_path, src, dst, constant_properties = None, shared_constant_properties = None, layer = None, layer_col = None))] fn load_edge_props_from_parquet( &self, parquet_path: PathBuf, src: &str, dst: &str, - const_properties: Option>, - shared_const_properties: Option>, + constant_properties: Option>, + shared_constant_properties: Option>, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { load_edge_props_from_parquet( &self.graph, parquet_path.as_path(), src, dst, - const_properties, - shared_const_properties, + constant_properties.as_ref().map(|props| props.as_ref()), + shared_constant_properties.as_ref(), layer, - layer_in_df, + layer_col, ) } } diff --git a/raphtory/src/python/graph/io/pandas_loaders.rs b/raphtory/src/python/graph/io/pandas_loaders.rs index b211de46a0..38c37af890 100644 --- a/raphtory/src/python/graph/io/pandas_loaders.rs +++ b/raphtory/src/python/graph/io/pandas_loaders.rs @@ -1,55 +1,47 @@ use crate::{ core::{utils::errors::GraphError, Prop}, - db::api::{storage::storage_ops::GraphStorage, view::internal::CoreGraphOps}, + db::api::{storage::graph::storage_ops::GraphStorage, view::internal::CoreGraphOps}, io::arrow::{dataframe::*, df_loaders::*}, python::graph::io::*, }; use polars_arrow::{array::Array, ffi}; -use pyo3::{ffi::Py_uintptr_t, prelude::*, types::IntoPyDict}; +use pyo3::{ + ffi::Py_uintptr_t, + prelude::*, + types::{IntoPyDict, PyDict}, +}; use std::collections::HashMap; pub fn load_nodes_from_pandas( graph: &GraphStorage, df: &PyAny, - id: &str, time: &str, + id: &str, node_type: Option<&str>, - node_type_in_df: Option, - properties: Option>, - const_properties: Option>, - shared_const_properties: Option>, + node_type_col: Option<&str>, + properties: Option<&[&str]>, + constant_properties: Option<&[&str]>, + shared_constant_properties: Option<&HashMap>, ) -> Result<(), GraphError> { Python::with_gil(|py| { - let size: usize = py - .eval( - "index.__len__()", - Some([("index", df.getattr("index")?)].into_py_dict(py)), - None, - )? - .extract()?; - let mut cols_to_check = vec![id, time]; - cols_to_check.extend(properties.as_ref().unwrap_or(&Vec::new())); - cols_to_check.extend(const_properties.as_ref().unwrap_or(&Vec::new())); - if node_type_in_df.unwrap_or(true) { - if let Some(ref node_type) = node_type { - cols_to_check.push(node_type.as_ref()); - } + cols_to_check.extend(properties.unwrap_or(&Vec::new())); + cols_to_check.extend(constant_properties.unwrap_or(&Vec::new())); + if let Some(ref node_type_col) = node_type_col { + cols_to_check.push(node_type_col.as_ref()); } - let df = process_pandas_py_df(df, py, cols_to_check.clone())?; - df.check_cols_exist(&cols_to_check)?; - + let df_view = process_pandas_py_df(df, py, cols_to_check.clone())?; + df_view.check_cols_exist(&cols_to_check)?; load_nodes_from_df( - &df, - size, - id, + df_view, time, + id, properties, - const_properties, - shared_const_properties, + constant_properties, + shared_constant_properties, node_type, - node_type_in_df.unwrap_or(true), + node_type_col, graph, ) .map_err(|e| GraphLoadException::new_err(format!("{:?}", e)))?; @@ -62,51 +54,38 @@ pub fn load_nodes_from_pandas( pub fn load_edges_from_pandas( graph: &GraphStorage, df: &PyAny, + time: &str, src: &str, dst: &str, - time: &str, - properties: Option>, - const_properties: Option>, - shared_const_properties: Option>, + properties: Option<&[&str]>, + constant_properties: Option<&[&str]>, + shared_constant_properties: Option<&HashMap>, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { Python::with_gil(|py| { - let size: usize = py - .eval( - "index.__len__()", - Some([("index", df.getattr("index")?)].into_py_dict(py)), - None, - )? - .extract()?; - let mut cols_to_check = vec![src, dst, time]; - cols_to_check.extend(properties.as_ref().unwrap_or(&Vec::new())); - cols_to_check.extend(const_properties.as_ref().unwrap_or(&Vec::new())); - if layer_in_df.unwrap_or(false) { - if let Some(ref layer) = layer { - cols_to_check.push(layer.as_ref()); - } + cols_to_check.extend(properties.unwrap_or(&Vec::new())); + cols_to_check.extend(constant_properties.unwrap_or(&Vec::new())); + if let Some(ref layer_col) = layer_col { + cols_to_check.push(layer_col.as_ref()); } - let df = process_pandas_py_df(df, py, cols_to_check.clone())?; - - df.check_cols_exist(&cols_to_check)?; + let df_view = process_pandas_py_df(df, py, cols_to_check.clone())?; + df_view.check_cols_exist(&cols_to_check)?; load_edges_from_df( - &df, - size, + df_view, + time, src, dst, - time, properties, - const_properties, - shared_const_properties, + constant_properties, + shared_constant_properties, layer, - layer_in_df.unwrap_or(true), + layer_col, graph, ) .map_err(|e| GraphLoadException::new_err(format!("{:?}", e)))?; - Ok::<(), PyErr>(()) }) .map_err(|e| GraphError::LoadFailure(format!("Failed to load graph {e:?}")))?; @@ -117,32 +96,29 @@ pub fn load_node_props_from_pandas( graph: &GraphStorage, df: &PyAny, id: &str, - const_properties: Option>, - shared_const_properties: Option>, + node_type: Option<&str>, + node_type_col: Option<&str>, + constant_properties: Option<&[&str]>, + shared_constant_properties: Option<&HashMap>, ) -> Result<(), GraphError> { Python::with_gil(|py| { - let size: usize = py - .eval( - "index.__len__()", - Some([("index", df.getattr("index")?)].into_py_dict(py)), - None, - )? - .extract()?; let mut cols_to_check = vec![id]; - cols_to_check.extend(const_properties.as_ref().unwrap_or(&Vec::new())); - let df = process_pandas_py_df(df, py, cols_to_check.clone())?; - df.check_cols_exist(&cols_to_check)?; - + cols_to_check.extend(constant_properties.unwrap_or(&Vec::new())); + if let Some(ref node_type_col) = node_type_col { + cols_to_check.push(node_type_col.as_ref()); + } + let df_view = process_pandas_py_df(df, py, cols_to_check.clone())?; + df_view.check_cols_exist(&cols_to_check)?; load_node_props_from_df( - &df, - size, + df_view, id, - const_properties, - shared_const_properties, + node_type, + node_type_col, + constant_properties, + shared_constant_properties, graph, ) .map_err(|e| GraphLoadException::new_err(format!("{:?}", e)))?; - Ok::<(), PyErr>(()) }) .map_err(|e| GraphError::LoadFailure(format!("Failed to load graph {e:?}")))?; @@ -154,98 +130,74 @@ pub fn load_edge_props_from_pandas( df: &PyAny, src: &str, dst: &str, - const_properties: Option>, - shared_const_properties: Option>, + constant_properties: Option<&[&str]>, + shared_constant_properties: Option<&HashMap>, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { Python::with_gil(|py| { - let size: usize = py - .eval( - "index.__len__()", - Some([("index", df.getattr("index")?)].into_py_dict(py)), - None, - )? - .extract()?; let mut cols_to_check = vec![src, dst]; - if layer_in_df.unwrap_or(false) { - if let Some(ref layer) = layer { - cols_to_check.push(layer.as_ref()); - } + if let Some(ref layer_col) = layer_col { + cols_to_check.push(layer_col.as_ref()); } - cols_to_check.extend(const_properties.as_ref().unwrap_or(&Vec::new())); - let df = process_pandas_py_df(df, py, cols_to_check.clone())?; - df.check_cols_exist(&cols_to_check)?; + cols_to_check.extend(constant_properties.unwrap_or(&Vec::new())); + let df_view = process_pandas_py_df(df, py, cols_to_check.clone())?; + df_view.check_cols_exist(&cols_to_check)?; load_edges_props_from_df( - &df, - size, + df_view, src, dst, - const_properties, - shared_const_properties, + constant_properties, + shared_constant_properties, layer, - layer_in_df.unwrap_or(true), + layer_col, graph, ) .map_err(|e| GraphLoadException::new_err(format!("{:?}", e)))?; - df.check_cols_exist(&cols_to_check)?; Ok::<(), PyErr>(()) }) .map_err(|e| GraphError::LoadFailure(format!("Failed to load graph {e:?}")))?; Ok(()) } -pub fn load_edges_deletions_from_pandas( +pub fn load_edge_deletions_from_pandas( graph: &GraphStorage, df: &PyAny, + time: &str, src: &str, dst: &str, - time: &str, layer: Option<&str>, - layer_in_df: Option, + layer_col: Option<&str>, ) -> Result<(), GraphError> { Python::with_gil(|py| { - let size: usize = py - .eval( - "index.__len__()", - Some([("index", df.getattr("index")?)].into_py_dict(py)), - None, - )? - .extract()?; - let mut cols_to_check = vec![src, dst, time]; - if layer_in_df.unwrap_or(true) { - if let Some(ref layer) = layer { - cols_to_check.push(layer.as_ref()); - } + if let Some(ref layer_col) = layer_col { + cols_to_check.push(layer_col.as_ref()); } - let df = process_pandas_py_df(df, py, cols_to_check.clone())?; - df.check_cols_exist(&cols_to_check)?; - - load_edges_deletions_from_df( - &df, - size, + let df_view = process_pandas_py_df(df, py, cols_to_check.clone())?; + df_view.check_cols_exist(&cols_to_check)?; + load_edge_deletions_from_df( + df_view, + time, src, dst, - time, layer, - layer_in_df.unwrap_or(true), + layer_col, graph.core_graph(), ) .map_err(|e| GraphLoadException::new_err(format!("{:?}", e)))?; - Ok::<(), PyErr>(()) }) .map_err(|e| GraphError::LoadFailure(format!("Failed to load graph {e:?}")))?; Ok(()) } -pub(crate) fn process_pandas_py_df( - df: &PyAny, - py: Python, +pub(crate) fn process_pandas_py_df<'a>( + df: &'a PyAny, + py: Python<'a>, col_names: Vec<&str>, -) -> PyResult { +) -> PyResult> + 'a>> { is_jupyter(py); py.import("pandas")?; let module = py.import("pyarrow")?; @@ -268,8 +220,11 @@ pub(crate) fn process_pandas_py_df( let _df_columns: Vec = dropped_df.getattr("columns")?.extract()?; let table = pa_table.call_method("from_pandas", (dropped_df,), None)?; - - let rb = table.call_method0("to_batches")?.extract::>()?; + let kwargs = PyDict::new(py); + kwargs.set_item("max_chunksize", 100000)?; + let rb = table + .call_method("to_batches", (), Some(kwargs))? + .extract::>()?; let names: Vec = if let Some(batch0) = rb.get(0) { let schema = batch0.getattr("schema")?; schema.getattr("names")?.extract::>()? @@ -280,21 +235,33 @@ pub(crate) fn process_pandas_py_df( .filter(|x| col_names.contains(&x.as_str())) .collect(); - let arrays = rb - .iter() - .map(|rb| { - (0..names.len()) - .map(|i| { - let array = rb.call_method1("column", (i,))?; - let arr = array_to_rust(array)?; - Ok::, PyErr>(arr) - }) - .collect::, PyErr>>() - }) - .collect::, PyErr>>()?; - - let df = DFView { names, arrays }; - Ok(df) + let names_len = names.len(); + let chunks = rb.into_iter().map(move |rb| { + let chunk = (0..names_len) + .map(|i| { + let array = rb + .call_method1("column", (i,)) + .map_err(|e| GraphError::from(e))?; + let arr = array_to_rust(array).map_err(|e| GraphError::from(e))?; + Ok::, GraphError>(arr) + }) + .collect::, GraphError>>()?; + + Ok(DFChunk { chunk }) + }); + let num_rows: usize = py + .eval( + "index.__len__()", + Some([("index", df.getattr("index")?)].into_py_dict(py)), + None, + )? + .extract()?; + + Ok(DFView { + names, + chunks, + num_rows, + }) } pub fn array_to_rust(obj: &PyAny) -> PyResult { diff --git a/raphtory/src/python/graph/views/graph_view.rs b/raphtory/src/python/graph/views/graph_view.rs index 724895cf16..ef293a636f 100644 --- a/raphtory/src/python/graph/views/graph_view.rs +++ b/raphtory/src/python/graph/views/graph_view.rs @@ -33,7 +33,7 @@ use crate::{ }, }; use chrono::prelude::*; -use pyo3::{prelude::*, types::PyBytes}; +use pyo3::prelude::*; use raphtory_api::core::storage::arc_str::ArcStr; impl IntoPy for MaterializedGraph { @@ -53,15 +53,7 @@ impl IntoPy for DynamicGraph { impl<'source> FromPyObject<'source> for DynamicGraph { fn extract(ob: &'source PyAny) -> PyResult { - ob.extract::>() - .map(|g| g.graph.clone()) - .or_else(|err| { - let res = ob.call_method0("bincode").map_err(|_| err)?; // return original error as probably more helpful - // assume we have a graph at this point, the res probably should not fail - let b = res.extract::<&[u8]>()?; - let g = MaterializedGraph::from_bincode(b)?; - Ok(g.into_dynamic()) - }) + ob.extract::>().map(|g| g.graph.clone()) } } /// Graph view is a read-only version of a graph at a certain point in time. @@ -350,12 +342,6 @@ impl PyGraphView { self.graph.materialize() } - /// Get bincode encoded graph - pub fn bincode<'py>(&'py self, py: Python<'py>) -> Result<&'py PyBytes, GraphError> { - let bytes = self.graph.materialize()?.bincode()?; - Ok(PyBytes::new(py, &bytes)) - } - /// Displays the graph pub fn __repr__(&self) -> String { self.repr() diff --git a/raphtory/src/python/types/macros/trait_impl/mod.rs b/raphtory/src/python/types/macros/trait_impl/mod.rs index fcd619dcae..eb786ae389 100644 --- a/raphtory/src/python/types/macros/trait_impl/mod.rs +++ b/raphtory/src/python/types/macros/trait_impl/mod.rs @@ -16,3 +16,6 @@ mod repr; mod iterable_mixin; mod node_state; + +#[macro_use] +mod serialise; diff --git a/raphtory/src/python/types/macros/trait_impl/serialise.rs b/raphtory/src/python/types/macros/trait_impl/serialise.rs new file mode 100644 index 0000000000..9f4ac453c6 --- /dev/null +++ b/raphtory/src/python/types/macros/trait_impl/serialise.rs @@ -0,0 +1,85 @@ +/// Macro for implementing all the Cache methods on a python wrapper +/// +/// # Arguments +/// * obj: The struct the methods should be implemented for +/// * field: The name of the struct field holding the rust struct implementing `Cache` +/// * base_type: The rust type of `field` +/// * name: The name of the object that appears in the docstring +macro_rules! impl_serialise { + ($obj:ty, $field:ident: $base_type:ty, $name:literal) => { + #[pyo3::pymethods] + impl $obj { + #[doc = concat!(" Write ", $name, " to cache file and initialise the cache.")] + /// + /// Future updates are tracked. Use `write_updates` to persist them to the + /// cache file. If the file already exists its contents are overwritten. + /// + /// Arguments: + /// path (str): The path to the cache file + fn cache(&self, path: &str) -> Result<(), GraphError> { + $crate::serialise::CacheOps::cache(&self.$field, path) + } + + /// Persist the new updates by appending them to the cache file. + fn write_updates(&self) -> Result<(), GraphError> { + $crate::serialise::CacheOps::write_updates(&self.$field) + } + + #[doc = concat!(" Load ", $name, " from a file and initialise it as a cache file.")] + /// + /// Future updates are tracked. Use `write_updates` to persist them to the + /// cache file. + /// + /// Arguments: + /// path (str): The path to the cache file + /// + /// Returns: + #[doc = concat!(" ", $name)] + #[staticmethod] + fn load_cached(path: &str) -> Result<$base_type, GraphError> { + <$base_type as $crate::serialise::CacheOps>::load_cached(path) + } + + #[doc = concat!(" Load ", $name, " from a file.")] + /// + /// Arguments: + /// path (str): The path to the file. + /// + /// Returns: + #[doc = concat!(" ", $name)] + #[staticmethod] + fn load_from_file(path: &str) -> Result<$base_type, GraphError> { + <$base_type as $crate::serialise::StableDecode>::decode(path) + } + + #[doc = concat!(" Saves the ", $name, " to the given path.")] + /// + /// Arguments: + /// path (str): The path to the file. + fn save_to_file(&self, path: &str) -> Result<(), GraphError> { + $crate::serialise::StableEncode::encode(&self.$field, path) + } + + #[doc = concat!(" Load ", $name, " from serialised bytes.")] + /// + /// Arguments: + /// bytes (Bytes): The serialised bytes to decode + /// + /// Returns: + #[doc = concat!(" ", $name)] + #[staticmethod] + fn deserialise(bytes: &[u8]) -> Result<$base_type, GraphError> { + <$base_type as $crate::serialise::StableDecode>::decode_from_bytes(bytes) + } + + #[doc = concat!(" Serialise ", $name, " to bytes.")] + /// + /// Returns: + /// Bytes + fn serialise<'py>(&self, py: Python<'py>) -> &'py pyo3::types::PyBytes { + let bytes = $crate::serialise::StableEncode::encode_to_vec(&self.$field); + pyo3::types::PyBytes::new(py, &bytes) + } + } + }; +} diff --git a/raphtory/src/search/mod.rs b/raphtory/src/search/mod.rs index c022fcd35e..c5170ebd9c 100644 --- a/raphtory/src/search/mod.rs +++ b/raphtory/src/search/mod.rs @@ -1,16 +1,8 @@ // search goes here pub mod into_indexed; - -use std::{collections::HashSet, ops::Deref, path::Path, sync::Arc}; - -use raphtory_api::core::storage::arc_str::ArcStr; -use rayon::{prelude::ParallelIterator, slice::ParallelSlice}; -use tantivy::{ - collector::TopDocs, - schema::{Field, Schema, SchemaBuilder, Value, FAST, INDEXED, STORED, TEXT}, - Index, IndexReader, IndexSettings, IndexWriter, TantivyDocument, TantivyError, -}; +#[cfg(feature = "proto")] +mod serialise; use crate::{ core::{ @@ -24,17 +16,27 @@ use crate::{ }, db::{ api::{ - mutation::internal::{InheritPropertyAdditionOps, InternalAdditionOps}, - storage::edges::edge_storage_ops::EdgeStorageOps, + mutation::internal::{ + InheritPropertyAdditionOps, InternalAdditionOps, InternalDeletionOps, + }, + storage::graph::edges::edge_storage_ops::EdgeStorageOps, view::{ internal::{DynamicGraph, InheritViewOps, IntoDynamic, Static}, - Base, MaterializedGraph, StaticGraphViewOps, + Base, StaticGraphViewOps, }, }, graph::{edge::EdgeView, node::NodeView}, }, prelude::*, }; +use raphtory_api::core::storage::{arc_str::ArcStr, dict_mapper::MaybeNew}; +use rayon::{prelude::ParallelIterator, slice::ParallelSlice}; +use std::{collections::HashSet, ops::Deref, sync::Arc}; +use tantivy::{ + collector::TopDocs, + schema::{Field, Schema, SchemaBuilder, Value, FAST, INDEXED, STORED, TEXT}, + Index, IndexReader, IndexSettings, IndexWriter, TantivyDocument, TantivyError, +}; #[derive(Clone)] pub struct IndexedGraph { @@ -93,12 +95,6 @@ impl + IntoDynamic> IndexedGraph { } } -impl IndexedGraph { - pub fn save_to_file(&self, path: impl AsRef) -> Result<(), GraphError> { - self.graph.save_to_file(path) - } -} - impl<'graph, G: GraphViewOps<'graph>> IndexedGraph { pub fn graph(&self) -> &G { &self.graph @@ -765,27 +761,31 @@ impl InternalAdditionOps for Indexe self.graph.next_event_id() } #[inline] - fn resolve_layer(&self, layer: Option<&str>) -> Result { + fn resolve_layer(&self, layer: Option<&str>) -> Result, GraphError> { self.graph.resolve_layer(layer) } #[inline] - fn set_node_type(&self, v_id: VID, node_type: &str) -> Result<(), GraphError> { - self.graph.set_node_type(v_id, node_type)?; - let node_type_field = self.node_index.schema().get_field(fields::NODE_TYPE)?; + fn resolve_node(&self, n: V) -> Result, GraphError> { + self.graph.resolve_node(n) + } + + fn resolve_node_and_type( + &self, + id: V, + node_type: &str, + ) -> Result, MaybeNew)>, GraphError> { + let res = self.graph.resolve_node_and_type(id, node_type)?; + let (vid, _) = res.inner(); let mut document = TantivyDocument::new(); + let node_type_field = self.node_index.schema().get_field(fields::NODE_TYPE)?; document.add_text(node_type_field, node_type); let node_id_field = self.node_index.schema().get_field(fields::VERTEX_ID)?; - document.add_u64(node_id_field, v_id.as_u64()); + document.add_u64(node_id_field, vid.inner().as_u64()); let mut writer = self.node_index.writer(50_000_000)?; writer.add_document(document)?; writer.commit()?; - Ok(()) - } - - #[inline] - fn resolve_node(&self, n: V) -> Result { - self.graph.resolve_node(n) + Ok(res) } #[inline] @@ -794,7 +794,7 @@ impl InternalAdditionOps for Indexe prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.graph.resolve_graph_property(prop, dtype, is_static) } @@ -804,7 +804,7 @@ impl InternalAdditionOps for Indexe prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.graph.resolve_node_property(prop, dtype, is_static) } @@ -814,7 +814,7 @@ impl InternalAdditionOps for Indexe prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.graph.resolve_edge_property(prop, dtype, is_static) } @@ -822,7 +822,7 @@ impl InternalAdditionOps for Indexe &self, t: TimeIndexEntry, v: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { let mut document = TantivyDocument::new(); // add time to the document @@ -847,38 +847,58 @@ impl InternalAdditionOps for Indexe // get the field from the index let node_id = self.node_index.schema().get_field(fields::VERTEX_ID)?; document.add_u64(node_id, v.as_u64()); - let mut writer = self.node_index.writer(50_000_000)?; - writer.add_document(document)?; - writer.commit()?; - Ok(()) + self.graph.internal_add_node(t, v, props) } fn internal_add_edge( &self, - _t: TimeIndexEntry, - _src: VID, - _dst: VID, - _props: Vec<(usize, Prop)>, - _layer: usize, - ) -> Result { - todo!() + t: TimeIndexEntry, + src: VID, + dst: VID, + props: &[(usize, Prop)], + layer: usize, + ) -> Result, GraphError> { + self.graph.internal_add_edge(t, src, dst, props, layer) } fn internal_add_edge_update( &self, - _t: TimeIndexEntry, - _edge: EID, - _props: Vec<(usize, Prop)>, - _layer: usize, + t: TimeIndexEntry, + edge: EID, + props: &[(usize, Prop)], + layer: usize, ) -> Result<(), GraphError> { - todo!() + self.graph.internal_add_edge_update(t, edge, props, layer) } } +impl InternalDeletionOps for IndexedGraph { + fn internal_delete_edge( + &self, + t: TimeIndexEntry, + src: VID, + dst: VID, + layer: usize, + ) -> Result, GraphError> { + self.graph.internal_delete_edge(t, src, dst, layer) + } + + fn internal_delete_existing_edge( + &self, + t: TimeIndexEntry, + eid: EID, + layer: usize, + ) -> Result<(), GraphError> { + self.graph.internal_delete_existing_edge(t, eid, layer) + } +} + +impl DeletionOps for IndexedGraph {} + #[cfg(test)] mod test { use std::time::SystemTime; @@ -915,9 +935,10 @@ mod test { } #[test] + #[cfg(feature = "proto")] #[ignore = "this test is for experiments with the jira graph"] fn load_jira_graph() -> Result<(), GraphError> { - let graph = Graph::load_from_file("/tmp/graphs/jira", false).expect("failed to load graph"); + let graph = Graph::decode("/tmp/graphs/jira").expect("failed to load graph"); assert!(graph.count_nodes() > 0); let now = SystemTime::now(); diff --git a/raphtory/src/search/serialise.rs b/raphtory/src/search/serialise.rs new file mode 100644 index 0000000000..7d53768eae --- /dev/null +++ b/raphtory/src/search/serialise.rs @@ -0,0 +1,34 @@ +use crate::{ + core::utils::errors::GraphError, + prelude::{GraphViewOps, StableDecode, StableEncode}, + search::IndexedGraph, + serialise::{ + incremental::{GraphWriter, InternalCache}, + ProtoGraph, + }, +}; +use std::path::Path; + +impl StableEncode for IndexedGraph { + fn encode_to_proto(&self) -> ProtoGraph { + self.graph.encode_to_proto() + } +} + +impl<'graph, G: StableDecode + GraphViewOps<'graph>> StableDecode for IndexedGraph { + fn decode_from_proto(graph: &ProtoGraph) -> Result { + let inner = G::decode_from_proto(graph)?; + let indexed = Self::from_graph(&inner)?; + Ok(indexed) + } +} + +impl InternalCache for IndexedGraph { + fn init_cache(&self, path: impl AsRef) -> Result<(), GraphError> { + self.graph.init_cache(path) + } + + fn get_cache(&self) -> Option<&GraphWriter> { + self.graph.get_cache() + } +} diff --git a/raphtory/src/graph.proto b/raphtory/src/serialise/graph.proto similarity index 97% rename from raphtory/src/graph.proto rename to raphtory/src/serialise/graph.proto index fe1c899c45..4c247923d0 100644 --- a/raphtory/src/graph.proto +++ b/raphtory/src/serialise/graph.proto @@ -100,6 +100,8 @@ message GraphUpdate { DelEdge del_edge = 5; UpdateEdgeCProps update_edge_cprops = 6; UpdateEdgeTProps update_edge_tprops = 7; + + UpdateNodeType update_node_type = 8; } message UpdateNodeTProps { @@ -114,6 +116,11 @@ message GraphUpdate { repeated PropPair properties = 2; } + message UpdateNodeType { + uint64 id = 1; + uint64 type_id = 2; + } + message UpdateEdgeTProps { uint64 eid = 1; int64 time = 2; diff --git a/raphtory/src/serialise/incremental.rs b/raphtory/src/serialise/incremental.rs new file mode 100644 index 0000000000..e02ea5d37b --- /dev/null +++ b/raphtory/src/serialise/incremental.rs @@ -0,0 +1,263 @@ +use crate::{ + core::{utils::errors::GraphError, Prop, PropType}, + db::{ + api::{storage::storage::Storage, view::MaterializedGraph}, + graph::views::deletion_graph::PersistentGraph, + }, + prelude::Graph, + serialise::{ + serialise::{CacheOps, StableDecode, StableEncode}, + ProtoGraph, + }, +}; +use parking_lot::Mutex; +use prost::Message; +use raphtory_api::core::{ + entities::{GidRef, EID, VID}, + storage::{dict_mapper::MaybeNew, timeindex::TimeIndexEntry}, +}; +use std::{ + fmt::Debug, + fs::{File, OpenOptions}, + io::Write, + mem, + ops::DerefMut, + path::Path, +}; + +#[derive(Debug)] +pub struct GraphWriter { + writer: Mutex, + proto_delta: Mutex, +} + +impl GraphWriter { + pub fn new(file: File) -> Self { + Self { + writer: Mutex::new(file), + proto_delta: Default::default(), + } + } + pub fn write(&self) -> Result<(), GraphError> { + let proto = mem::take(self.proto_delta.lock().deref_mut()); + let bytes = proto.encode_to_vec(); + if !bytes.is_empty() { + self.writer.lock().write_all(&bytes)?; + } + Ok(()) + } + + #[inline] + pub fn resolve_layer(&self, layer: Option<&str>, layer_id: MaybeNew) { + layer_id.if_new(|id| { + if let Some(layer) = layer { + self.proto_delta.lock().new_layer(layer, id) + } + }); + } + + pub fn resolve_node(&self, vid: MaybeNew, gid: GidRef) { + vid.if_new(|vid| self.proto_delta.lock().new_node(gid, vid, 0)); + } + + pub fn resolve_node_and_type( + &self, + node_and_type: MaybeNew<(MaybeNew, MaybeNew)>, + node_type: &str, + gid: GidRef, + ) { + if let MaybeNew::New((MaybeNew::Existing(node_id), type_id)) = node_and_type { + // type assignment changed but node already exists + self.proto_delta + .lock() + .update_node_type(node_id, type_id.inner()); + } + if let (MaybeNew::New(node_id), type_id) = node_and_type.inner() { + self.proto_delta + .lock() + .new_node(gid, node_id, type_id.inner()); + } + if let (_, MaybeNew::New(type_id)) = node_and_type.inner() { + self.proto_delta.lock().new_node_type(node_type, type_id); + } + } + + pub fn resolve_graph_property( + &self, + prop: &str, + prop_id: MaybeNew, + dtype: PropType, + is_static: bool, + ) { + prop_id.if_new(|id| { + if is_static { + self.proto_delta.lock().new_graph_cprop(prop, id); + } else { + self.proto_delta.lock().new_graph_tprop(prop, id, &dtype); + } + }); + } + + pub fn resolve_node_property( + &self, + prop: &str, + prop_id: MaybeNew, + dtype: PropType, + is_static: bool, + ) { + prop_id.if_new(|id| { + if is_static { + self.proto_delta.lock().new_node_cprop(prop, id, &dtype); + } else { + self.proto_delta.lock().new_node_tprop(prop, id, &dtype); + } + }); + } + + pub fn resolve_edge_property( + &self, + prop: &str, + prop_id: MaybeNew, + dtype: PropType, + is_static: bool, + ) { + prop_id.if_new(|id| { + if is_static { + self.proto_delta.lock().new_edge_cprop(prop, id, &dtype); + } else { + self.proto_delta.lock().new_edge_tprop(prop, id, &dtype); + } + }); + } + + pub fn add_node_update(&self, t: TimeIndexEntry, v: VID, props: &[(usize, Prop)]) { + self.proto_delta + .lock() + .update_node_tprops(v, t, props.iter().map(|(id, prop)| (*id, prop))) + } + + pub fn resolve_edge(&self, eid: MaybeNew, src: VID, dst: VID) { + eid.if_new(|eid| self.proto_delta.lock().new_edge(src, dst, eid)); + } + + pub fn add_edge_update( + &self, + t: TimeIndexEntry, + edge: EID, + props: &[(usize, Prop)], + layer: usize, + ) { + self.proto_delta.lock().update_edge_tprops( + edge, + t, + layer, + props.iter().map(|(id, prop)| (*id, prop)), + ) + } + pub fn add_graph_tprops(&self, t: TimeIndexEntry, props: &[(usize, Prop)]) { + self.proto_delta + .lock() + .update_graph_tprops(t, props.iter().map(|(id, prop)| (*id, prop))) + } + + pub fn add_graph_cprops(&self, props: &[(usize, Prop)]) { + self.proto_delta + .lock() + .update_graph_cprops(props.iter().map(|(id, prop)| (*id, prop))) + } + + pub fn add_node_cprops(&self, node: VID, props: &[(usize, Prop)]) { + self.proto_delta + .lock() + .update_node_cprops(node, props.iter().map(|(id, prop)| (*id, prop))) + } + + pub fn add_edge_cprops(&self, edge: EID, layer: usize, props: &[(usize, Prop)]) { + self.proto_delta.lock().update_edge_cprops( + edge, + layer, + props.iter().map(|(id, prop)| (*id, prop)), + ) + } + + pub fn delete_edge(&self, edge: EID, t: TimeIndexEntry, layer: usize) { + self.proto_delta.lock().del_edge(edge, layer, t) + } +} + +pub(crate) trait InternalCache { + /// Initialise the cache by pointing it at a proto file. + /// Future updates will be appended to the cache. + fn init_cache(&self, path: impl AsRef) -> Result<(), GraphError>; + + /// Get the cache writer if it is initialised. + fn get_cache(&self) -> Option<&GraphWriter>; +} + +impl InternalCache for Storage { + fn init_cache(&self, path: impl AsRef) -> Result<(), GraphError> { + self.cache.get_or_try_init(|| { + let file = OpenOptions::new().append(true).open(path)?; + Ok::<_, GraphError>(GraphWriter::new(file)) + })?; + Ok(()) + } + + fn get_cache(&self) -> Option<&GraphWriter> { + self.cache.get() + } +} + +impl InternalCache for Graph { + fn init_cache(&self, path: impl AsRef) -> Result<(), GraphError> { + self.inner.init_cache(path) + } + + fn get_cache(&self) -> Option<&GraphWriter> { + self.inner.get_cache() + } +} + +impl InternalCache for PersistentGraph { + fn init_cache(&self, path: impl AsRef) -> Result<(), GraphError> { + self.0.init_cache(path) + } + + fn get_cache(&self) -> Option<&GraphWriter> { + self.0.get_cache() + } +} + +impl InternalCache for MaterializedGraph { + fn init_cache(&self, path: impl AsRef) -> Result<(), GraphError> { + match self { + MaterializedGraph::EventGraph(g) => g.init_cache(path), + MaterializedGraph::PersistentGraph(g) => g.init_cache(path), + } + } + + fn get_cache(&self) -> Option<&GraphWriter> { + match self { + MaterializedGraph::EventGraph(g) => g.get_cache(), + MaterializedGraph::PersistentGraph(g) => g.get_cache(), + } + } +} + +impl CacheOps for G { + fn cache(&self, path: impl AsRef) -> Result<(), GraphError> { + self.encode(path.as_ref())?; + self.init_cache(path) + } + + fn write_updates(&self) -> Result<(), GraphError> { + let cache = self.get_cache().ok_or(GraphError::CacheNotInnitialised)?; + cache.write() + } + + fn load_cached(path: impl AsRef) -> Result { + let graph = Self::decode(path.as_ref())?; + graph.init_cache(path)?; + Ok(graph) + } +} diff --git a/raphtory/src/serialise/mod.rs b/raphtory/src/serialise/mod.rs new file mode 100644 index 0000000000..f418fc8417 --- /dev/null +++ b/raphtory/src/serialise/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod incremental; +mod serialise; + +mod proto { + include!(concat!(env!("OUT_DIR"), "/serialise.rs")); +} + +pub use proto::Graph as ProtoGraph; +pub use serialise::{CacheOps, StableDecode, StableEncode}; diff --git a/raphtory/src/db/api/view/serialise.rs b/raphtory/src/serialise/serialise.rs similarity index 82% rename from raphtory/src/db/api/view/serialise.rs rename to raphtory/src/serialise/serialise.rs index c7e32d10a5..7a36156fef 100644 --- a/raphtory/src/db/api/view/serialise.rs +++ b/raphtory/src/serialise/serialise.rs @@ -1,4 +1,3 @@ -use super::MaterializedGraph; use crate::{ core::{ entities::{ @@ -14,19 +13,21 @@ use crate::{ mutation::internal::{ InternalAdditionOps, InternalDeletionOps, InternalPropertyAdditionOps, }, - storage::{ + storage::graph::{ edges::edge_storage_ops::EdgeStorageOps, nodes::node_storage_ops::NodeStorageOps, storage_ops::GraphStorage, tprop_storage_ops::TPropOps, }, - view::internal::CoreGraphOps, + view::{internal::CoreGraphOps, MaterializedGraph}, }, graph::views::deletion_graph::PersistentGraph, }, prelude::Graph, - serialise, serialise::{ - graph_update::*, new_meta::*, new_node, new_node::Gid, prop, - prop_type::PropType as SPropType, GraphUpdate, NewEdge, NewMeta, NewNode, + proto, + proto::{ + graph_update::*, new_meta::*, new_node, new_node::Gid, prop, + prop_type::PropType as SPropType, GraphUpdate, NewEdge, NewMeta, NewNode, + }, }, }; use chrono::{DateTime, Datelike, NaiveDate, NaiveDateTime, NaiveTime, Timelike}; @@ -47,17 +48,17 @@ macro_rules! zip_tprop_updates { &$iter .map(|(key, values)| values.iter().map(move |(t, v)| (t, (key, v)))) .kmerge_by(|(left_t, _), (right_t, _)| left_t <= right_t) - .group_by(|(t, _)| *t) + .chunk_by(|(t, _)| *t) }; } -pub trait StableEncoder { - fn encode_to_proto(&self) -> serialise::Graph; +pub trait StableEncode { + fn encode_to_proto(&self) -> proto::Graph; fn encode_to_vec(&self) -> Vec { self.encode_to_proto().encode_to_vec() } - fn stable_serialise(&self, path: impl AsRef) -> Result<(), GraphError> { + fn encode(&self, path: impl AsRef) -> Result<(), GraphError> { let mut file = File::create(path)?; let bytes = self.encode_to_vec(); file.write_all(&bytes)?; @@ -66,9 +67,9 @@ pub trait StableEncoder { } pub trait StableDecode: Sized { - fn decode_from_proto(graph: &serialise::Graph) -> Result; + fn decode_from_proto(graph: &proto::Graph) -> Result; fn decode_from_bytes(bytes: &[u8]) -> Result { - let graph = serialise::Graph::decode(bytes)?; + let graph = proto::Graph::decode(bytes)?; Self::decode_from_proto(&graph) } fn decode(path: impl AsRef) -> Result { @@ -79,6 +80,19 @@ pub trait StableDecode: Sized { } } +pub trait CacheOps: Sized { + /// Write graph to file and append future updates to the same file. + /// + /// If the file already exists, it's contents are overwritten + fn cache(&self, path: impl AsRef) -> Result<(), GraphError>; + + /// Persist the new updates by appending them to the cache file. + fn write_updates(&self) -> Result<(), GraphError>; + + /// Load graph from file and append future updates to the same file + fn load_cached(path: impl AsRef) -> Result; +} + fn as_proto_prop_type(p_type: &PropType) -> SPropType { match p_type { PropType::Str => SPropType::Str, @@ -214,6 +228,14 @@ impl GraphUpdate { Self::new(Update::UpdateGraphTprops(inner)) } + fn update_node_type(node_id: VID, type_id: usize) -> Self { + let inner = UpdateNodeType { + id: node_id.as_u64(), + type_id: type_id as u64, + }; + Self::new(Update::UpdateNodeType(inner)) + } + fn update_node_cprops( node_id: VID, properties: impl Iterator)>, @@ -313,8 +335,8 @@ impl PropPair { } } -impl serialise::Graph { - fn new_edge(&mut self, src: VID, dst: VID, eid: EID) { +impl proto::Graph { + pub fn new_edge(&mut self, src: VID, dst: VID, eid: EID) { let edge = NewEdge { src: src.as_u64(), dst: dst.as_u64(), @@ -323,7 +345,7 @@ impl serialise::Graph { self.edges.push(edge); } - fn new_node(&mut self, gid: GidRef, vid: VID, type_id: usize) { + pub fn new_node(&mut self, gid: GidRef, vid: VID, type_id: usize) { let type_id = type_id as u64; let gid = match gid { GidRef::U64(id) => new_node::Gid::GidU64(id), @@ -337,43 +359,46 @@ impl serialise::Graph { self.nodes.push(node); } - fn new_graph_cprop(&mut self, key: &str, id: usize) { + pub fn new_graph_cprop(&mut self, key: &str, id: usize) { self.metas.push(NewMeta::new_graph_cprop(key, id)); } - fn new_graph_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { + pub fn new_graph_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { self.metas.push(NewMeta::new_graph_tprop(key, id, dtype)); } - fn new_node_cprop(&mut self, key: &str, id: usize, dtype: &PropType) { + pub fn new_node_cprop(&mut self, key: &str, id: usize, dtype: &PropType) { self.metas.push(NewMeta::new_node_cprop(key, id, dtype)); } - fn new_node_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { + pub fn new_node_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { self.metas.push(NewMeta::new_node_tprop(key, id, dtype)); } - fn new_edge_cprop(&mut self, key: &str, id: usize, dtype: &PropType) { + pub fn new_edge_cprop(&mut self, key: &str, id: usize, dtype: &PropType) { self.metas.push(NewMeta::new_edge_cprop(key, id, dtype)); } - fn new_edge_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { + pub fn new_edge_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { self.metas.push(NewMeta::new_edge_tprop(key, id, dtype)) } - fn new_layer(&mut self, layer: &str, id: usize) { + pub fn new_layer(&mut self, layer: &str, id: usize) { self.metas.push(NewMeta::new_layer(layer, id)); } - fn new_node_type(&mut self, node_type: &str, id: usize) { + pub fn new_node_type(&mut self, node_type: &str, id: usize) { self.metas.push(NewMeta::new_node_type(node_type, id)); } - fn update_graph_cprops(&mut self, values: impl Iterator)>) { + pub fn update_graph_cprops( + &mut self, + values: impl Iterator)>, + ) { self.updates.push(GraphUpdate::update_graph_cprops(values)); } - fn update_graph_tprops( + pub fn update_graph_tprops( &mut self, time: TimeIndexEntry, values: impl IntoIterator)>, @@ -382,7 +407,11 @@ impl serialise::Graph { .push(GraphUpdate::update_graph_tprops(time, values)); } - fn update_node_cprops( + pub fn update_node_type(&mut self, node_id: VID, type_id: usize) { + self.updates + .push(GraphUpdate::update_node_type(node_id, type_id)) + } + pub fn update_node_cprops( &mut self, node_id: VID, properties: impl Iterator)>, @@ -391,7 +420,7 @@ impl serialise::Graph { .push(GraphUpdate::update_node_cprops(node_id, properties)); } - fn update_node_tprops( + pub fn update_node_tprops( &mut self, node_id: VID, time: TimeIndexEntry, @@ -401,7 +430,7 @@ impl serialise::Graph { .push(GraphUpdate::update_node_tprops(node_id, time, properties)); } - fn update_edge_tprops( + pub fn update_edge_tprops( &mut self, eid: EID, time: TimeIndexEntry, @@ -413,7 +442,7 @@ impl serialise::Graph { )); } - fn update_edge_cprops( + pub fn update_edge_cprops( &mut self, eid: EID, layer_id: usize, @@ -423,14 +452,14 @@ impl serialise::Graph { .push(GraphUpdate::update_edge_cprops(eid, layer_id, properties)); } - fn del_edge(&mut self, eid: EID, layer_id: usize, time: TimeIndexEntry) { + pub fn del_edge(&mut self, eid: EID, layer_id: usize, time: TimeIndexEntry) { self.updates .push(GraphUpdate::del_edge(eid, layer_id, time)) } } -impl StableEncoder for GraphStorage { - fn encode_to_proto(&self) -> serialise::Graph { +impl StableEncode for GraphStorage { + fn encode_to_proto(&self) -> proto::Graph { #[cfg(feature = "storage")] if let GraphStorage::Disk(storage) = self { assert!( @@ -440,7 +469,7 @@ impl StableEncoder for GraphStorage { } let storage = self.lock(); - let mut graph = serialise::Graph::default(); + let mut graph = proto::Graph::default(); // Graph Properties let graph_meta = storage.graph_meta(); @@ -467,7 +496,7 @@ impl StableEncoder for GraphStorage { .collect::>() }) .kmerge_by(|(left_t, _), (right_t, _)| left_t <= right_t) - .group_by(|(t, _)| *t) + .chunk_by(|(t, _)| *t) { graph.update_graph_tprops(t, group.map(|(_, v)| v)); } @@ -590,24 +619,33 @@ impl StableEncoder for GraphStorage { } } -impl StableEncoder for Graph { - fn encode_to_proto(&self) -> serialise::Graph { +impl StableEncode for Graph { + fn encode_to_proto(&self) -> proto::Graph { let mut graph = self.core_graph().encode_to_proto(); - graph.set_graph_type(serialise::GraphType::Event); + graph.set_graph_type(proto::GraphType::Event); graph } } -impl StableEncoder for PersistentGraph { - fn encode_to_proto(&self) -> serialise::Graph { +impl StableEncode for PersistentGraph { + fn encode_to_proto(&self) -> proto::Graph { let mut graph = self.core_graph().encode_to_proto(); - graph.set_graph_type(serialise::GraphType::Persistent); + graph.set_graph_type(proto::GraphType::Persistent); graph } } +impl StableEncode for MaterializedGraph { + fn encode_to_proto(&self) -> proto::Graph { + match self { + MaterializedGraph::EventGraph(graph) => graph.encode_to_proto(), + MaterializedGraph::PersistentGraph(graph) => graph.encode_to_proto(), + } + } +} + impl StableDecode for TemporalGraph { - fn decode_from_proto(graph: &serialise::Graph) -> Result { + fn decode_from_proto(graph: &proto::Graph) -> Result { let storage = Self::default(); graph.metas.par_iter().for_each(|meta| { if let Some(meta) = meta.meta.as_ref() { @@ -691,23 +729,24 @@ impl StableDecode for TemporalGraph { Update::UpdateNodeCprops(props) => { storage.internal_update_constant_node_properties( VID(props.id as usize), - collect_props(&props.properties)?, + &collect_props(&props.properties)?, )?; } Update::UpdateNodeTprops(props) => { let time = TimeIndexEntry(props.time, props.secondary as usize); let node = VID(props.id as usize); let props = collect_props(&props.properties)?; - storage.internal_add_node(time, node, props)?; + storage.internal_add_node(time, node, &props)?; } Update::UpdateGraphCprops(props) => { - storage.internal_update_constant_properties(collect_props( + storage.internal_update_constant_properties(&collect_props( &props.properties, )?)?; } Update::UpdateGraphTprops(props) => { let time = TimeIndexEntry(props.time, props.secondary as usize); - storage.internal_add_properties(time, collect_props(&props.properties)?)?; + storage + .internal_add_properties(time, &collect_props(&props.properties)?)?; } Update::DelEdge(del_edge) => { let time = TimeIndexEntry(del_edge.time, del_edge.secondary as usize); @@ -721,7 +760,7 @@ impl StableDecode for TemporalGraph { storage.internal_update_constant_edge_properties( EID(props.eid as usize), props.layer_id as usize, - collect_props(&props.properties)?, + &collect_props(&props.properties)?, )?; } Update::UpdateEdgeTprops(props) => { @@ -730,10 +769,15 @@ impl StableDecode for TemporalGraph { storage.internal_add_edge_update( time, eid, - collect_props(&props.properties)?, + &collect_props(&props.properties)?, props.layer_id as usize, )?; } + Update::UpdateNodeType(update) => { + let id = VID(update.id as usize); + let type_id = update.type_id as usize; + storage.storage.get_node_mut(id).node_type = type_id; + } } } Ok::<_, GraphError>(()) @@ -743,7 +787,7 @@ impl StableDecode for TemporalGraph { } impl StableDecode for GraphStorage { - fn decode_from_proto(graph: &serialise::Graph) -> Result { + fn decode_from_proto(graph: &proto::Graph) -> Result { Ok(GraphStorage::Unlocked(Arc::new( TemporalGraph::decode_from_proto(graph)?, ))) @@ -751,11 +795,11 @@ impl StableDecode for GraphStorage { } impl StableDecode for MaterializedGraph { - fn decode_from_proto(graph: &serialise::Graph) -> Result { + fn decode_from_proto(graph: &proto::Graph) -> Result { let storage = GraphStorage::decode_from_proto(graph)?; let graph = match graph.graph_type() { - serialise::GraphType::Event => Self::EventGraph(Graph::from_internal_graph(storage)), - serialise::GraphType::Persistent => { + proto::GraphType::Event => Self::EventGraph(Graph::from_internal_graph(storage)), + proto::GraphType::Persistent => { Self::PersistentGraph(PersistentGraph::from_internal_graph(storage)) } }; @@ -764,22 +808,22 @@ impl StableDecode for MaterializedGraph { } impl StableDecode for Graph { - fn decode_from_proto(graph: &serialise::Graph) -> Result { + fn decode_from_proto(graph: &proto::Graph) -> Result { match graph.graph_type() { - serialise::GraphType::Event => { + proto::GraphType::Event => { let storage = GraphStorage::decode_from_proto(graph)?; Ok(Graph::from_internal_graph(storage)) } - serialise::GraphType::Persistent => Err(GraphError::GraphLoadError), + proto::GraphType::Persistent => Err(GraphError::GraphLoadError), } } } impl StableDecode for PersistentGraph { - fn decode_from_proto(graph: &serialise::Graph) -> Result { + fn decode_from_proto(graph: &proto::Graph) -> Result { match graph.graph_type() { - serialise::GraphType::Event => Err(GraphError::GraphLoadError), - serialise::GraphType::Persistent => { + proto::GraphType::Event => Err(GraphError::GraphLoadError), + proto::GraphType::Persistent => { let storage = GraphStorage::decode_from_proto(graph)?; Ok(PersistentGraph::from_internal_graph(storage)) } @@ -883,7 +927,7 @@ fn collect_props<'a>( iter.into_iter().map(as_prop).collect() } -fn as_proto_prop(prop: &Prop) -> serialise::Prop { +fn as_proto_prop(prop: &Prop) -> proto::Prop { let value: prop::Value = match prop { Prop::Bool(b) => prop::Value::BoolValue(*b), Prop::U8(u) => prop::Value::U8((*u).into()), @@ -951,7 +995,7 @@ fn as_proto_prop(prop: &Prop) -> serialise::Prop { } }; - serialise::Prop { value: Some(value) } + proto::Prop { value: Some(value) } } #[cfg(test)] @@ -966,7 +1010,7 @@ mod proto_test { graph::graph::assert_graph_equal, }, prelude::*, - serialise::GraphType, + serialise::{proto::GraphType, ProtoGraph}, }; #[test] @@ -974,7 +1018,7 @@ mod proto_test { let temp_file = tempfile::NamedTempFile::new().unwrap(); let g1 = Graph::new(); g1.add_node(1, "Alice", NO_PROPS, None).unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); } @@ -986,7 +1030,7 @@ mod proto_test { g1.add_node(1, "Alice", NO_PROPS, None).unwrap(); g1.add_node(2, "Bob", [("age", Prop::U32(47))], None) .unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); } @@ -1003,7 +1047,7 @@ mod proto_test { n1.update_constant_properties([("name", Prop::Str("Bob".into()))]) .expect("Failed to update constant properties"); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); } @@ -1015,7 +1059,7 @@ mod proto_test { g1.add_node(1, "Alice", NO_PROPS, None).unwrap(); g1.add_node(2, "Bob", NO_PROPS, None).unwrap(); g1.add_edge(3, "Alice", "Bob", NO_PROPS, None).unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); } @@ -1026,7 +1070,7 @@ mod proto_test { let g1 = Graph::new().persistent_graph(); g1.add_edge(3, "Alice", "Bob", NO_PROPS, None).unwrap(); g1.delete_edge(19, "Alice", "Bob", None).unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = PersistentGraph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); @@ -1043,7 +1087,7 @@ mod proto_test { g1.add_node(2, "Bob", NO_PROPS, None).unwrap(); g1.add_edge(3, "Alice", "Bob", [("kind", "friends")], None) .unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); } @@ -1055,7 +1099,7 @@ mod proto_test { let e1 = g1.add_edge(3, "Alice", "Bob", NO_PROPS, None).unwrap(); e1.update_constant_properties([("friends", true)], None) .expect("Failed to update constant properties"); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); } @@ -1068,7 +1112,7 @@ mod proto_test { .unwrap(); g1.add_edge(7, "Bob", "Charlie", [("friends", false)], Some("two")) .unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); } @@ -1081,7 +1125,7 @@ mod proto_test { let temp_file = tempfile::NamedTempFile::new().unwrap(); let g1 = Graph::new(); g1.add_node(1, "Alice", props.clone(), None).unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); @@ -1107,7 +1151,7 @@ mod proto_test { let temp_file = tempfile::NamedTempFile::new().unwrap(); let g1 = Graph::new(); g1.add_edge(1, "Alice", "Bob", props.clone(), None).unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); @@ -1135,7 +1179,7 @@ mod proto_test { let e = g1.add_edge(1, "Alice", "Bob", NO_PROPS, Some("a")).unwrap(); e.update_constant_properties(props.clone(), Some("a")) .expect("Failed to update constant properties"); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); @@ -1161,7 +1205,7 @@ mod proto_test { let n = g1.add_node(1, "Alice", NO_PROPS, None).unwrap(); n.update_constant_properties(props.clone()) .expect("Failed to update constant properties"); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); @@ -1186,7 +1230,7 @@ mod proto_test { .expect("Failed to add constant properties"); let temp_file = tempfile::NamedTempFile::new().unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); @@ -1208,7 +1252,7 @@ mod proto_test { } let temp_file = tempfile::NamedTempFile::new().unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); @@ -1230,7 +1274,7 @@ mod proto_test { #[test] fn manually_test_append() { - let mut graph1 = serialise::Graph::default(); + let mut graph1 = proto::Graph::default(); graph1.set_graph_type(GraphType::Event); graph1.new_node(GidRef::Str("1"), VID(0), 0); graph1.new_node(GidRef::Str("2"), VID(1), 0); @@ -1243,7 +1287,7 @@ mod proto_test { ); let mut bytes1 = graph1.encode_to_vec(); - let mut graph2 = serialise::Graph::default(); + let mut graph2 = proto::Graph::default(); graph2.new_node(GidRef::Str("3"), VID(2), 0); graph2.new_edge(VID(0), VID(2), EID(1)); graph2.update_edge_tprops( @@ -1306,6 +1350,106 @@ mod proto_test { } } + #[test] + fn test_incremental_writing_on_graph() { + let g = Graph::new(); + let mut props = vec![]; + write_props_to_vec(&mut props); + let temp_cache_file = tempfile::NamedTempFile::new().unwrap(); + + g.cache(temp_cache_file.path()).unwrap(); + + for t in 0..props.len() { + g.add_properties(t as i64, (&props[t..t + 1]).to_vec()) + .expect("Failed to add constant properties"); + } + g.write_updates().unwrap(); + + g.add_constant_properties(props.clone()) + .expect("Failed to add constant properties"); + g.write_updates().unwrap(); + + let n = g.add_node(1, "Alice", NO_PROPS, None).unwrap(); + n.update_constant_properties(props.clone()) + .expect("Failed to update constant properties"); + g.write_updates().unwrap(); + + let e = g.add_edge(1, "Alice", "Bob", NO_PROPS, Some("a")).unwrap(); + e.update_constant_properties(props.clone(), Some("a")) + .expect("Failed to update constant properties"); + g.write_updates().unwrap(); + + g.add_edge(2, "Alice", "Bob", props.clone(), None).unwrap(); + g.add_node(1, "Charlie", props.clone(), None).unwrap(); + g.write_updates().unwrap(); + + g.add_edge(7, "Alice", "Bob", NO_PROPS, Some("one")) + .unwrap(); + g.add_edge(7, "Bob", "Charlie", [("friends", false)], Some("two")) + .unwrap(); + g.write_updates().unwrap(); + println!("{g:?}"); + + let g2 = Graph::decode(temp_cache_file.path()).unwrap(); + println!("{g2:?}"); + + assert_graph_equal(&g, &g2); + } + + #[test] + fn test_incremental_writing_on_persistent_graph() { + let g = PersistentGraph::new(); + let mut props = vec![]; + write_props_to_vec(&mut props); + let temp_cache_file = tempfile::NamedTempFile::new().unwrap(); + + g.cache(temp_cache_file.path()).unwrap(); + + for t in 0..props.len() { + g.add_properties(t as i64, (&props[t..t + 1]).to_vec()) + .expect("Failed to add constant properties"); + } + g.write_updates().unwrap(); + + g.add_constant_properties(props.clone()) + .expect("Failed to add constant properties"); + g.write_updates().unwrap(); + + let n = g.add_node(1, "Alice", NO_PROPS, None).unwrap(); + n.update_constant_properties(props.clone()) + .expect("Failed to update constant properties"); + g.write_updates().unwrap(); + + let e = g.add_edge(1, "Alice", "Bob", NO_PROPS, Some("a")).unwrap(); + e.update_constant_properties(props.clone(), Some("a")) + .expect("Failed to update constant properties"); + g.write_updates().unwrap(); + + g.add_edge(2, "Alice", "Bob", props.clone(), None).unwrap(); + g.add_node(1, "Charlie", props.clone(), None).unwrap(); + g.write_updates().unwrap(); + + g.add_edge(7, "Alice", "Bob", NO_PROPS, Some("one")) + .unwrap(); + g.add_edge(7, "Bob", "Charlie", [("friends", false)], Some("two")) + .unwrap(); + g.write_updates().unwrap(); + println!("{g:?}"); + + let g2 = PersistentGraph::decode(temp_cache_file.path()).unwrap(); + println!("{g2:?}"); + + assert_graph_equal(&g, &g2); + } + + // we rely on this to make sure writing no updates does not actually write anything to file + #[test] + fn empty_proto_is_empty_bytes() { + let proto = ProtoGraph::default(); + let bytes = proto.encode_to_vec(); + assert!(bytes.is_empty()) + } + fn write_props_to_vec(props: &mut Vec<(&str, Prop)>) { props.push(("name", Prop::Str("Alice".into()))); props.push(("age", Prop::U32(47))); diff --git a/raphtory/src/vectors/embeddings.rs b/raphtory/src/vectors/embeddings.rs index 18e949c339..337d95a98a 100644 --- a/raphtory/src/vectors/embeddings.rs +++ b/raphtory/src/vectors/embeddings.rs @@ -13,6 +13,7 @@ pub async fn openai_embedding(texts: Vec) -> Vec { input: EmbeddingInput::StringArray(texts), user: None, encoding_format: None, + dimensions: None, }; let response = client.embeddings().create(request).await.unwrap(); println!("Generated embeddings successfully");