diff --git a/.github/workflows/test_rust_workflow.yml b/.github/workflows/test_rust_workflow.yml index d57d2e8f57..e0547924bc 100644 --- a/.github/workflows/test_rust_workflow.yml +++ b/.github/workflows/test_rust_workflow.yml @@ -42,10 +42,6 @@ jobs: uses: arduino/setup-protoc@v3 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: webfactory/ssh-agent@v0.7.0 - name: Load pometry-storage key - with: - ssh-private-key: ${{ secrets.RA_SSH_PRIVATE_KEY }} - name: Rust version run: rustc --version --verbose - uses: Swatinem/rust-cache@v2 @@ -81,10 +77,6 @@ jobs: name: Cargo cache with: cache-all-crates: true - - uses: webfactory/ssh-agent@v0.9.0 - name: Load raphtory-disk_graph key - with: - ssh-private-key: ${{ secrets.RA_SSH_PRIVATE_KEY }} - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/Cargo.lock b/Cargo.lock index 8f443fd41f..573c22a418 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aead" version = "0.5.2" @@ -70,9 +76,8 @@ checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "const-random", - "getrandom 0.2.15", + "getrandom", "once_cell", - "serde", "version_check", "zerocopy", ] @@ -136,9 +141,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.14" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526" dependencies = [ "anstyle", "anstyle-parse", @@ -151,33 +156,33 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anstyle-parse" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a" dependencies = [ "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8" dependencies = [ "anstyle", "windows-sys 0.52.0", @@ -204,16 +209,6 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" -[[package]] -name = "archery" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8967cd1cc9e9e1954f644e14fbd6042fe9a37da96c52a67e44a2ac18261f8561" -dependencies = [ - "static_assertions", - "triomphe", -] - [[package]] name = "array-init-cursor" version = "0.2.0" @@ -222,15 +217,15 @@ checksum = "bf7d0a018de4f6aa429b9d33d69edf69072b1c5b1cb8d3e4a5f7ef898fc3eb76" [[package]] name = "arrayref" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" +checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" @@ -518,9 +513,9 @@ checksum = "71938f30533e4d95a6d17aa530939da3842c2ab6f4f84b9dae68447e4129f74a" [[package]] name = "async-compression" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" +checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" dependencies = [ "bzip2", "flate2", @@ -545,9 +540,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "7.0.6" +version = "7.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf338d20ba5bab309f55ce8df95d65ee19446f7737f06f4a64593ab2c6b546ad" +checksum = "2b76aba2f176af685c2229633881a3adeae51f87ae1811781e73910b7001c93e" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -579,26 +574,26 @@ dependencies = [ [[package]] name = "async-graphql-derive" -version = "7.0.6" +version = "7.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc51fd6b7102acda72bc94e8ae1543844d5688ff394a6cf7c21f2a07fe2d64e4" +checksum = "72e2e26a6b44bc61df3ca8546402cf9204c28e30c06084cc8e75cd5e34d4f150" dependencies = [ "Inflector", "async-graphql-parser", - "darling 0.20.9", + "darling", "proc-macro-crate 3.1.0", "proc-macro2", "quote", "strum", - "syn 2.0.66", + "syn 2.0.75", "thiserror", ] [[package]] name = "async-graphql-parser" -version = "7.0.6" +version = "7.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75361eefd64e39f89bead4cb45fddbaf60ddb0e7b15fb7c852b6088bcd63071f" +checksum = "f801451484b4977d6fe67b29030f81353cabdcbb754e5a064f39493582dac0cf" dependencies = [ "async-graphql-value", "pest", @@ -608,9 +603,9 @@ dependencies = [ [[package]] name = "async-graphql-poem" -version = "7.0.6" +version = "7.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "072888eb96e1dc0398d8e163015c3c24323a94e71b65757af4ddc07beb38df18" +checksum = "6fead92136d9c0c68b6da5ea16962604163b1f4d59750ec7b36155ca9ff105d3" dependencies = [ "async-graphql", "futures-util", @@ -625,9 +620,9 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "7.0.6" +version = "7.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1f665d2d52b41c4ed1f01c43f3ef27a2fe0af2452ed5c8bc7ac9b1a8719afaa" +checksum = "69117c43c01d81a69890a9f5dd6235f2f027ca8d1ec62d6d3c5e01ca0edb4f2b" dependencies = [ "bytes", "indexmap", @@ -637,18 +632,19 @@ dependencies = [ [[package]] name = "async-openai" -version = "0.17.1" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c6ce3eb96d0957f9d5af15b5b0d651e27da8b567d126ef7fefb889d21dda06" +checksum = "dc0e5ff98f9e7c605df4c88783a0439d1dc667ce86bd79e99d4164f8b0c05ccc" dependencies = [ "async-convert", "backoff", - "base64 0.21.7", + "base64 0.22.1", "bytes", "derive_builder", + "eventsource-stream", "futures", - "rand 0.8.5", - "reqwest", + "rand", + "reqwest 0.12.7", "reqwest-eventsource", "secrecy", "serde", @@ -679,18 +675,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "async-trait" -version = "0.1.80" +version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -718,10 +714,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ "futures-core", - "getrandom 0.2.15", + "getrandom", "instant", "pin-project-lite", - "rand 0.8.5", + "rand", "tokio", ] @@ -735,7 +731,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.4", "object", "rustc-demangle", ] @@ -799,9 +795,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" dependencies = [ "serde", ] @@ -826,9 +822,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.1" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" dependencies = [ "arrayref", "arrayvec", @@ -875,22 +871,22 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78834c15cb5d5efe3452d58b1e8ba890dd62d21907f867f383358198e56ebca5" +checksum = "6fd4c6dcc3b0aea2f5c0b4b82c2b15fe39ddbc76041a310848f4706edf76bb31" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee891b04274a59bd38b412188e24b849617b2e45a0fd8d057deb63e7403761b" +checksum = "0cc8b54b395f2fcfbb3d90c47b01c7f444d94d05bdeb775811dec868ac3bbc26" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -901,9 +897,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" dependencies = [ "serde", ] @@ -937,13 +933,13 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.99" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" +checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48" dependencies = [ "jobserver", "libc", - "once_cell", + "shlex", ] [[package]] @@ -960,9 +956,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cfg_aliases" -version = "0.1.1" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" @@ -976,7 +972,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -1040,9 +1036,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.7" +version = "4.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" +checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" dependencies = [ "clap_builder", "clap_derive", @@ -1050,39 +1046,39 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.7" +version = "4.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" +checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim 0.11.1", + "strsim", ] [[package]] name = "clap_derive" -version = "4.5.5" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" +checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "clap_lex" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "colorchoice" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0" [[package]] name = "comfy-table" @@ -1102,7 +1098,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7328b20597b53c2454f0b1919720c25c7339051c02b72b7e05409e00b14132be" dependencies = [ "async-trait", - "convert_case 0.6.0", + "convert_case", "json5", "lazy_static", "nom", @@ -1140,7 +1136,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.15", + "getrandom", "once_cell", "tiny-keccak", ] @@ -1151,12 +1147,6 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" -[[package]] -name = "convert_case" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" - [[package]] name = "convert_case" version = "0.6.0" @@ -1177,7 +1167,7 @@ dependencies = [ "hkdf", "hmac", "percent-encoding", - "rand 0.8.5", + "rand", "sha2", "subtle", "time", @@ -1196,15 +1186,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" dependencies = [ "libc", ] @@ -1316,7 +1306,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", - "rand_core 0.6.4", + "rand_core", "typenum", ] @@ -1352,81 +1342,60 @@ dependencies = [ [[package]] name = "darling" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" -dependencies = [ - "darling_core 0.14.4", - "darling_macro 0.14.4", -] - -[[package]] -name = "darling" -version = "0.20.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b2eb4d90d12bdda5ed17de686c2acb4c57914f8f921b8da7e112b5a36f3fe1" -dependencies = [ - "darling_core 0.20.9", - "darling_macro 0.20.9", -] - -[[package]] -name = "darling_core" -version = "0.14.4" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 1.0.109", + "darling_core", + "darling_macro", ] [[package]] name = "darling_core" -version = "0.20.9" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622687fe0bac72a04e5599029151f5796111b90f1baaa9b544d807a5e31cd120" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", - "strsim 0.11.1", - "syn 2.0.66", + "strsim", + "syn 2.0.75", ] [[package]] name = "darling_macro" -version = "0.14.4" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ - "darling_core 0.14.4", + "darling_core", "quote", - "syn 1.0.109", + "syn 2.0.75", ] [[package]] -name = "darling_macro" -version = "0.20.9" +name = "dashmap" +version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733cabb43482b1a1b53eee8583c2b9e8684d592215ea83efd305dd31bc2f0178" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ - "darling_core 0.20.9", - "quote", - "syn 2.0.66", + "cfg-if", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", ] [[package]] name = "dashmap" -version = "5.5.3" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +checksum = "804c8821570c3f8b70230c2ba75ffa5c0f9a4189b9a432b6656c536712acae28" dependencies = [ "cfg-if", + "crossbeam-utils", "hashbrown 0.14.5", "lock_api", "once_cell", @@ -1456,7 +1425,7 @@ dependencies = [ "bytes", "bzip2", "chrono", - "dashmap", + "dashmap 5.5.3", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -1479,7 +1448,7 @@ dependencies = [ "parking_lot", "parquet", "pin-project-lite", - "rand 0.8.5", + "rand", "sqlparser", "tempfile", "tokio", @@ -1518,7 +1487,7 @@ checksum = "3f9c93043081487e335399a21ebf8295626367a647ac5cb87d41d18afad7d0f7" dependencies = [ "arrow", "chrono", - "dashmap", + "dashmap 5.5.3", "datafusion-common", "datafusion-expr", "futures", @@ -1526,7 +1495,7 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand 0.8.5", + "rand", "tempfile", "url", ] @@ -1623,7 +1592,7 @@ dependencies = [ "md-5", "paste", "petgraph", - "rand 0.8.5", + "rand", "regex", "sha2", "unicode-segmentation", @@ -1656,7 +1625,7 @@ dependencies = [ "once_cell", "parking_lot", "pin-project-lite", - "rand 0.8.5", + "rand", "tokio", "uuid", ] @@ -1696,9 +1665,20 @@ checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" [[package]] name = "deflate64" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83ace6c86376be0b6cdcf3fb41882e81d94b31587573d1cfa9d01cd06bba210d" +checksum = "da692b8d1080ea3045efaab14434d40468c3d8657e42abddfffca87b428f4c1b" + +[[package]] +name = "delegate" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee5df75c70b95bd3aacc8e2fd098797692fb1d54121019c4de481e42f04c8a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] [[package]] name = "deranged" @@ -1718,51 +1698,38 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "derive_builder" -version = "0.12.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7" dependencies = [ "derive_builder_macro", ] [[package]] name = "derive_builder_core" -version = "0.12.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d" dependencies = [ - "darling 0.14.4", + "darling", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.75", ] [[package]] name = "derive_builder_macro" -version = "0.12.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b" dependencies = [ "derive_builder_core", - "syn 1.0.109", -] - -[[package]] -name = "derive_more" -version = "0.99.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" -dependencies = [ - "convert_case 0.4.0", - "proc-macro2", - "quote", - "rustc_version", - "syn 1.0.109", + "syn 2.0.75", ] [[package]] @@ -1784,19 +1751,19 @@ dependencies = [ [[package]] name = "display-error-chain" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f77af9e75578c1ab34f5f04545a8b05be0c36fbd7a9bb3cf2d2a971e435fdbb9" +checksum = "7d305e5a3904ee14166439a70feef04853c1234226dbb27ede127b88dc5a4a9d" [[package]] name = "displaydoc" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -1850,11 +1817,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d8b5d72ddb5add27d0e071039643c894ebdfd14fac88eed61d9111af8395d27" dependencies = [ "Inflector", - "darling 0.20.9", + "darling", "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", "thiserror", ] @@ -1882,17 +1849,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.66", -] - -[[package]] -name = "env_logger" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -dependencies = [ - "log", - "regex", + "syn 2.0.75", ] [[package]] @@ -1943,7 +1900,7 @@ name = "examples" version = "0.10.0" dependencies = [ "chrono", - "itertools 0.12.1", + "itertools 0.13.0", "raphtory", "regex", "serde", @@ -2000,12 +1957,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.30" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.8.0", ] [[package]] @@ -2039,12 +1996,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - [[package]] name = "futures" version = "0.3.30" @@ -2101,7 +2052,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -2140,20 +2091,6 @@ dependencies = [ "slab", ] -[[package]] -name = "generator" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186014d53bc231d0090ef8d6f03e0920c54d85a5ed22f4f2f74315ec56cf83fb" -dependencies = [ - "cc", - "cfg-if", - "libc", - "log", - "rustversion", - "windows", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -2164,17 +2101,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "getrandom" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.9.0+wasi-snapshot-preview1", -] - [[package]] name = "getrandom" version = "0.2.15" @@ -2184,7 +2110,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] @@ -2206,9 +2132,9 @@ checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "glam" -version = "0.25.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "151665d9be52f9bb40fc7966565d39666f2d1e69233571b71b87791c7e0528b3" +checksum = "c28091a37a5d09b555cb6628fd954da299b536433834f5b8e59eba78e0cbbf8a" [[package]] name = "glob" @@ -2237,9 +2163,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", "bytes", @@ -2338,6 +2264,12 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hermit-abi" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" + [[package]] name = "hex" version = "0.4.3" @@ -2403,9 +2335,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http 1.1.0", @@ -2420,15 +2352,15 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "pin-project-lite", ] [[package]] name = "httparse" -version = "1.9.3" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0e7a4dd27b9476dc40cb050d3632d3bba3a70ddbff012285f7f8559a1e7e545" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" [[package]] name = "httpdate" @@ -2444,9 +2376,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.29" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", @@ -2468,22 +2400,23 @@ dependencies = [ [[package]] name = "hyper" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.5", + "h2 0.4.6", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", "pin-project-lite", "smallvec", "tokio", + "want", ] [[package]] @@ -2494,25 +2427,49 @@ checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http 0.2.12", - "hyper 0.14.29", - "rustls", + "hyper 0.14.30", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.4.1", + "hyper-util", + "rustls 0.23.12", + "rustls-native-certs", + "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", + "tower-service", + "webpki-roots 0.26.3", ] [[package]] name = "hyper-util" -version = "0.1.5" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" dependencies = [ "bytes", + "futures-channel", "futures-util", "http 1.1.0", - "http-body 1.0.0", - "hyper 1.3.1", + "http-body 1.0.1", + "hyper 1.4.1", "pin-project-lite", + "socket2", "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -2526,7 +2483,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows-core 0.52.0", + "windows-core", ] [[package]] @@ -2538,124 +2495,6 @@ dependencies = [ "cc", ] -[[package]] -name = "icu_collections" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locid" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - -[[package]] -name = "icu_normalizer" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "utf16_iter", - "utf8_iter", - "write16", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" - -[[package]] -name = "icu_properties" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locid_transform", - "icu_properties_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" - -[[package]] -name = "icu_provider" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_provider_macros", - "stable_deref_trait", - "tinystr", - "writeable", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "ident_case" version = "1.0.1" @@ -2664,21 +2503,19 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "1.0.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ - "icu_normalizer", - "icu_properties", - "smallvec", - "utf8_iter", + "unicode-bidi", + "unicode-normalization", ] [[package]] name = "indexmap" -version = "2.2.6" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -2732,20 +2569,20 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "is-terminal" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ - "hermit-abi", + "hermit-abi 0.4.0", "libc", "windows-sys 0.52.0", ] [[package]] name = "is_terminal_polyfill" -version = "1.70.0" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" [[package]] name = "itertools" @@ -2756,6 +2593,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -2765,6 +2611,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -2773,9 +2628,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] @@ -2796,9 +2651,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -2816,13 +2671,14 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "8.3.0" +version = "9.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" +checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f" dependencies = [ "base64 0.21.7", + "js-sys", "pem", - "ring 0.16.20", + "ring", "serde", "serde_json", "simple_asn1", @@ -2841,9 +2697,9 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "levenshtein_automata" @@ -2917,9 +2773,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" [[package]] name = "libm" @@ -2939,12 +2795,6 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" -[[package]] -name = "litemap" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" - [[package]] name = "lock_api" version = "0.4.12" @@ -2964,38 +2814,24 @@ checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" [[package]] name = "log" -version = "0.4.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" - -[[package]] -name = "loom" -version = "0.7.2" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" -dependencies = [ - "cfg-if", - "generator", - "pin-utils", - "scoped-tls", - "tracing", - "tracing-subscriber", -] +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lru" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" +checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" dependencies = [ "hashbrown 0.14.5", ] [[package]] name = "lz4" -version = "1.25.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6eab492fe7f8651add23237ea56dbf11b3c4ff762ab83d40a47f11433421f91" +checksum = "958b4caa893816eea05507c20cfe47574a43d9a697138a7872990bba8a0ece68" dependencies = [ "libc", "lz4-sys", @@ -3003,9 +2839,9 @@ dependencies = [ [[package]] name = "lz4-sys" -version = "1.9.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9764018d143cc854c9f17f0b907de70f14393b1f502da6375dce70f00514eb3" +checksum = "109de74d5d2353660401699a4174a4ff23fcc649caf553df71933c7fb45ad868" dependencies = [ "cc", "libc", @@ -3072,9 +2908,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.3" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0d8b92cd8358e8d229c11df9358decae64d137c5be540952c5ca7b25aea768" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" @@ -3102,14 +2938,24 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" -version = "2.0.4" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" dependencies = [ "mime", "unicase", ] +[[package]] +name = "minicov" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c71e683cd655513b99affab7d317deb690528255a0d5f717f1024093c12b169" +dependencies = [ + "cc", + "walkdir", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -3118,29 +2964,39 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" -version = "0.8.11" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ + "hermit-abi 0.3.9", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.48.0", + "wasi", + "windows-sys 0.52.0", ] [[package]] name = "moka" -version = "0.12.7" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0d88686dc561d743b40de8269b26eaf0dc58781bde087b0984646602021d08" +checksum = "32cf62eb4dd975d2dde76432fb1075c49e3ee2331cf36f1f8fd4b66550d32b6f" dependencies = [ "crossbeam-channel", "crossbeam-epoch", @@ -3169,8 +3025,7 @@ dependencies = [ "httparse", "memchr", "mime", - "spin 0.9.8", - "tokio", + "spin", "version_check", ] @@ -3210,42 +3065,49 @@ checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" [[package]] name = "neo4rs" -version = "0.6.2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f1d8019cfd20115fd664e4e1ecdf144ab4fce8bf7f7ddc7b6523a8835a770a2" +checksum = "43dd99fe7dbc68f754759874d83ec2ca43a61ab7d51c10353d024094805382be" dependencies = [ "async-trait", + "backoff", "bytes", "chrono", + "chrono-tz", "deadpool", + "delegate", "futures", "log", "neo4rs-macros", + "paste", "pin-project-lite", + "rustls-native-certs", + "rustls-pemfile 2.1.3", + "serde", "thiserror", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.0", "url", - "webpki-roots 0.23.1", + "webpki-roots 0.26.3", ] [[package]] name = "neo4rs-macros" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cf52bfa6042343585f458f38f18094a53e8d8c417221867918e9f0a6885f42a" +checksum = "53a0d57c55d2d1dc62a2b1d16a0a1079eb78d67c36bdf468d582ab4482ec7002" dependencies = [ "quote", - "syn 1.0.109", + "syn 2.0.75", ] [[package]] name = "nix" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cfg-if", "cfg_aliases", "libc", @@ -3287,9 +3149,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", @@ -3357,7 +3219,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi", + "hermit-abi 0.3.9", "libc", ] @@ -3369,10 +3231,10 @@ checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" dependencies = [ "base64 0.13.1", "chrono", - "getrandom 0.2.15", + "getrandom", "http 0.2.12", - "rand 0.8.5", - "reqwest", + "rand", + "reqwest 0.11.27", "serde", "serde_json", "serde_path_to_error", @@ -3383,9 +3245,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.0" +version = "0.36.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" +checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" dependencies = [ "memchr", ] @@ -3419,18 +3281,15 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oneshot" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071d1cf3298ad8e543dca18217d198cb6a3884443d204757b9624b935ef09fa0" -dependencies = [ - "loom", -] +checksum = "e296cf87e61c9cfc1a61c3c63a0f7f286ed4554e0e22be84e8a38e1d264a2a29" [[package]] name = "oorandom" -version = "11.1.3" +version = "11.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" [[package]] name = "opaque-debug" @@ -3446,25 +3305,23 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "opentelemetry" -version = "0.21.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" +checksum = "1b69a91d4893e713e06f724597ad630f1fa76057a5e1026c0ca67054a9032a76" dependencies = [ "futures-core", "futures-sink", - "indexmap", "js-sys", "once_cell", "pin-project-lite", "thiserror", - "urlencoding", ] [[package]] name = "opentelemetry-jaeger" -version = "0.20.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e617c66fd588e40e0dbbd66932fdc87393095b125d4459b1a3a10feb1712f8a1" +checksum = "501b471b67b746d9a07d4c29f8be00f952d1a2eca356922ede0098cbaddff19f" dependencies = [ "async-trait", "futures-core", @@ -3478,30 +3335,27 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.13.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" -dependencies = [ - "opentelemetry", -] +checksum = "1869fb4bb9b35c5ba8a1e40c9b128a7b4c010d07091e864a29da19e4fe2ca4d7" [[package]] name = "opentelemetry_sdk" -version = "0.21.2" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" +checksum = "ae312d58eaa90a82d2e627fd86e075cf5230b3f11794e2ed74199ebbe572d4fd" dependencies = [ "async-trait", - "crossbeam-channel", "futures-channel", "futures-executor", "futures-util", "glob", + "lazy_static", "once_cell", "opentelemetry", - "ordered-float 4.2.0", + "ordered-float 4.2.2", "percent-encoding", - "rand 0.8.5", + "rand", "thiserror", "tokio", "tokio-stream", @@ -3518,9 +3372,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.2.0" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" +checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6" dependencies = [ "num-traits", ] @@ -3557,7 +3411,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -3595,7 +3449,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -3672,11 +3526,12 @@ dependencies = [ [[package]] name = "pem" -version = "1.1.1" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8" +checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" dependencies = [ - "base64 0.13.1", + "base64 0.22.1", + "serde", ] [[package]] @@ -3687,9 +3542,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" +checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" dependencies = [ "memchr", "thiserror", @@ -3698,9 +3553,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" +checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" dependencies = [ "pest", "pest_generator", @@ -3708,22 +3563,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" +checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "pest_meta" -version = "2.7.10" +version = "2.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" +checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" dependencies = [ "once_cell", "pest", @@ -3766,7 +3621,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared", - "rand 0.8.5", + "rand", ] [[package]] @@ -3778,6 +3633,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.75", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -3835,9 +3710,9 @@ dependencies = [ [[package]] name = "poem" -version = "3.0.1" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88b6912ed1e8833d7c22c9c986c517f4518d7d37e3c04566d917c789aaea591" +checksum = "f1ba1c27f8f89e1bccdda0c680f72790545a11a8d8555819472f5839d7a8ca9d" dependencies = [ "base64 0.22.1", "bytes", @@ -3847,29 +3722,24 @@ dependencies = [ "headers", "http 1.1.0", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.1", "hyper-util", "mime", - "multer", "nix", "parking_lot", "percent-encoding", "pin-project-lite", "poem-derive", - "quick-xml", "regex", "rfc7239", "serde", "serde_json", "serde_urlencoded", - "serde_yaml", "smallvec", "sync_wrapper 1.0.1", - "tempfile", "thiserror", "time", "tokio", - "tokio-stream", "tokio-tungstenite", "tokio-util", "tracing", @@ -3878,57 +3748,14 @@ dependencies = [ [[package]] name = "poem-derive" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2b961d58a6c53380c20236394381d9292fda03577f902b158f1638932964dcf" -dependencies = [ - "proc-macro-crate 3.1.0", - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "poem-openapi" -version = "5.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6445b50be2e26f142d4e554d15773fc1e7510b994083c9625a65eba0d3f4287" -dependencies = [ - "base64 0.22.1", - "bytes", - "derive_more", - "futures-util", - "indexmap", - "mime", - "num-traits", - "poem", - "poem-openapi-derive", - "quick-xml", - "regex", - "serde", - "serde_json", - "serde_urlencoded", - "serde_yaml", - "thiserror", - "tokio", -] - -[[package]] -name = "poem-openapi-derive" -version = "5.0.2" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e890165626ff447a1ff3d6f2293e6ccacbf7fcbdd4c94086aa548de655735b03" +checksum = "a62fea1692d80a000126f9b28d865012a160b80000abb53ccf152b428222c155" dependencies = [ - "darling 0.20.9", - "http 1.1.0", - "indexmap", - "mime", "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "regex", - "syn 2.0.66", - "thiserror", + "syn 2.0.75", ] [[package]] @@ -3950,7 +3777,7 @@ dependencies = [ "ethnum", "fast-float", "foreign_vec", - "getrandom 0.2.15", + "getrandom", "hashbrown 0.14.5", "itoa", "multiversion", @@ -4048,9 +3875,9 @@ version = "0.10.0" [[package]] name = "portable-atomic" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" [[package]] name = "powerfmt" @@ -4060,9 +3887,12 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "pretty_assertions" @@ -4081,7 +3911,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -4105,9 +3935,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -4120,24 +3950,24 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", "version_check", "yansi 1.0.1", ] [[package]] name = "proptest" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" +checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.5.0", + "bitflags 2.6.0", "lazy_static", "num-traits", - "rand 0.8.5", - "rand_chacha 0.3.1", + "rand", + "rand_chacha", "rand_xorshift", "regex-syntax 0.8.4", "rusty-fork", @@ -4147,9 +3977,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.12.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +checksum = "e13db3d3fde688c61e2446b4d843bc27a7e8af269a69440c0308021dc92333cc" dependencies = [ "bytes", "prost-derive", @@ -4157,13 +3987,13 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.12.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +checksum = "5bb182580f71dd070f88d01ce3de9f4da5021db7115d2e1c3605a754153b77c1" dependencies = [ "bytes", "heck 0.5.0", - "itertools 0.12.1", + "itertools 0.13.0", "log", "multimap", "once_cell", @@ -4172,28 +4002,28 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.66", + "syn 2.0.75", "tempfile", ] [[package]] name = "prost-derive" -version = "0.12.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +checksum = "18bec9b0adc4eba778b33684b7ba3e7137789434769ee3ce3930463ef904cfca" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "prost-types" -version = "0.12.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +checksum = "cee5168b05f49d4b0ca581206eb14a7b22fafd963efe729ac48eb03266e25cc2" dependencies = [ "prost", ] @@ -4256,7 +4086,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -4269,7 +4099,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -4288,7 +4118,7 @@ dependencies = [ "libc", "once_cell", "raw-cpuid", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "web-sys", "winapi", ] @@ -4299,37 +4129,15 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quick-xml" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" -dependencies = [ - "memchr", - "serde", -] - -[[package]] -name = "quickcheck" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44883e74aa97ad63db83c4bf8ca490f02b2fc02f92575e720c8551e843c945f" -dependencies = [ - "env_logger 0.7.1", - "log", - "rand 0.7.3", - "rand_core 0.5.1", -] - [[package]] name = "quickcheck" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "588f6378e4dd99458b60ec275b4477add41ce4fa9f64dcba6f15adccb19b50d6" dependencies = [ - "env_logger 0.8.4", + "env_logger", "log", - "rand 0.8.5", + "rand", ] [[package]] @@ -4344,59 +4152,71 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.36" +name = "quinn" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" dependencies = [ - "proc-macro2", + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash 2.0.0", + "rustls 0.23.12", + "socket2", + "thiserror", + "tokio", + "tracing", ] [[package]] -name = "rand" -version = "0.4.6" +name = "quinn-proto" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" dependencies = [ - "fuchsia-cprng", - "libc", - "rand_core 0.3.1", - "rdrand", - "winapi", + "bytes", + "rand", + "ring", + "rustc-hash 2.0.0", + "rustls 0.23.12", + "slab", + "thiserror", + "tinyvec", + "tracing", ] [[package]] -name = "rand" -version = "0.7.3" +name = "quinn-udp" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" dependencies = [ - "getrandom 0.1.16", "libc", - "rand_chacha 0.2.2", - "rand_core 0.5.1", - "rand_hc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", ] [[package]] -name = "rand" -version = "0.8.5" +name = "quote" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ - "libc", - "rand_chacha 0.3.1", - "rand_core 0.6.4", + "proc-macro2", ] [[package]] -name = "rand_chacha" -version = "0.2.2" +name = "rand" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ - "ppv-lite86", - "rand_core 0.5.1", + "libc", + "rand_chacha", + "rand_core", ] [[package]] @@ -4406,31 +4226,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.4", -] - -[[package]] -name = "rand_core" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" -dependencies = [ - "rand_core 0.4.2", -] - -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom 0.1.16", + "rand_core", ] [[package]] @@ -4439,7 +4235,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.15", + "getrandom", ] [[package]] @@ -4449,16 +4245,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", - "rand 0.8.5", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core 0.5.1", + "rand", ] [[package]] @@ -4467,23 +4254,20 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" dependencies = [ - "rand_core 0.6.4", + "rand_core", ] [[package]] name = "raphtory" version = "0.10.0" dependencies = [ - "ahash", "async-openai", "async-trait", "bincode", - "bytemuck", - "bytes", "bzip2", "chrono", "csv", - "dashmap", + "dashmap 6.0.1", "display-error-chain", "dotenv", "either", @@ -4491,7 +4275,7 @@ dependencies = [ "flate2", "futures-util", "glam", - "itertools 0.12.1", + "itertools 0.13.0", "kdam", "lock_api", "memmap2", @@ -4500,12 +4284,11 @@ dependencies = [ "num-integer", "num-traits", "once_cell", - "ordered-float 4.2.0", + "ordered-float 4.2.2", "ouroboros", "parking_lot", "polars-arrow", "polars-parquet", - "polars-utils", "pometry-storage", "pretty_assertions", "proptest", @@ -4514,27 +4297,23 @@ dependencies = [ "prost-types", "pyo3", "quad-rand", - "quickcheck 1.0.3", + "quickcheck", "quickcheck_macros", - "rand 0.8.5", + "rand", "rand_distr", "raphtory-api", "rayon", "regex", - "reqwest", - "rpds", - "rustc-hash", + "reqwest 0.12.7", + "rustc-hash 2.0.0", "serde", "serde_json", "sorted_vector_map", "streaming-stats", "tantivy", - "tempdir", "tempfile", "thiserror", - "thread_local", "tokio", - "twox-hash", "zip", ] @@ -4543,17 +4322,17 @@ name = "raphtory-api" version = "0.10.0" dependencies = [ "chrono", - "dashmap", + "dashmap 6.0.1", "lock_api", "num-traits", "parking_lot", "proptest", "pyo3", - "quickcheck 1.0.3", + "quickcheck", "quickcheck_macros", - "rand 0.8.5", + "rand", "rayon", - "rustc-hash", + "rustc-hash 2.0.0", "serde", "twox-hash", ] @@ -4562,21 +4341,17 @@ dependencies = [ name = "raphtory-benchmark" version = "0.10.0" dependencies = [ - "chrono", "clap", "criterion", "csv", "flate2", - "polars-arrow", "pometry-storage", - "rand 0.8.5", + "rand", "raphtory", "raphtory-api", - "raphtory-graphql", "rayon", "sorted_vector_map", "tempfile", - "tokio", ] [[package]] @@ -4591,7 +4366,7 @@ dependencies = [ "clap", "datafusion", "futures", - "itertools 0.12.1", + "itertools 0.13.0", "lazy_static", "pest", "pest_derive", @@ -4599,7 +4374,6 @@ dependencies = [ "pometry-storage", "pretty_assertions", "proptest", - "rand 0.8.5", "raphtory", "rayon", "serde", @@ -4616,17 +4390,14 @@ version = "0.10.0" dependencies = [ "async-graphql", "async-graphql-poem", - "async-stream", - "base64 0.21.7", + "base64 0.22.1", "base64-compat", - "bincode", "chrono", "config", "crossbeam-channel", - "dotenv", "dynamic-graphql", "futures-util", - "itertools 0.12.1", + "itertools 0.13.0", "jsonwebtoken", "moka", "oauth2", @@ -4634,21 +4405,18 @@ dependencies = [ "opentelemetry", "opentelemetry-jaeger", "opentelemetry_sdk", - "ordered-float 4.2.0", + "ordered-float 4.2.2", "parking_lot", "poem", - "poem-openapi", "pyo3", "raphtory", "raphtory-api", - "reqwest", + "reqwest 0.12.7", "serde", "serde_json", "tempfile", "thiserror", - "time", "tokio", - "toml", "tracing", "tracing-opentelemetry", "tracing-subscriber", @@ -4661,18 +4429,10 @@ dependencies = [ name = "raphtory-pymodule" version = "0.10.0" dependencies = [ - "async-graphql", - "crossbeam-channel", - "dynamic-graphql", - "itertools 0.12.1", "pyo3", "pyo3-build-config", "raphtory", "raphtory-graphql", - "reqwest", - "serde", - "serde_json", - "tokio", ] [[package]] @@ -4687,11 +4447,11 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.0.2" +version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e29830cbb1290e404f24c73af91c5d8d631ce7e128691e9477556b540cd01ecd" +checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] @@ -4714,29 +4474,20 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "redox_syscall" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] name = "regex" -version = "1.10.5" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ "aho-corasick", "memchr", @@ -4776,15 +4527,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" -[[package]] -name = "remove_dir_all" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] - [[package]] name = "reqwest" version = "0.11.27" @@ -4799,42 +4541,85 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", - "hyper-rustls", + "hyper 0.14.30", + "hyper-rustls 0.24.2", "ipnet", "js-sys", "log", "mime", - "mime_guess", "once_cell", "percent-encoding", "pin-project-lite", - "rustls", - "rustls-native-certs", - "rustls-pemfile", + "rustls 0.21.12", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", "system-configuration", "tokio", - "tokio-rustls", - "tokio-util", + "tokio-rustls 0.24.1", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", "web-sys", "webpki-roots 0.25.4", "winreg", ] +[[package]] +name = "reqwest" +version = "0.12.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.4.1", + "hyper-rustls 0.27.2", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "mime_guess", + "once_cell", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls 0.23.12", + "rustls-native-certs", + "rustls-pemfile 2.1.3", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 1.0.1", + "tokio", + "tokio-rustls 0.26.0", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "webpki-roots 0.26.3", + "windows-registry", +] + [[package]] name = "reqwest-eventsource" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f03f570355882dd8d15acc3a313841e6e90eddbc76a93c748fd82cc13ba9f51" +checksum = "632c55746dbb44275691640e7b40c907c16a2dc1a5842aa98aaec90da6ec6bde" dependencies = [ "eventsource-stream", "futures-core", @@ -4842,7 +4627,7 @@ dependencies = [ "mime", "nom", "pin-project-lite", - "reqwest", + "reqwest 0.12.7", "thiserror", ] @@ -4858,22 +4643,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b106a85eeb5b0336d16d6a20eab857f92861d4fbb1eb9a239866fb98fb6a1063" dependencies = [ - "uncased", -] - -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", + "uncased", ] [[package]] @@ -4884,10 +4654,10 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.15", + "getrandom", "libc", - "spin 0.9.8", - "untrusted 0.9.0", + "spin", + "untrusted", "windows-sys 0.52.0", ] @@ -4898,21 +4668,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" dependencies = [ "base64 0.21.7", - "bitflags 2.5.0", + "bitflags 2.6.0", "serde", "serde_derive", ] -[[package]] -name = "rpds" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0e15515d3ce3313324d842629ea4905c25a13f81953eadb88f85516f59290a4" -dependencies = [ - "archery", - "serde", -] - [[package]] name = "rust-ini" version = "0.19.0" @@ -4945,6 +4705,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc_version" version = "0.4.0" @@ -4960,7 +4726,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", @@ -4974,19 +4740,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", - "ring 0.17.8", + "ring", "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki 0.102.6", + "subtle", + "zeroize", +] + [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "04182dffc9091a404e0fc069ea5cd60e5b866c3adf881eff99a32d048242dffa" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 2.1.3", + "rustls-pki-types", "schannel", "security-framework", ] @@ -5001,23 +4782,40 @@ dependencies = [ ] [[package]] -name = "rustls-webpki" -version = "0.100.3" +name = "rustls-pemfile" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6a5fc258f1c1276dfe3016516945546e2d5383911efc0fc4f1cdc5df3a4ae3" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ - "ring 0.16.20", - "untrusted 0.7.1", + "base64 0.22.1", + "rustls-pki-types", ] +[[package]] +name = "rustls-pki-types" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" + [[package]] name = "rustls-webpki" version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", + "ring", + "untrusted", +] + +[[package]] +name = "rustls-webpki" +version = "0.102.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -5080,8 +4878,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.8", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] @@ -5096,11 +4894,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", @@ -5109,9 +4907,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" dependencies = [ "core-foundation-sys", "libc", @@ -5131,9 +4929,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.203" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" dependencies = [ "serde_derive", ] @@ -5151,22 +4949,23 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -5183,9 +4982,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" dependencies = [ "serde", ] @@ -5202,19 +5001,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "sha1" version = "0.10.6" @@ -5246,6 +5032,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -5360,19 +5152,14 @@ dependencies = [ [[package]] name = "sorted_vector_map" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "834e4ea0bc5317590acf76b0cd074e1724668da00c82f58e653c7fde0e755d52" +checksum = "d9167648c2ababdbe45294fe7f7b0ab56555fd754990a7637a5a420774461368" dependencies = [ - "quickcheck 0.9.2", + "itertools 0.11.0", + "quickcheck", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -5397,7 +5184,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -5461,12 +5248,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - [[package]] name = "strsim" version = "0.11.1" @@ -5475,9 +5256,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ "strum_macros", ] @@ -5492,14 +5273,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "subtle" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -5514,9 +5295,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.66" +version = "2.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" dependencies = [ "proc-macro2", "quote", @@ -5538,17 +5319,6 @@ dependencies = [ "futures-core", ] -[[package]] -name = "synstructure" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "system-configuration" version = "0.5.1" @@ -5608,7 +5378,7 @@ dependencies = [ "rayon", "regex", "rust-stemmers", - "rustc-hash", + "rustc-hash 1.1.0", "serde", "serde_json", "sketches-ddsketch", @@ -5725,30 +5495,21 @@ checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" [[package]] name = "target-lexicon" -version = "0.12.14" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" - -[[package]] -name = "tempdir" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" -dependencies = [ - "rand 0.4.6", - "remove_dir_all", -] +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.10.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5763,22 +5524,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.61" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.61" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -5853,16 +5614,6 @@ dependencies = [ "crunchy", ] -[[package]] -name = "tinystr" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" -dependencies = [ - "displaydoc", - "zerovec", -] - [[package]] name = "tinytemplate" version = "1.2.1" @@ -5873,34 +5624,48 @@ dependencies = [ "serde_json", ] +[[package]] +name = "tinyvec" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" -version = "1.38.0" +version = "1.39.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -5909,7 +5674,18 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.12", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls 0.23.12", + "rustls-pki-types", "tokio", ] @@ -5926,9 +5702,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.21.0" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" +checksum = "c6989540ced10490aaf14e6bad2e3d33728a2813310a0c71d1574304c49631cd" dependencies = [ "futures-util", "log", @@ -5952,21 +5728,21 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.14" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.14", + "toml_edit 0.22.20", ] [[package]] name = "toml_datetime" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] @@ -5995,22 +5771,43 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.14" +version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ "indexmap", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.13", + "winnow 0.6.18", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", ] +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -6031,7 +5828,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -6057,9 +5854,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.22.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" +checksum = "f68803492bf28ab40aeccaecc7021096bd256baf7ca77c3d425d89b35a7be4e4" dependencies = [ "js-sys", "once_cell", @@ -6093,9 +5890,9 @@ dependencies = [ [[package]] name = "triomphe" -version = "0.1.12" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b2cb4fbb9995eeb36ac86fadf24031ccd58f99d6b4b2d7b911db70bddb80d90" +checksum = "859eb650cfee7434994602c3a68b25d77ad9e68c8a6cd491616ef86661382eb3" [[package]] name = "try-lock" @@ -6105,9 +5902,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.21.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" +checksum = "6e2e2ce1e47ed2994fd43b04c8f618008d4cabdd5ee34027cf14f9d918edd9c8" dependencies = [ "byteorder", "bytes", @@ -6115,10 +5912,9 @@ dependencies = [ "http 1.1.0", "httparse", "log", - "rand 0.8.5", + "rand", "sha1", "thiserror", - "url", "utf-8", ] @@ -6129,7 +5925,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand 0.8.5", + "rand", "static_assertions", ] @@ -6169,12 +5965,27 @@ dependencies = [ "version_check", ] +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.11.0" @@ -6203,18 +6014,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" @@ -6223,9 +6022,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.1" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", @@ -6233,36 +6032,18 @@ dependencies = [ "serde", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - [[package]] name = "utf-8" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8-ranges" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - [[package]] name = "utf8parse" version = "0.2.2" @@ -6271,11 +6052,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ - "getrandom 0.2.15", + "getrandom", "serde", ] @@ -6287,9 +6068,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "wait-timeout" @@ -6319,12 +6100,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -6333,34 +6108,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" dependencies = [ "cfg-if", "js-sys", @@ -6370,9 +6146,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6380,31 +6156,32 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "wasm-bindgen-test" -version = "0.3.42" +version = "0.3.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9bf62a58e0780af3e852044583deee40983e5886da43a271dd772379987667b" +checksum = "68497a05fb21143a08a7d24fc81763384a3072ee43c44e86aad1744d6adef9d9" dependencies = [ "console_error_panic_hook", "js-sys", + "minicov", "scoped-tls", "wasm-bindgen", "wasm-bindgen-futures", @@ -6413,13 +6190,13 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.42" +version = "0.3.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f89739351a2e03cb94beb799d47fb2cac01759b40ec441f7de39b00cbf7ef0" +checksum = "4b8220be1fa9e4c889b30fd207d4906657e7e90b12e0e6b0c8b8d8709f5de021" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] @@ -6437,9 +6214,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", @@ -6447,9 +6224,9 @@ dependencies = [ [[package]] name = "web-time" -version = "0.2.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ "js-sys", "wasm-bindgen", @@ -6457,18 +6234,18 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.23.1" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" -dependencies = [ - "rustls-webpki 0.100.3", -] +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "webpki-roots" -version = "0.25.4" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +dependencies = [ + "rustls-pki-types", +] [[package]] name = "wildmatch" @@ -6494,11 +6271,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6507,42 +6284,43 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.54.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" -dependencies = [ - "windows-core 0.54.0", - "windows-targets 0.52.5", -] - [[package]] name = "windows-core" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] -name = "windows-core" -version = "0.54.0" +name = "windows-registry" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ "windows-result", - "windows-targets 0.52.5", + "windows-strings", + "windows-targets 0.52.6", ] [[package]] name = "windows-result" -version = "0.1.2" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ - "windows-targets 0.52.5", + "windows-result", + "windows-targets 0.52.6", ] [[package]] @@ -6560,7 +6338,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", ] [[package]] @@ -6580,18 +6367,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -6602,9 +6389,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -6614,9 +6401,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -6626,15 +6413,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -6644,9 +6431,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -6656,9 +6443,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -6668,9 +6455,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -6680,9 +6467,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" @@ -6695,9 +6482,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.13" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" dependencies = [ "memchr", ] @@ -6712,18 +6499,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - -[[package]] -name = "writeable" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" - [[package]] name = "xz2" version = "0.1.7" @@ -6754,69 +6529,25 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" -[[package]] -name = "yoke" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", - "synstructure", -] - [[package]] name = "zerocopy" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - -[[package]] -name = "zerofrom" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.4" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", - "synstructure", + "syn 2.0.75", ] [[package]] @@ -6836,36 +6567,14 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", -] - -[[package]] -name = "zerovec" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", + "syn 2.0.75", ] [[package]] name = "zip" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775a2b471036342aa69bc5a602bc889cb0a06cda00477d0c69566757d5553d39" +checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" dependencies = [ "aes", "arbitrary", @@ -6881,7 +6590,7 @@ dependencies = [ "lzma-rs", "memchr", "pbkdf2", - "rand 0.8.5", + "rand", "sha1", "thiserror", "time", @@ -6906,27 +6615,27 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.1.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index c8b715eed7..5cf60e1008 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,18 +38,18 @@ pometry-storage = { version = ">=0.8.1", path = "pometry-storage" } #[private-storage] # pometry-storage = { path = "pometry-storage-private", package = "pometry-storage-private" } async-graphql = { version = "7.0.5", features = ["dynamic-schema"] } +bincode = "1.3.3" async-graphql-poem = "7.0.5" dynamic-graphql = "0.9.0" -reqwest = { version = "0.11.22", default-features = false, features = ["rustls-tls"] } +reqwest = { version = "0.12.7", default-features = false, features = ["rustls-tls", "multipart", "json"] } serde = { version = "1.0.197", features = ["derive", "rc"] } serde_json = "1.0.114" pyo3 = { version = "0.20.0", features = ["multiple-pymethods", "chrono"] } -pyo3-asyncio = { version = "0.20.0", features = ["tokio-runtime"] } pyo3-build-config = "0.20.0" -itertools = "0.12.1" +itertools = "0.13.0" rand = "0.8.5" rayon = "1.8.1" -sorted_vector_map = "0.1.0" +sorted_vector_map = "0.2.0" tokio = { version = "1.36.0", features = ["full"] } once_cell = "1.19.0" parking_lot = { version = "0.12.1", features = ["serde", "arc_lock", "send_guard"] } @@ -58,30 +58,28 @@ chrono = { version = "0.4.38", features = ["serde"] } tempfile = "3.10.0" futures-util = "0.3.30" thiserror = "1.0.57" -bincode = "1.3.3" dotenv = "0.15.0" csv = "1.3.0" flate2 = "1.0.28" regex = "1.10.3" -genawaiter = "0.99.1" num-traits = "0.2.18" num-integer = "0.1" rand_distr = "0.4.3" -rustc-hash = "1.1.0" +rustc-hash = "2.0.0" twox-hash = "1.6.3" lock_api = { version = "0.4.11", features = ["arc_lock", "serde"] } -dashmap = { version = "5.5.3", features = ["serde"] } +dashmap = { version = "6.0.1", features = ["serde"] } enum_dispatch = "0.3.12" -glam = "0.25.0" +glam = "0.29.0" quad-rand = "0.2.1" zip = "2.1.3" -neo4rs = "0.6.2" +neo4rs = "0.8.0" bzip2 = "0.4.4" tantivy = "0.22" async-trait = "0.1.77" -async-openai = "0.17.1" +async-openai = "0.23.4" oauth2 = "4.0" -jsonwebtoken = "8.0" +jsonwebtoken = "9.3.0" num = "0.4.1" display-error-chain = "0.2.0" polars-arrow = "0.39.2" @@ -91,28 +89,23 @@ kdam = { version = "0.5.1" } pretty_assertions = "1.4.0" quickcheck = "1.0.3" quickcheck_macros = "1.0.0" -tempdir = "0.3.7" streaming-stats = "0.2.3" proptest = "1.4.0" criterion = "0.5.1" crossbeam-channel = "0.5.11" -base64 = "0.21.7" -poem = "3.0.1" -poem-openapi = "5.0.2" -async-stream = "0.3.5" -opentelemetry = "0.21.0" -opentelemetry_sdk = { version = "0.21.0", features = ["rt-tokio"] } -opentelemetry-jaeger = { version = "0.20.0", features = ["rt-tokio"] } +base64 = "0.22.1" +poem = { version = "3.0.1", features = ["cookie"] } +opentelemetry = "0.23.0" +opentelemetry_sdk = { version = "0.23.0", features = ["rt-tokio"] } +opentelemetry-jaeger = { version = "0.22.0", features = ["rt-tokio"] } tracing = "0.1.37" -tracing-opentelemetry = "0.22.0" +tracing-opentelemetry = "0.24.0" tracing-subscriber = { version = "0.3.16", features = ["std", "env-filter"] } walkdir = "2" uuid = { version = "1.0", features = ["v4"] } config = "0.14.0" either = "=1.11.0" -toml = "0.8.10" clap = { version = "4.3.11", features = ["derive"] } -tar = "0.4.38" wasm-bindgen = "0.2.91" serde-wasm-bindgen = "0.6.4" js-sys = "0.3.68" @@ -120,20 +113,15 @@ console_error_panic_hook = "0.1.7" wasm-bindgen-test = "0.3.41" memmap2 = { version = "0.9.4" } ahash = { version = "0.8.3", features = ["serde"] } -xxhash-rust = "0.8.7" strum = { version = "0.26.1", features = ["derive"] } bytemuck = { version = "1.15.0" } -rpds = { version = "1.1.0", features = ["serde"] } -thread_local = "1.1.8" ouroboros = "0.18.3" url = "2.2" base64-compat = { package = "base64-compat", version = "1.0.0" } -time = "0.3.36" -prost = "0.12" -prost-types = "0.12" -bytes = "1.6.0" -prost-build = "0.12" +prost = "0.13.1" +prost-types = "0.13.1" +prost-build = "0.13.1" lazy_static = "1.4.0" pest = "2.7.8" @@ -145,7 +133,6 @@ futures = "0.3" arrow = { version = "50" } arrow-buffer = { version = "50" } arrow-schema = { version = "50" } -arrow-data = { version = "50" } arrow-array = { version = "50" } moka = { version = "0.12.7", features = ["sync"] } diff --git a/examples/rust/src/bin/bench/main.rs b/examples/rust/src/bin/bench/main.rs index 783c6fb065..d6cb8ee065 100644 --- a/examples/rust/src/bin/bench/main.rs +++ b/examples/rust/src/bin/bench/main.rs @@ -36,7 +36,7 @@ fn main() { println!("Loading data"); let graph = if encoded_data_dir.exists() { let now = Instant::now(); - let g = Graph::load_from_file(encoded_data_dir.as_path(), false) + let g = Graph::decode(encoded_data_dir.as_path()) .expect("Failed to load graph from encoded data files"); println!( @@ -74,8 +74,7 @@ fn main() { now.elapsed().as_secs() ); - g.save_to_file(encoded_data_dir) - .expect("Failed to save graph"); + g.encode(encoded_data_dir).expect("Failed to save graph"); g }; diff --git a/examples/rust/src/bin/btc/main.rs b/examples/rust/src/bin/btc/main.rs index 994e336ff9..1364085c42 100644 --- a/examples/rust/src/bin/btc/main.rs +++ b/examples/rust/src/bin/btc/main.rs @@ -62,7 +62,7 @@ fn main() { let graph = if encoded_data_dir.exists() { let now = Instant::now(); - let g = Graph::load_from_file(encoded_data_dir.as_path(), false) + let g = Graph::decode(encoded_data_dir.as_path()) .expect("Failed to load graph from encoded data files"); println!( @@ -109,8 +109,7 @@ fn main() { now.elapsed().as_secs() ); - g.save_to_file(encoded_data_dir) - .expect("Failed to save graph"); + g.encode(encoded_data_dir).expect("Failed to save graph"); g }; diff --git a/examples/rust/src/bin/hulongbay/main.rs b/examples/rust/src/bin/hulongbay/main.rs index 1b44ecffb1..677bcb2073 100644 --- a/examples/rust/src/bin/hulongbay/main.rs +++ b/examples/rust/src/bin/hulongbay/main.rs @@ -63,7 +63,7 @@ pub fn loader(data_dir: &Path) -> Result> { let encoded_data_dir = data_dir.join("graphdb.bincode"); if encoded_data_dir.exists() { let now = Instant::now(); - let g = Graph::load_from_file(encoded_data_dir.as_path(), false)?; + let g = Graph::decode(encoded_data_dir.as_path())?; println!( "Loaded graph from path {} with {} nodes, {} edges, took {} seconds", @@ -104,7 +104,7 @@ pub fn loader(data_dir: &Path) -> Result> { now.elapsed().as_secs() ); - g.save_to_file(encoded_data_dir)?; + g.encode(encoded_data_dir)?; Ok(g) } } diff --git a/examples/rust/src/bin/lotr/main.rs b/examples/rust/src/bin/lotr/main.rs index d96fe32443..acaab631f3 100644 --- a/examples/rust/src/bin/lotr/main.rs +++ b/examples/rust/src/bin/lotr/main.rs @@ -37,7 +37,7 @@ fn main() { let graph = if encoded_data_dir.exists() { let now = Instant::now(); - let g = Graph::load_from_file(encoded_data_dir.as_path(), false) + let g = Graph::decode(encoded_data_dir.as_path()) .expect("Failed to load graph from encoded data files"); println!( @@ -90,8 +90,7 @@ fn main() { now.elapsed().as_secs() ); - g.save_to_file(encoded_data_dir) - .expect("Failed to save graph"); + g.encode(encoded_data_dir).expect("Failed to save graph"); g }; diff --git a/examples/rust/src/bin/pokec/main.rs b/examples/rust/src/bin/pokec/main.rs index 2be1a7737d..00edba5e11 100644 --- a/examples/rust/src/bin/pokec/main.rs +++ b/examples/rust/src/bin/pokec/main.rs @@ -22,7 +22,7 @@ fn main() { let data_dir = Path::new(args.get(1).expect("No data directory provided")); let g = if std::path::Path::new("/tmp/pokec").exists() { - Graph::load_from_file("/tmp/pokec", false).unwrap() + Graph::decode("/tmp/pokec").unwrap() } else { let g = Graph::new(); CsvLoader::new(data_dir) @@ -34,7 +34,7 @@ fn main() { }) .expect("Failed to load graph from encoded data files"); - g.save_to_file("/tmp/pokec") + g.encode("/tmp/pokec") .expect("Failed to save graph to file"); g }; diff --git a/pometry-storage-private b/pometry-storage-private index 84e0e1d484..942230d446 160000 --- a/pometry-storage-private +++ b/pometry-storage-private @@ -1 +1 @@ -Subproject commit 84e0e1d48482588b14626b491a2e366479963a58 +Subproject commit 942230d44640241016f5780c348ee795bc633568 diff --git a/python/Cargo.toml b/python/Cargo.toml index 80db325dd1..1cded88684 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -21,14 +21,6 @@ crate-type = ["cdylib"] pyo3 = { workspace = true } raphtory_core = { path = "../raphtory", version = "0.10.0", features = ["python", "search", "vectors", "proto"], package = "raphtory" } raphtory-graphql = { path = "../raphtory-graphql", version = "0.10.0",features = ["python"] } -serde_json = { workspace = true } -reqwest = { workspace = true, features = ["multipart"] } -tokio = { workspace = true } -crossbeam-channel = { workspace = true } -serde = { workspace = true } -async-graphql = { workspace = true } -dynamic-graphql = { workspace = true } -itertools = { workspace = true } [features] default = ["extension-module"] diff --git a/python/python/raphtory/__init__.pyi b/python/python/raphtory/__init__.pyi index e3384bcdf6..2bc0a333fd 100644 --- a/python/python/raphtory/__init__.pyi +++ b/python/python/raphtory/__init__.pyi @@ -1043,8 +1043,16 @@ class Graph: A GraphView object. """ - def bincode(self): - """Get bincode encoded graph""" + def cache(self, path): + """ + Write Graph to cache file and initialise the cache. + + Future updates are tracked. Use `write_updates` to persist them to the + cache file. If the file already exists its contents are overwritten. + + Arguments: + path (str): The path to the cache file + """ def count_edges(self): """ @@ -1077,6 +1085,18 @@ class Graph: GraphView: The layered view """ + @staticmethod + def deserialise(bytes): + """ + Load Graph from serialised bytes. + + Arguments: + bytes (Bytes): The serialised bytes to decode + + Returns: + Graph + """ + @property def earliest_date_time(self): """ @@ -1282,8 +1302,6 @@ class Graph: edges (List(edges)) - A vector of PyEdge objects representing the edges to be imported. force (boolean) - An optional boolean flag indicating whether to force the import of the edges. - Returns: - Result), GraphError> - A Result object which is Ok if the edges were successfully imported, and Err otherwise. """ def import_node(self, node, force=False): @@ -1313,8 +1331,6 @@ class Graph: nodes (List(Node))- A vector of PyNode objects representing the nodes to be imported. force (boolean) - An optional boolean flag indicating whether to force the import of the nodes. - Returns: - Result), GraphError> - A Result object which is Ok if the nodes were successfully imported, and Err otherwise. """ def index(self): @@ -1378,6 +1394,21 @@ class Graph: GraphView: The layered view """ + @staticmethod + def load_cached(path): + """ + Load Graph from a file and initialise it as a cache file. + + Future updates are tracked. Use `write_updates` to persist them to the + cache file. + + Arguments: + path (str): The path to the cache file + + Returns: + Graph + """ + def load_edge_props_from_pandas(self, df, src, dst, const_properties=None, shared_const_properties=None, layer=None, layer_in_df=True): """ Load edge properties from a Pandas DataFrame. @@ -1451,15 +1482,15 @@ class Graph: """ @staticmethod - def load_from_file(path, force=False): + def load_from_file(path): """ - Loads a graph from the given path. + Load Graph from a file. Arguments: - path (str): The path to the graph. + path (str): The path to the file. Returns: - Graph: The loaded graph. + Graph """ @staticmethod @@ -1637,13 +1668,18 @@ class Graph: def save_to_file(self, path): """ - Saves the graph to the given path. + Saves the Graph to the given path. Arguments: - path (str): The path to the graph. + path (str): The path to the file. + """ + + def serialise(self): + """ + Serialise Graph to bytes. Returns: - None + Bytes """ def shrink_end(self, end): @@ -1818,6 +1854,9 @@ class Graph: def window_size(self): """ Get the window size (difference between start and end) for this GraphView""" + def write_updates(self): + """Persist the new updates by appending them to the cache file.""" + class GraphIndex: """ A searchable Index for a `Graph`. This allows for fuzzy and exact searches of nodes and edges. @@ -3681,8 +3720,16 @@ class PersistentGraph: A GraphView object. """ - def bincode(self): - """Get bincode encoded graph""" + def cache(self, path): + """ + Write PersistentGraph to cache file and initialise the cache. + + Future updates are tracked. Use `write_updates` to persist them to the + cache file. If the file already exists its contents are overwritten. + + Arguments: + path (str): The path to the cache file + """ def count_edges(self): """ @@ -3729,6 +3776,18 @@ class PersistentGraph: None or a GraphError if the edge could not be deleted """ + @staticmethod + def deserialise(bytes): + """ + Load PersistentGraph from serialised bytes. + + Arguments: + bytes (Bytes): The serialised bytes to decode + + Returns: + PersistentGraph + """ + @property def earliest_date_time(self): """ @@ -3937,8 +3996,6 @@ class PersistentGraph: edges (List(edges)) - A vector of PyEdge objects representing the edges to be imported. force (boolean) - An optional boolean flag indicating whether to force the import of the edges. - Returns: - Result), GraphError> - A Result object which is Ok if the edges were successfully imported, and Err otherwise. """ def import_node(self, node, force=False): @@ -3968,8 +4025,6 @@ class PersistentGraph: nodes (List(Node))- A vector of PyNode objects representing the nodes to be imported. force (boolean) - An optional boolean flag indicating whether to force the import of the nodes. - Returns: - Result), GraphError> - A Result object which is Ok if the nodes were successfully imported, and Err otherwise. """ def index(self): @@ -4021,6 +4076,21 @@ class PersistentGraph: GraphView: The layered view """ + @staticmethod + def load_cached(path): + """ + Load PersistentGraph from a file and initialise it as a cache file. + + Future updates are tracked. Use `write_updates` to persist them to the + cache file. + + Arguments: + path (str): The path to the cache file + + Returns: + PersistentGraph + """ + def load_edge_props_from_pandas(self, df, src, dst, const_properties=None, shared_const_properties=None, layer=None, layer_in_df=True): """ Load edge properties from a Pandas DataFrame. @@ -4126,15 +4196,15 @@ class PersistentGraph: """ @staticmethod - def load_from_file(path, force=False): + def load_from_file(path): """ - Loads a graph from the given path. + Load PersistentGraph from a file. Arguments: - path (str): The path to the graph. + path (str): The path to the file. Returns: - Graph: The loaded graph. + PersistentGraph """ @staticmethod @@ -4309,13 +4379,18 @@ class PersistentGraph: def save_to_file(self, path): """ - Saves the graph to the given path. + Saves the PersistentGraph to the given path. Arguments: - path (str): The path to the graph. + path (str): The path to the file. + """ + + def serialise(self): + """ + Serialise PersistentGraph to bytes. Returns: - None + Bytes """ def shrink_end(self, end): @@ -4490,6 +4565,9 @@ class PersistentGraph: def window_size(self): """ Get the window size (difference between start and end) for this GraphView""" + def write_updates(self): + """Persist the new updates by appending them to the cache file.""" + class Properties: """A view of the properties of an entity""" diff --git a/python/python/raphtory/graphql/__init__.pyi b/python/python/raphtory/graphql/__init__.pyi index d795c6ed6b..de8f0d0e21 100644 --- a/python/python/raphtory/graphql/__init__.pyi +++ b/python/python/raphtory/graphql/__init__.pyi @@ -41,16 +41,47 @@ class RaphtoryClient: def __init__(self, url): """Initialize self. See help(type(self)) for accurate signature.""" - def load_graphs_from_path(self, path, overwrite=False): + def copy_graph(self, path, new_path): """ - Set the server to load all the graphs from its path `path`. + Copy graph from a path `path` on the server to a `new_path` on the server Arguments: - * `path`: the path to load the graphs from. - * `overwrite`: whether or not to overwrite existing graphs (defaults to False) + * `path`: the path of the graph to be copied + * `new_path`: the new path of the copied graph Returns: - The `data` field from the graphQL response after executing the mutation. + Copy status as boolean + """ + + def delete_graph(self, path): + """ + Delete graph from a path `path` on the server + + Arguments: + * `path`: the path of the graph to be deleted + + Returns: + Delete status as boolean + """ + + def is_server_online(self): + """ + Check if the server is online. + + Returns: + Returns true if server is online otherwise false. + """ + + def move_graph(self, path, new_path): + """ + Move graph from a path `path` on the server to a `new_path` on the server + + Arguments: + * `path`: the path of the graph to be moved + * `new_path`: the new path of the moved graph + + Returns: + Move status as boolean """ def query(self, query, variables=None): @@ -65,33 +96,50 @@ class RaphtoryClient: The `data` field from the graphQL response. """ - def send_graph(self, name, graph): + def receive_graph(self, path): + """ + Receive graph from a path `path` on the server + + Arguments: + * `path`: the path of the graph to be received + + Returns: + Graph as string + """ + + def send_graph(self, path, graph, overwrite=False): """ - Send a graph to the server. + Send a graph to the server Arguments: - * `name`: the name of the graph sent. - * `graph`: the graph to send. + * `path`: the path of the graph + * `graph`: the graph to send + * `overwrite`: overwrite existing graph (defaults to False) Returns: The `data` field from the graphQL response after executing the mutation. """ - def wait_for_online(self, millis=None): + def upload_graph(self, path, file_path, overwrite=False): """ - Wait for the server to be online. + Upload graph file from a path `file_path` on the client Arguments: - * `millis`: the minimum number of milliseconds to wait (default 5000). + * `path`: the name of the graph + * `file_path`: the path of the graph on the client + * `overwrite`: overwrite existing graph (defaults to False) + + Returns: + The `data` field from the graphQL response after executing the mutation. """ class RaphtoryServer: """A class for defining and running a Raphtory GraphQL server""" - def __init__(self, graphs=None, graph_dir=None): + def __init__(self, work_dir, cache_capacity=None, cache_tti_seconds=None, log_level=None, config_path=None): """Initialize self. See help(type(self)) for accurate signature.""" - def run(self, port=1736, log_level=..., enable_tracing=False, enable_auth=False): + def run(self, port=1736, timeout_ms=...): """ Run the server until completion. @@ -99,12 +147,13 @@ class RaphtoryServer: * `port`: the port to use (defaults to 1736). """ - def start(self, port=1736, log_level=..., enable_tracing=False, enable_auth=False): + def start(self, port=1736, timeout_ms=None): """ Start the server and return a handle to it. Arguments: * `port`: the port to use (defaults to 1736). + * `timeout_ms`: wait for server to be online (defaults to 5000). The server is stopped if not online within timeout_ms but manages to come online as soon as timeout_ms finishes! """ def with_document_search_function(self, name, input, function): @@ -169,52 +218,8 @@ class RunningRaphtoryServer: def __init__(self): """Initialize self. See help(type(self)) for accurate signature.""" - def load_graphs_from_path(self, path, overwrite=False): - """ - Set the server to load all the graphs from its path `path`. - - Arguments: - * `path`: the path to load the graphs from. - * `overwrite`: whether or not to overwrite existing graphs (defaults to False) - - Returns: - The `data` field from the graphQL response after executing the mutation. - """ - - def query(self, query, variables=None): - """ - Make a graphQL query against the server. - - Arguments: - * `query`: the query to make. - * `variables`: a dict of variables present on the query and their values. - - Returns: - The `data` field from the graphQL response. - """ - - def send_graph(self, name, graph): - """ - Send a graph to the server. - - Arguments: - * `name`: the name of the graph sent. - * `graph`: the graph to send. - - Returns: - The `data` field from the graphQL response after executing the mutation. - """ + def get_client(self): + ... def stop(self): - """Stop the server.""" - - def wait(self): - """Wait until server completion.""" - - def wait_for_online(self, timeout_millis=None): - """ - Wait for the server to be online. - - Arguments: - * `timeout_millis`: the timeout in milliseconds (default 5000). - """ + """Stop the server and wait for it to finish""" diff --git a/python/tests/test_algorithms.py b/python/tests/test_algorithms.py index 35ea620beb..5f671e82c2 100644 --- a/python/tests/test_algorithms.py +++ b/python/tests/test_algorithms.py @@ -258,7 +258,7 @@ def test_temporal_reachability(): actual = algorithms.temporally_reachable_nodes(g, 20, 11, [1, 2], [4, 5]) expected = { "1": [(11, "start")], - "2": [(11, "start"), (12, "1"), (11, "1")], + "2": [(11, "1"), (11, "start"), (12, "1"), ], "3": [], "4": [(12, "2")], "5": [(13, "2")], @@ -321,12 +321,12 @@ def test_single_source_shortest_path(): "4": ["1", "4"], } assert ( - res_two.get_all_with_names() - == {"1": ["1"], "2": ["1", "2"], "3": ["1", "2", "3"], "4": ["1", "4"]} - ) or ( - res_two.get_all_with_names() - == {"1": ["1"], "3": ["1", "4", "3"], "2": ["1", "2"], "4": ["1", "4"]} - ) + res_two.get_all_with_names() + == {"1": ["1"], "2": ["1", "2"], "3": ["1", "2", "3"], "4": ["1", "4"]} + ) or ( + res_two.get_all_with_names() + == {"1": ["1"], "3": ["1", "4", "3"], "2": ["1", "2"], "4": ["1", "4"]} + ) def test_dijsktra_shortest_paths(): diff --git a/python/tests/test_graphql.py b/python/tests/test_graphql.py index e0b9ec6c77..a1c7e9cea8 100644 --- a/python/tests/test_graphql.py +++ b/python/tests/test_graphql.py @@ -1,5 +1,6 @@ import base64 import os +import pickle import tempfile import time @@ -264,7 +265,7 @@ def assert_graph_fetch(path): client.send_graph(path=path, graph=g, overwrite=True) assert "Backslash not allowed in path" in str(excinfo.value) - #Test if we can escape through a symlink + # Test if we can escape through a symlink tmp_dir2 = tempfile.mkdtemp() nested_dir = os.path.join(tmp_work_dir, "shivam", "graphs") os.makedirs(nested_dir) @@ -422,6 +423,7 @@ def test_upload_graph_succeeds_if_graph_already_exists_at_namespace_with_overwri } } + def test_get_graph_fails_if_graph_not_found(): work_dir = tempfile.mkdtemp() with RaphtoryServer(work_dir).start() as server: @@ -554,8 +556,7 @@ def test_receive_graph_succeeds_if_graph_found(): received_graph = client.query(query)['receiveGraph'] decoded_bytes = base64.b64decode(received_graph) - - g = Graph.from_bincode(decoded_bytes) + g = Graph.deserialise(decoded_bytes); assert g.nodes.name == ["ben", "hamza", "haaroon"] @@ -603,7 +604,7 @@ def test_receive_graph_succeeds_if_graph_found_at_namespace(): decoded_bytes = base64.b64decode(received_graph) - g = Graph.from_bincode(decoded_bytes) + g = Graph.deserialise(decoded_bytes) assert g.nodes.name == ["ben", "hamza", "haaroon"] @@ -1263,11 +1264,11 @@ def test_create_graph_fail_if_graph_already_exists(): def test_create_graph_fail_if_graph_already_exists_at_namespace(): work_dir = tempfile.mkdtemp() os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) - + g = Graph() g.save_to_file(os.path.join(work_dir, "g0")) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - + with RaphtoryServer(work_dir).start() as server: client = server.get_client() query = """mutation { @@ -2382,4 +2383,4 @@ def test_graph_properties_query(): # tmp_work_dir = tempfile.mkdtemp() # with RaphtoryServer(tmp_work_dir).start() as server: # client = server.get_client() -# client.upload_graph(path="g", graph=g) \ No newline at end of file +# client.upload_graph(path="g", graph=g) diff --git a/raphtory-api/Cargo.toml b/raphtory-api/Cargo.toml index 51f99f0e0d..aedc155f06 100644 --- a/raphtory-api/Cargo.toml +++ b/raphtory-api/Cargo.toml @@ -24,13 +24,14 @@ parking_lot = { workspace = true } pyo3 = { workspace = true, optional = true } rayon = { workspace = true } rand = { workspace = true } -quickcheck = { workspace = true } quickcheck_macros = { workspace = true } num-traits = { workspace = true } twox-hash.workspace = true [dev-dependencies] proptest.workspace = true +quickcheck.workspace = true +quickcheck_macros.workspace = true [features] default = [] diff --git a/raphtory-api/src/core/storage/dict_mapper.rs b/raphtory-api/src/core/storage/dict_mapper.rs index a80e14bbdf..58619246cc 100644 --- a/raphtory-api/src/core/storage/dict_mapper.rs +++ b/raphtory-api/src/core/storage/dict_mapper.rs @@ -1,7 +1,12 @@ use crate::core::storage::{arc_str::ArcStr, locked_vec::ArcReadLockedVec, FxDashMap}; +use dashmap::mapref::entry::Entry; use parking_lot::RwLock; use serde::{Deserialize, Serialize}; -use std::{borrow::Borrow, hash::Hash, sync::Arc}; +use std::{ + borrow::{Borrow, BorrowMut}, + hash::Hash, + sync::Arc, +}; #[derive(Serialize, Deserialize, Default, Debug)] pub struct DictMapper { @@ -9,24 +14,100 @@ pub struct DictMapper { reverse_map: Arc>>, //FIXME: a boxcar vector would be a great fit if it was serializable... } +#[derive(Copy, Clone, Debug)] +pub enum MaybeNew { + New(Index), + Existing(Index), +} + +impl PartialEq for MaybeNew +where + Index: PartialEq, + T: Borrow, +{ + fn eq(&self, other: &T) -> bool { + other.borrow() == self.borrow() + } +} + +impl MaybeNew { + #[inline] + pub fn inner(self) -> Index { + match self { + MaybeNew::New(inner) => inner, + MaybeNew::Existing(inner) => inner, + } + } + + #[inline] + pub fn map(self, map_fn: impl FnOnce(Index) -> R) -> MaybeNew { + match self { + MaybeNew::New(inner) => MaybeNew::New(map_fn(inner)), + MaybeNew::Existing(inner) => MaybeNew::Existing(map_fn(inner)), + } + } + + #[inline] + pub fn as_ref(&self) -> MaybeNew<&Index> { + match self { + MaybeNew::New(inner) => MaybeNew::New(inner), + MaybeNew::Existing(inner) => MaybeNew::Existing(inner), + } + } + + #[inline] + pub fn as_mut(&mut self) -> MaybeNew<&mut Index> { + match self { + MaybeNew::New(inner) => MaybeNew::New(inner), + MaybeNew::Existing(inner) => MaybeNew::Existing(inner), + } + } + + #[inline] + pub fn if_new(self, map_fn: impl FnOnce(Index) -> R) -> Option { + match self { + MaybeNew::New(inner) => Some(map_fn(inner)), + MaybeNew::Existing(_) => None, + } + } +} + +impl Borrow for MaybeNew { + #[inline] + fn borrow(&self) -> &Index { + self.as_ref().inner() + } +} + +impl BorrowMut for MaybeNew { + #[inline] + fn borrow_mut(&mut self) -> &mut Index { + self.as_mut().inner() + } +} + impl DictMapper { - pub fn get_or_create_id(&self, name: &Q) -> usize + pub fn get_or_create_id(&self, name: &Q) -> MaybeNew where Q: Hash + Eq + ?Sized + ToOwned + Borrow, T: Into, { if let Some(existing_id) = self.map.get(name.borrow()) { - return *existing_id; + return MaybeNew::Existing(*existing_id); } let name = name.to_owned().into(); - let new_id = self.map.entry(name.clone()).or_insert_with(|| { - let mut reverse = self.reverse_map.write(); - let id = reverse.len(); - reverse.push(name); - id - }); - *new_id + let new_id = match self.map.entry(name.clone()) { + Entry::Occupied(entry) => MaybeNew::Existing(*entry.get()), + Entry::Vacant(entry) => { + let mut reverse = self.reverse_map.write(); + let id = reverse.len(); + reverse.push(name); + entry.insert(id); + MaybeNew::New(id) + } + }; + new_id } pub fn get_id(&self, name: &str) -> Option { @@ -88,11 +169,11 @@ mod test { #[test] fn test_dict_mapper() { let mapper = DictMapper::default(); - assert_eq!(mapper.get_or_create_id("test"), 0); - assert_eq!(mapper.get_or_create_id("test"), 0); - assert_eq!(mapper.get_or_create_id("test2"), 1); - assert_eq!(mapper.get_or_create_id("test2"), 1); - assert_eq!(mapper.get_or_create_id("test"), 0); + assert_eq!(mapper.get_or_create_id("test"), 0usize); + assert_eq!(mapper.get_or_create_id("test").inner(), 0); + assert_eq!(mapper.get_or_create_id("test2").inner(), 1); + assert_eq!(mapper.get_or_create_id("test2").inner(), 1); + assert_eq!(mapper.get_or_create_id("test").inner(), 0); } #[quickcheck] @@ -110,7 +191,7 @@ mod test { write_s.shuffle(&mut rng); for s in write_s { let id = mapper.get_or_create_id(s.as_str()); - ids.insert(s, id); + ids.insert(s, id.inner()); } ids }) @@ -147,7 +228,7 @@ mod test { let mut actual = vec!["test", "test2", "test3", "test4", "test5"] .into_iter() - .map(|name| mapper.get_or_create_id(name)) + .map(|name| mapper.get_or_create_id(name).inner()) .collect::>(); actual.sort(); diff --git a/raphtory-benchmark/Cargo.toml b/raphtory-benchmark/Cargo.toml index 08ca97469f..e51e4dc6f2 100644 --- a/raphtory-benchmark/Cargo.toml +++ b/raphtory-benchmark/Cargo.toml @@ -9,18 +9,14 @@ edition = "2021" criterion = { workspace = true } raphtory = { path = "../raphtory", features = ["io"], version = "0.10.0" } raphtory-api = { path = "../raphtory-api", version = "0.10.0" } -raphtory-graphql = { path = "../raphtory-graphql", version = "0.10.0" } pometry-storage.workspace = true sorted_vector_map = { workspace = true } rand = { workspace = true } rayon = { workspace = true } -polars-arrow = { workspace = true } tempfile = { workspace = true } -chrono = { workspace = true } clap = { workspace = true } csv = { workspace = true } flate2 = { workspace = true } -tokio = { workspace = true } [[bench]] name = "tgraph_benchmarks" diff --git a/raphtory-cypher/Cargo.toml b/raphtory-cypher/Cargo.toml index 7eec1c729f..52dfd956f4 100644 --- a/raphtory-cypher/Cargo.toml +++ b/raphtory-cypher/Cargo.toml @@ -42,7 +42,6 @@ itertools.workspace = true proptest.workspace = true pretty_assertions.workspace = true tempfile.workspace = true -rand.workspace = true tokio.workspace = true clap.workspace = true diff --git a/raphtory-graphql/Cargo.toml b/raphtory-graphql/Cargo.toml index 616c5288d0..9be98ec2f7 100644 --- a/raphtory-graphql/Cargo.toml +++ b/raphtory-graphql/Cargo.toml @@ -15,24 +15,20 @@ homepage.workspace = true [dependencies] raphtory = { path = "../raphtory", version = "0.10.0", features = ['vectors', 'search', "io"] } raphtory-api = { path = "../raphtory-api", version = "0.10.0" } -bincode = { workspace = true } base64 = { workspace = true } thiserror = { workspace = true } -dotenv = { workspace = true } itertools = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } once_cell = { workspace = true } poem = { workspace = true } -poem-openapi = { workspace = true } oauth2 = { workspace = true } tokio = { workspace = true } -async-graphql = { workspace = true, features=["apollo_tracing"] } +async-graphql = { workspace = true, features = ["apollo_tracing"] } dynamic-graphql = { workspace = true } async-graphql-poem = { workspace = true } parking_lot = { workspace = true } futures-util = { workspace = true } -async-stream = { workspace = true } jsonwebtoken = { workspace = true } opentelemetry = { workspace = true } opentelemetry_sdk = { workspace = true } @@ -45,13 +41,10 @@ ordered-float = { workspace = true } uuid = { workspace = true } chrono = { workspace = true } config = { workspace = true } -toml = { workspace = true } url = { workspace = true } base64-compat = { workspace = true } -time = { workspace = true } reqwest = { workspace = true } moka = { workspace = true } -kdam = { workspace = true} # python binding optional dependencies diff --git a/raphtory-graphql/src/data.rs b/raphtory-graphql/src/data.rs index c62971b50f..3cdaee7d37 100644 --- a/raphtory-graphql/src/data.rs +++ b/raphtory-graphql/src/data.rs @@ -12,6 +12,7 @@ use raphtory::{ }, }, db::api::view::MaterializedGraph, + prelude::*, search::IndexedGraph, }; use std::{ @@ -32,9 +33,14 @@ impl Data { pub fn new(work_dir: &Path, configs: &AppConfig) -> Self { let cache_configs = &configs.cache; - let graphs_cache_builder = Cache::builder() + let graphs_cache_builder = Cache::<_, IndexedGraph>::builder() .max_capacity(cache_configs.capacity) .time_to_idle(std::time::Duration::from_secs(cache_configs.tti_seconds)) + .eviction_listener(|_, value, _| { + value + .write_updates() + .unwrap_or_else(|err| println!("Write on eviction failed: {err:?}")) + }) .build(); let graphs_cache: Cache> = graphs_cache_builder; @@ -250,9 +256,9 @@ fn get_graph_from_path(path: &Path) -> Result, G return Err(PathIsDirectory(path.to_path_buf()).into()); } } else { - let graph = load_bincode_graph(path)?; + let graph = MaterializedGraph::load_cached(path)?; println!("Graph loaded = {}", path.display()); - Ok(IndexedGraph::from_graph(&graph.into())?) + Ok(IndexedGraph::from_graph(&graph)?) } } @@ -297,11 +303,6 @@ pub(crate) fn get_graph_name(path: &Path) -> Result { } //should not happen, but means we always get a name } -pub(crate) fn load_bincode_graph(path: &Path) -> Result { - let graph = MaterializedGraph::load_from_file(path, false)?; - Ok(graph) -} - #[cfg(feature = "storage")] fn load_disk_graph(path: &Path) -> Result { let disk_graph = DiskGraphStorage::load_from_dir(path) @@ -336,7 +337,7 @@ pub(crate) mod data_tests { use raphtory::core::utils::errors::{GraphError, InvalidPathReason}; #[cfg(feature = "storage")] use raphtory::{ - db::api::storage::storage_ops::GraphStorage, db::api::view::internal::CoreGraphOps, + db::api::storage::graph::storage_ops::GraphStorage, db::api::view::internal::CoreGraphOps, disk_graph::DiskGraphStorage, }; #[cfg(feature = "storage")] @@ -390,12 +391,12 @@ pub(crate) mod data_tests { #[cfg(feature = "storage")] copy_dir_recursive(disk_graph_path, &full_path)?; } else { - graph.save_to_path(&full_path)?; + graph.encode(&full_path)?; } #[cfg(not(feature = "storage"))] { - graph.save_to_path(&full_path)?; + graph.encode(&full_path)?; } } Ok(()) @@ -499,13 +500,9 @@ pub(crate) mod data_tests { .add_edge(0, 1, 3, [("name", "test_e2")], None) .unwrap(); - graph - .save_to_file(&tmp_work_dir.path().join("test_g")) - .unwrap(); + graph.encode(&tmp_work_dir.path().join("test_g")).unwrap(); let _ = DiskGraphStorage::from_graph(&graph, &tmp_work_dir.path().join("test_dg")).unwrap(); - graph - .save_to_file(&tmp_work_dir.path().join("test_g2")) - .unwrap(); + graph.encode(&tmp_work_dir.path().join("test_g2")).unwrap(); let configs = AppConfigBuilder::new() .with_cache_capacity(1) diff --git a/raphtory-graphql/src/lib.rs b/raphtory-graphql/src/lib.rs index 6ff3ccc6b6..ac2b6dcb46 100644 --- a/raphtory-graphql/src/lib.rs +++ b/raphtory-graphql/src/lib.rs @@ -666,7 +666,7 @@ mod graphql_test { g.add_node(0, 1, NO_PROPS, None).unwrap(); let tmp_file = tempfile::NamedTempFile::new().unwrap(); let path = tmp_file.path(); - g.save_to_file(path).unwrap(); + g.encode(path).unwrap(); let file = std::fs::File::open(path).unwrap(); let upload_val = UploadValue { filename: "test".into(), diff --git a/raphtory-graphql/src/model/mod.rs b/raphtory-graphql/src/model/mod.rs index 174ba90ea5..56eacc4de1 100644 --- a/raphtory-graphql/src/model/mod.rs +++ b/raphtory-graphql/src/model/mod.rs @@ -4,10 +4,9 @@ use crate::{ algorithms::global_plugins::GlobalPlugins, graph::{graph::GqlGraph, graphs::GqlGraphs, vectorised_graph::GqlVectorisedGraph}, }, - url_encode::url_decode_graph, + url_encode::{url_decode_graph, url_encode_graph}, }; use async_graphql::Context; -use base64::{engine::general_purpose::STANDARD, Engine}; use chrono::Utc; use dynamic_graphql::{ App, Mutation, MutationFields, MutationRoot, ResolvedObject, ResolvedObjectFields, Result, @@ -15,19 +14,21 @@ use dynamic_graphql::{ }; use itertools::Itertools; #[cfg(feature = "storage")] -use raphtory::db::api::{storage::storage_ops::GraphStorage, view::internal::CoreGraphOps}; +use raphtory::db::api::{storage::graph::storage_ops::GraphStorage, view::internal::CoreGraphOps}; use raphtory::{ core::{utils::errors::GraphError, Prop}, db::api::view::MaterializedGraph, - prelude::{GraphViewOps, ImportOps, PropertyAdditionOps}, + prelude::*, }; use raphtory_api::core::storage::arc_str::ArcStr; use std::{ error::Error, fmt::{Display, Formatter}, fs, - io::Read, + fs::File, + io::copy, path::Path, + sync::Arc, }; pub mod algorithms; @@ -100,11 +101,12 @@ impl QueryRoot { data.global_plugins.clone() } - async fn receive_graph<'a>(ctx: &Context<'a>, path: String) -> Result { - let path = Path::new(&path); + async fn receive_graph<'a>(ctx: &Context<'a>, path: String) -> Result> { + let path = path.as_ref(); let data = ctx.data_unchecked::(); - let g = data.get_graph(path)?.materialize()?; - Ok(STANDARD.encode(g.bincode()?)) + let g = data.get_graph(path)?.graph.clone(); + let res = url_encode_graph(g)?; + Ok(res) } } @@ -160,7 +162,7 @@ impl Mut { graph.update_constant_properties([("lastUpdated", Prop::I64(timestamp * 1000))])?; graph.update_constant_properties([("lastOpened", Prop::I64(timestamp * 1000))])?; create_dirs_if_not_present(&new_full_path)?; - graph.save_to_file(&new_full_path)?; + graph.graph.encode(&new_full_path)?; delete_graph(&full_path)?; data.graphs.remove(&path.to_path_buf()); @@ -197,7 +199,7 @@ impl Mut { let new_graph = graph.materialize()?; new_graph.update_constant_properties([("lastOpened", Prop::I64(timestamp * 1000))])?; create_dirs_if_not_present(&new_full_path)?; - new_graph.save_to_file(&new_full_path)?; + new_graph.encode(&new_full_path)?; } Ok(true) @@ -217,9 +219,7 @@ impl Mut { graph.update_constant_properties([("lastOpened", Prop::I64(timestamp * 1000))])?; - let full_path = data.construct_graph_full_path(path)?; - graph.save_to_file(full_path)?; - data.graphs.insert(path.to_path_buf(), graph); + graph.write_updates()?; Ok(true) } @@ -261,7 +261,7 @@ impl Mut { new_subgraph.update_constant_properties([("isArchive", Prop::U8(is_archive))])?; create_dirs_if_not_present(&new_graph_full_path)?; - new_subgraph.save_to_file(new_graph_full_path)?; + new_subgraph.cache(new_graph_full_path)?; data.graphs .insert(new_graph_path.to_path_buf(), new_subgraph.into()); @@ -354,7 +354,7 @@ impl Mut { new_subgraph.update_constant_properties([("uiProps", Prop::Str(props.into()))])?; new_subgraph.update_constant_properties([("isArchive", Prop::U8(is_archive))])?; - new_subgraph.save_to_file(new_graph_full_path)?; + new_subgraph.cache(new_graph_full_path)?; data.graphs.remove(&graph_path.to_path_buf()); data.graphs @@ -398,12 +398,11 @@ impl Mut { return Err(GraphError::GraphNameAlreadyExists(path.to_path_buf()).into()); } - let mut buffer = Vec::new(); - let mut buff_read = graph.value(ctx)?.content; - buff_read.read_to_end(&mut buffer)?; - let g: MaterializedGraph = MaterializedGraph::from_bincode(&buffer)?; + let mut in_file = graph.value(ctx)?.content; create_dirs_if_not_present(&full_path)?; - g.save_to_file(&full_path)?; + let mut out_file = File::create(&full_path)?; + copy(&mut in_file, &mut out_file)?; + let g = MaterializedGraph::load_cached(&full_path)?; data.graphs.insert(path.to_path_buf(), g.into()); Ok(path.display().to_string()) } @@ -411,7 +410,7 @@ impl Mut { /// Send graph bincode as base64 encoded string /// /// Returns:: - /// name of the new graph + /// path of the new graph async fn send_graph<'a>( ctx: &Context<'a>, path: String, @@ -426,7 +425,7 @@ impl Mut { } let g: MaterializedGraph = url_decode_graph(graph)?; create_dirs_if_not_present(&full_path)?; - g.save_to_file(&full_path)?; + g.cache(&full_path)?; data.graphs.insert(path.to_path_buf(), g.into()); Ok(path.display().to_string()) } @@ -441,12 +440,7 @@ impl Mut { } graph.update_constant_properties([("isArchive", Prop::U8(is_archive))])?; - - let full_path = data.construct_graph_full_path(path)?; - graph.save_to_file(full_path)?; - - data.graphs.insert(path.to_path_buf(), graph); - + graph.write_updates()?; Ok(true) } } diff --git a/raphtory-graphql/src/python/graphql.rs b/raphtory-graphql/src/python/graphql.rs index 8a8c4ad4cc..5498a12aa5 100644 --- a/raphtory-graphql/src/python/graphql.rs +++ b/raphtory-graphql/src/python/graphql.rs @@ -4,14 +4,13 @@ use crate::{ global_plugins::GlobalPlugins, vector_algorithms::VectorAlgorithms, }, server_config::*, - url_encode::url_encode_graph, + url_encode::{url_decode_graph, url_encode_graph}, RaphtoryServer, }; use async_graphql::{ dynamic::{Field, FieldFuture, FieldValue, InputValue, Object, TypeRef, ValueAccessor}, Value as GraphqlValue, }; -use base64::{engine::general_purpose, Engine as _}; use crossbeam_channel::Sender as CrossbeamSender; use dynamic_graphql::internal::{Registry, TypeName}; use itertools::intersperse; @@ -951,10 +950,7 @@ impl PyRaphtoryClient { let data = self.query_with_json_variables(query.clone(), variables.into())?; match data.get("receiveGraph") { Some(JsonValue::String(graph)) => { - let decoded_bytes = general_purpose::STANDARD - .decode(graph.clone()) - .map_err(|err| PyException::new_err(format!("Base64 decode error: {}", err)))?; - let mat_graph = MaterializedGraph::from_bincode(&decoded_bytes)?; + let mat_graph = url_decode_graph(graph)?; Ok(mat_graph) } _ => Err(PyException::new_err(format!( diff --git a/raphtory-graphql/src/python/mod.rs b/raphtory-graphql/src/python/mod.rs index 51dfed10cd..363054a62e 100644 --- a/raphtory-graphql/src/python/mod.rs +++ b/raphtory-graphql/src/python/mod.rs @@ -1,2 +1,12 @@ +use crate::url_encode::UrlDecodeError; +use pyo3::PyErr; +use raphtory::python::utils::errors::adapt_err_value; + pub mod graphql; pub mod pymodule; + +impl From for PyErr { + fn from(value: UrlDecodeError) -> Self { + adapt_err_value(&value) + } +} diff --git a/raphtory-graphql/src/url_encode.rs b/raphtory-graphql/src/url_encode.rs index c51688a011..95a43fd02f 100644 --- a/raphtory-graphql/src/url_encode.rs +++ b/raphtory-graphql/src/url_encode.rs @@ -1,5 +1,9 @@ use base64::{prelude::BASE64_URL_SAFE, DecodeError, Engine}; -use raphtory::{core::utils::errors::GraphError, db::api::view::MaterializedGraph}; +use raphtory::{ + core::utils::errors::GraphError, + db::api::view::MaterializedGraph, + serialise::{StableDecode, StableEncode}, +}; #[derive(thiserror::Error, Debug)] pub enum UrlDecodeError { @@ -17,11 +21,11 @@ pub enum UrlDecodeError { pub fn url_encode_graph>(graph: G) -> Result { let g: MaterializedGraph = graph.into(); - Ok(BASE64_URL_SAFE.encode(g.bincode()?)) + Ok(BASE64_URL_SAFE.encode(g.encode_to_vec())) } pub fn url_decode_graph>(graph: T) -> Result { - Ok(MaterializedGraph::from_bincode( + Ok(MaterializedGraph::decode_from_bytes( &BASE64_URL_SAFE.decode(graph)?, )?) } diff --git a/raphtory/Cargo.toml b/raphtory/Cargo.toml index 66f5dae492..597d604ba4 100644 --- a/raphtory/Cargo.toml +++ b/raphtory/Cargo.toml @@ -16,7 +16,6 @@ homepage.workspace = true [dependencies] raphtory-api = { path = "../raphtory-api", version = "0.10.0" } -bincode = { workspace = true } chrono = { workspace = true } itertools = { workspace = true } num-traits = { workspace = true } @@ -31,7 +30,6 @@ rustc-hash = { workspace = true } serde = { workspace = true } sorted_vector_map = { workspace = true } thiserror = { workspace = true } -twox-hash = { workspace = true } lock_api = { workspace = true } dashmap = { workspace = true } enum_dispatch = { workspace = true } @@ -41,7 +39,7 @@ quad-rand = { workspace = true } serde_json = { workspace = true } ouroboros = { workspace = true } either = { workspace = true } -kdam = { workspace = true} +kdam = { workspace = true } # io optional dependencies @@ -60,6 +58,7 @@ tantivy = { workspace = true, optional = true } futures-util = { workspace = true, optional = true } async-trait = { workspace = true, optional = true } async-openai = { workspace = true, optional = true } +bincode = { workspace = true, optional = true } # python binding optional dependencies pyo3 = { workspace = true, optional = true } @@ -67,18 +66,12 @@ num = { workspace = true, optional = true } display-error-chain = { workspace = true, optional = true } polars-arrow = { workspace = true, optional = true } polars-parquet = { workspace = true, optional = true } -polars-utils = { workspace = true, optional = true } memmap2 = { workspace = true, optional = true } -ahash = { workspace = true, optional = true } tempfile = { workspace = true, optional = true } -bytemuck = { workspace = true, optional = true } -rpds = { workspace = true, optional = true } -thread_local = { workspace = true, optional = true } pometry-storage = { workspace = true, optional = true } -prost = { workspace = true, optional = true} -prost-types = { workspace = true, optional = true} -bytes = { workspace = true, optional = true} +prost = { workspace = true, optional = true } +prost-types = { workspace = true, optional = true } [dev-dependencies] csv = { workspace = true } @@ -86,14 +79,13 @@ pretty_assertions = { workspace = true } quickcheck = { workspace = true } quickcheck_macros = { workspace = true } tempfile = { workspace = true } -tempdir = { workspace = true } tokio = { workspace = true } # for vector testing dotenv = { workspace = true } # for vector testing streaming-stats = { workspace = true } proptest = { workspace = true } [build-dependencies] -prost-build = { workspace = true, optional = true} +prost-build = { workspace = true, optional = true } [features] default = [] @@ -106,12 +98,13 @@ io = [ "dep:csv", "dep:reqwest", "dep:tokio", + "proto", ] # search search = ["dep:tantivy"] # vectors -vectors = ["dep:futures-util", "dep:async-trait", "dep:async-openai"] +vectors = ["dep:futures-util", "dep:async-trait", "dep:async-openai", "dep:bincode"] # Enables generating the pyo3 python bindings python = [ @@ -125,7 +118,6 @@ python = [ "dep:display-error-chain", "polars-arrow?/compute", "raphtory-api/python", - "dep:rpds", "kdam/notebook" ] @@ -133,13 +125,8 @@ python = [ storage = [ "arrow", "pometry-storage", - "dep:polars-utils", "dep:memmap2", - "dep:ahash", "dep:tempfile", - "dep:bytemuck", - "dep:rpds", - "dep:thread_local", "polars-arrow?/io_ipc", "polars-arrow?/arrow_rs", ] @@ -152,7 +139,6 @@ arrow = [ proto = [ "dep:prost", "dep:prost-types", - "dep:bytes", "dep:prost-build", "dep:memmap2", ] diff --git a/raphtory/build.rs b/raphtory/build.rs index 1ff0547e1d..be1eda9fde 100644 --- a/raphtory/build.rs +++ b/raphtory/build.rs @@ -1,7 +1,8 @@ use std::io::Result; #[cfg(feature = "proto")] fn main() -> Result<()> { - prost_build::compile_protos(&["src/graph.proto"], &["src/"])?; + prost_build::compile_protos(&["src/serialise/graph.proto"], &["src/serialise"])?; + println!("cargo::rerun-if-changed=src/serialise/graph.proto"); Ok(()) } diff --git a/raphtory/src/algorithms/pathing/temporal_reachability.rs b/raphtory/src/algorithms/pathing/temporal_reachability.rs index df487e4fea..62a61d7ffc 100644 --- a/raphtory/src/algorithms/pathing/temporal_reachability.rs +++ b/raphtory/src/algorithms/pathing/temporal_reachability.rs @@ -191,10 +191,12 @@ pub fn temporally_reachable_nodes( None, |_, ess, _, _| { ess.finalize(&taint_history, |taint_history| { - taint_history + let mut hist = taint_history .into_iter() .map(|tmsg| (tmsg.event_time, tmsg.src_node)) - .collect_vec() + .collect_vec(); + hist.sort(); + hist }) }, threads, diff --git a/raphtory/src/core/entities/edges/edge_store.rs b/raphtory/src/core/entities/edges/edge_store.rs index cb2376f026..c4defa93d8 100644 --- a/raphtory/src/core/entities/edges/edge_store.rs +++ b/raphtory/src/core/entities/edges/edge_store.rs @@ -13,7 +13,7 @@ use crate::{ Prop, }, db::api::{ - storage::edges::edge_storage_ops::{EdgeStorageIntoOps, EdgeStorageOps}, + storage::graph::edges::edge_storage_ops::{EdgeStorageIntoOps, EdgeStorageOps}, view::IntoDynBoxed, }, }; diff --git a/raphtory/src/core/entities/graph/logical_to_physical.rs b/raphtory/src/core/entities/graph/logical_to_physical.rs index 374fe98c86..559baa3e1d 100644 --- a/raphtory/src/core/entities/graph/logical_to_physical.rs +++ b/raphtory/src/core/entities/graph/logical_to_physical.rs @@ -1,12 +1,12 @@ +use crate::core::utils::errors::{GraphError, MutateGraphError}; +use dashmap::mapref::entry::Entry; use once_cell::sync::OnceCell; - use raphtory_api::core::{ entities::{GidRef, VID}, - storage::FxDashMap, + storage::{dict_mapper::MaybeNew, FxDashMap}, }; use serde::{Deserialize, Deserializer, Serialize}; - -use crate::core::utils::errors::{GraphError, MutateGraphError}; +use std::hash::Hash; #[derive(Debug, Deserialize, Serialize)] enum Map { @@ -52,13 +52,13 @@ impl Mapping { &self, gid: GidRef, f_init: impl FnOnce() -> VID, - ) -> Result { + ) -> Result, GraphError> { let map = self.map.get_or_init(|| match &gid { GidRef::U64(_) => Map::U64(FxDashMap::default()), GidRef::Str(_) => Map::Str(FxDashMap::default()), }); let vid = match gid { - GidRef::U64(id) => map.as_u64().map(|m| *(m.entry(id).or_insert_with(f_init))), + GidRef::U64(id) => map.as_u64().map(|m| get_or_new(m, id, f_init)), GidRef::Str(id) => map.as_str().map(|m| optim_get_or_insert(m, id, f_init)), }; vid.ok_or(GraphError::FailedToMutateGraph { @@ -77,10 +77,31 @@ impl Mapping { } } -fn optim_get_or_insert(m: &FxDashMap, id: &str, f_init: impl FnOnce() -> VID) -> VID { +#[inline] +fn optim_get_or_insert( + m: &FxDashMap, + id: &str, + f_init: impl FnOnce() -> VID, +) -> MaybeNew { m.get(id) - .map(|vid| *vid) - .unwrap_or_else(|| *(m.entry(id.to_owned()).or_insert_with(f_init))) + .map(|vid| MaybeNew::Existing(*vid)) + .unwrap_or_else(|| get_or_new(m, id.to_owned(), f_init)) +} + +#[inline] +fn get_or_new( + m: &FxDashMap, + id: K, + f_init: impl FnOnce() -> VID, +) -> MaybeNew { + match m.entry(id) { + Entry::Occupied(entry) => MaybeNew::Existing(*entry.get()), + Entry::Vacant(entry) => { + let id = f_init(); + entry.insert(id); + MaybeNew::New(id) + } + } } impl<'de> Deserialize<'de> for Mapping { diff --git a/raphtory/src/core/entities/graph/mod.rs b/raphtory/src/core/entities/graph/mod.rs index a101097477..2c0c4e997e 100644 --- a/raphtory/src/core/entities/graph/mod.rs +++ b/raphtory/src/core/entities/graph/mod.rs @@ -5,37 +5,27 @@ pub(crate) mod timer; #[cfg(test)] mod test { - use crate::{ - core::PropType, - db::api::{mutation::internal::InternalAdditionOps, storage::storage_ops::GraphStorage}, - prelude::*, - }; + use crate::{core::PropType, db::api::mutation::internal::InternalAdditionOps, prelude::*}; #[test] fn test_neighbours_multiple_layers() { - let g = GraphStorage::default(); - let l_btc = g.resolve_layer(Some("btc")).unwrap(); - let l_eth = g.resolve_layer(Some("eth")).unwrap(); - let l_tether = g.resolve_layer(Some("tether")).unwrap(); - let v1 = g.resolve_node(1).unwrap(); - let v2 = g.resolve_node(2).unwrap(); + let g = Graph::default(); + let l_btc = g.resolve_layer(Some("btc")).unwrap().inner(); + let l_eth = g.resolve_layer(Some("eth")).unwrap().inner(); + let l_tether = g.resolve_layer(Some("tether")).unwrap().inner(); + let v1 = g.resolve_node(1).unwrap().inner(); + let v2 = g.resolve_node(2).unwrap().inner(); let tx_sent_id = g .resolve_edge_property("tx_sent", PropType::I32, false) + .unwrap() + .inner(); + g.internal_add_edge(1.into(), v1, v2, &[(tx_sent_id, Prop::I32(10))], l_btc) .unwrap(); - g.internal_add_edge(1.into(), v1, v2, vec![(tx_sent_id, Prop::I32(10))], l_btc) + g.internal_add_edge(1.into(), v1, v2, &[(tx_sent_id, Prop::I32(20))], l_eth) .unwrap(); - g.internal_add_edge(1.into(), v1, v2, vec![(tx_sent_id, Prop::I32(20))], l_eth) + g.internal_add_edge(1.into(), v1, v2, &[(tx_sent_id, Prop::I32(70))], l_tether) .unwrap(); - g.internal_add_edge( - 1.into(), - v1, - v2, - vec![(tx_sent_id, Prop::I32(70))], - l_tether, - ) - .unwrap(); - let g = Graph::from_internal_graph(g); let first = g .node(v1) .and_then(|n| n.layers(["btc", "eth"]).ok()) diff --git a/raphtory/src/core/entities/graph/tgraph.rs b/raphtory/src/core/entities/graph/tgraph.rs index a0fd464221..e10434c3c1 100644 --- a/raphtory/src/core/entities/graph/tgraph.rs +++ b/raphtory/src/core/entities/graph/tgraph.rs @@ -19,14 +19,14 @@ use crate::{ utils::errors::GraphError, Direction, Prop, }, - db::api::{storage::edges::edge_storage_ops::EdgeStorageOps, view::Layer}, + db::api::{storage::graph::edges::edge_storage_ops::EdgeStorageOps, view::Layer}, }; use dashmap::DashSet; use itertools::Itertools; use raphtory_api::core::{ entities::{edges::edge_ref::EdgeRef, GidRef}, input::input_node::InputNode, - storage::arc_str::ArcStr, + storage::{arc_str::ArcStr, dict_mapper::MaybeNew}, }; use rustc_hash::FxHasher; use serde::{Deserialize, Serialize}; @@ -98,10 +98,10 @@ impl TemporalGraph { } } - pub(crate) fn process_prop_value(&self, prop: Prop) -> Prop { + pub(crate) fn process_prop_value(&self, prop: &Prop) -> Prop { match prop { Prop::Str(value) => Prop::Str(self.resolve_str(value)), - _ => prop, + _ => prop.clone(), } } @@ -365,15 +365,15 @@ impl TemporalGraph { t: TimeIndexEntry, layer: usize, edge_fn: F, - ) -> Result { + ) -> Result, GraphError> { let mut node_pair = self.storage.pair_node_mut(src_id, dst_id); let src = node_pair.get_i(); let mut edge = match src.find_edge_eid(dst_id, &LayerIds::All) { - Some(edge_id) => self.storage.get_edge_mut(edge_id), - None => self.storage.push_edge(EdgeStore::new(src_id, dst_id)), + Some(edge_id) => MaybeNew::Existing(self.storage.get_edge_mut(edge_id)), + None => MaybeNew::New(self.storage.push_edge(EdgeStore::new(src_id, dst_id))), }; - self.link_nodes_inner(&mut node_pair, &mut edge, t, layer, edge_fn)?; - Ok(edge.edge_store().eid) + self.link_nodes_inner(&mut node_pair, edge.as_mut().inner(), t, layer, edge_fn)?; + Ok(edge.map(|e| e.edge_store().eid)) } pub(crate) fn resolve_node_ref(&self, v: NodeRef) -> Option { @@ -389,18 +389,15 @@ impl TemporalGraph { /// Checks if the same string value already exists and returns a pointer to the same existing value if it exists, /// otherwise adds the string to the pool. - fn resolve_str(&self, value: ArcStr) -> ArcStr { - match self.string_pool.get(&value) { + fn resolve_str(&self, value: &ArcStr) -> ArcStr { + match self.string_pool.get(value) { Some(value) => value.clone(), None => { - if self.string_pool.insert(value.clone()) { - value - } else { - self.string_pool - .get(&value) - .expect("value exists due to insert above returning false") - .clone() - } + self.string_pool.insert(value.clone()); + self.string_pool + .get(value) + .expect("value should exist as inserted above") + .clone() } } } diff --git a/raphtory/src/core/entities/nodes/node_ref.rs b/raphtory/src/core/entities/nodes/node_ref.rs index 00073c93c8..a6d7280ee4 100644 --- a/raphtory/src/core/entities/nodes/node_ref.rs +++ b/raphtory/src/core/entities/nodes/node_ref.rs @@ -72,6 +72,12 @@ impl AsNodeRef for GID { } } +impl<'a> AsNodeRef for GidRef<'a> { + fn as_node_ref(&self) -> NodeRef { + NodeRef::External(*self) + } +} + impl<'a> NodeRef<'a> { /// Makes a new node reference from an internal `VID`. /// Values are unchecked and the node is assumed to exist so use with caution! diff --git a/raphtory/src/core/entities/properties/graph_meta.rs b/raphtory/src/core/entities/properties/graph_meta.rs index 98ee907f86..ae4afc4613 100644 --- a/raphtory/src/core/entities/properties/graph_meta.rs +++ b/raphtory/src/core/entities/properties/graph_meta.rs @@ -5,7 +5,10 @@ use crate::core::{ Prop, PropType, }; use raphtory_api::core::storage::{ - arc_str::ArcStr, dict_mapper::DictMapper, locked_vec::ArcReadLockedVec, FxDashMap, + arc_str::ArcStr, + dict_mapper::{DictMapper, MaybeNew}, + locked_vec::ArcReadLockedVec, + FxDashMap, }; use serde::{Deserialize, Serialize}; use std::ops::{Deref, DerefMut}; @@ -44,7 +47,7 @@ impl GraphMeta { name: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { if is_static { Ok(self.constant_mapper.get_or_create_id(name)) } else { diff --git a/raphtory/src/core/entities/properties/props.rs b/raphtory/src/core/entities/properties/props.rs index b56c581e51..aa694c85cf 100644 --- a/raphtory/src/core/entities/properties/props.rs +++ b/raphtory/src/core/entities/properties/props.rs @@ -8,11 +8,13 @@ use crate::{ utils::errors::GraphError, Prop, PropType, }, - db::api::storage::tprop_storage_ops::TPropOps, + db::api::storage::graph::tprop_storage_ops::TPropOps, }; use parking_lot::RwLock; use raphtory_api::core::storage::{ - arc_str::ArcStr, dict_mapper::DictMapper, locked_vec::ArcReadLockedVec, + arc_str::ArcStr, + dict_mapper::{DictMapper, MaybeNew}, + locked_vec::ArcReadLockedVec, }; use serde::{Deserialize, Serialize}; use std::{fmt::Debug, hash::Hash, ops::Deref, sync::Arc}; @@ -153,7 +155,7 @@ impl Meta { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { if is_static { self.meta_prop_constant .get_or_create_and_validate(prop, dtype) @@ -173,7 +175,7 @@ impl Meta { } #[inline] - pub fn get_or_create_layer_id(&self, name: &str) -> usize { + pub fn get_or_create_layer_id(&self, name: &str) -> MaybeNew { self.meta_layer.get_or_create_id(name) } @@ -183,7 +185,7 @@ impl Meta { } #[inline] - pub fn get_or_create_node_type_id(&self, node_type: &str) -> usize { + pub fn get_or_create_node_type_id(&self, node_type: &str) -> MaybeNew { self.meta_node_type.get_or_create_id(node_type) } @@ -264,13 +266,14 @@ impl PropMapper { &self, prop: &str, dtype: PropType, - ) -> Result { - let id = self.id_mapper.get_or_create_id(prop); + ) -> Result, GraphError> { + let wrapped_id = self.id_mapper.get_or_create_id(prop); + let id = wrapped_id.inner(); let dtype_read = self.dtypes.read_recursive(); if let Some(old_type) = dtype_read.get(id) { if !matches!(old_type, PropType::Empty) { return if *old_type == dtype { - Ok(id) + Ok(wrapped_id) } else { Err(GraphError::PropertyTypeError { name: prop.to_owned(), @@ -287,11 +290,11 @@ impl PropMapper { if matches!(old_type, PropType::Empty) { // vector already resized but this id is not filled yet, set the dtype and return id dtype_write[id] = dtype; - Ok(id) + Ok(wrapped_id) } else { // already filled because a different thread won the race for this id, check the type matches if old_type == dtype { - Ok(id) + Ok(wrapped_id) } else { Err(GraphError::PropertyTypeError { name: prop.to_owned(), @@ -305,7 +308,7 @@ impl PropMapper { // vector not resized yet, resize it and set the dtype and return id dtype_write.resize(id + 1, PropType::Empty); dtype_write[id] = dtype; - Ok(id) + Ok(wrapped_id) } } } diff --git a/raphtory/src/core/entities/properties/tprop.rs b/raphtory/src/core/entities/properties/tprop.rs index 223854d356..3a8351e4f7 100644 --- a/raphtory/src/core/entities/properties/tprop.rs +++ b/raphtory/src/core/entities/properties/tprop.rs @@ -4,7 +4,7 @@ use crate::{ utils::errors::GraphError, DocumentInput, Prop, PropType, }, db::{ - api::storage::tprop_storage_ops::TPropOps, + api::storage::graph::tprop_storage_ops::TPropOps, graph::{graph::Graph, views::deletion_graph::PersistentGraph}, }, }; diff --git a/raphtory/src/core/storage/locked_view.rs b/raphtory/src/core/storage/locked_view.rs index a0addb3224..ee1d5df9f7 100644 --- a/raphtory/src/core/storage/locked_view.rs +++ b/raphtory/src/core/storage/locked_view.rs @@ -1,18 +1,17 @@ use dashmap::mapref::one::Ref; use parking_lot::{MappedRwLockReadGuard, RwLockReadGuard}; -use rustc_hash::FxHasher; use std::{ borrow::Borrow, cmp::Ordering, fmt::{Debug, Formatter}, - hash::{BuildHasherDefault, Hash, Hasher}, + hash::{Hash, Hasher}, ops::Deref, }; pub enum LockedView<'a, T> { LockMapped(MappedRwLockReadGuard<'a, T>), Locked(RwLockReadGuard<'a, T>), - DashMap(Ref<'a, usize, T, BuildHasherDefault>), + DashMap(Ref<'a, usize, T>), } impl<'a, T, O> AsRef for LockedView<'a, O> @@ -77,8 +76,8 @@ impl<'a, T> From> for LockedView<'a, T> { } } -impl<'a, T> From>> for LockedView<'a, T> { - fn from(value: Ref<'a, usize, T, BuildHasherDefault>) -> Self { +impl<'a, T> From> for LockedView<'a, T> { + fn from(value: Ref<'a, usize, T>) -> Self { Self::DashMap(value) } } diff --git a/raphtory/src/core/storage/raw_edges.rs b/raphtory/src/core/storage/raw_edges.rs index 5d16e08602..b2edcf5e77 100644 --- a/raphtory/src/core/storage/raw_edges.rs +++ b/raphtory/src/core/storage/raw_edges.rs @@ -18,7 +18,7 @@ use crate::{ edges::edge_store::{EdgeDataLike, EdgeLayer, EdgeStore}, LayerIds, }, - db::api::storage::edges::edge_storage_ops::{EdgeStorageOps, MemEdge}, + db::api::storage::graph::edges::edge_storage_ops::{EdgeStorageOps, MemEdge}, }; use super::{resolve, timeindex::TimeIndex}; diff --git a/raphtory/src/core/utils/errors.rs b/raphtory/src/core/utils/errors.rs index 7fad87b50f..6cdb238df0 100644 --- a/raphtory/src/core/utils/errors.rs +++ b/raphtory/src/core/utils/errors.rs @@ -1,6 +1,8 @@ use crate::core::{utils::time::error::ParseTimeError, Prop, PropType}; #[cfg(feature = "arrow")] use polars_arrow::legacy::error; +#[cfg(feature = "storage")] +use pometry_storage::RAError; #[cfg(feature = "python")] use pyo3::PyErr; use raphtory_api::core::{entities::GID, storage::arc_str::ArcStr}; @@ -120,12 +122,6 @@ pub enum GraphError { src: String, dst: String, }, - #[error("Bincode operation failed")] - BinCodeError { - #[from] - source: Box, - }, - #[error("The loaded graph is of the wrong type. Did you mean Graph / PersistentGraph?")] GraphLoadError, @@ -145,6 +141,10 @@ pub enum GraphError { )] ColumnDoesNotExist(String), + #[cfg(feature = "storage")] + #[error("Raphtory Arrow Error: {0}")] + DiskGraphError(#[from] RAError), + #[cfg(feature = "search")] #[error("Index operation failed")] IndexError { @@ -159,11 +159,6 @@ pub enum GraphError { source: QueryParserError, }, - #[error( - "Failed to load the graph as the bincode version {0} is different to supported version {1}" - )] - BincodeVersionError(u32, u32), - #[error("The layer_name function is only available once an edge has been exploded via .explode_layers() or .explode(). If you want to retrieve the layers for this edge you can use .layer_names")] LayerNameAPIError, @@ -185,6 +180,10 @@ pub enum GraphError { #[error("Failed to deserialise graph: {0}")] DeserialisationError(String), + #[cfg(feature = "proto")] + #[error("Cache is not initialised")] + CacheNotInnitialised, + #[error("Immutable graph is .. immutable!")] AttemptToMutateImmutableGraph, diff --git a/raphtory/src/db/api/mutation/addition_ops.rs b/raphtory/src/db/api/mutation/addition_ops.rs index fc31c05364..6068fab448 100644 --- a/raphtory/src/db/api/mutation/addition_ops.rs +++ b/raphtory/src/db/api/mutation/addition_ops.rs @@ -108,14 +108,18 @@ impl AdditionOps for G { props: PI, node_type: Option<&str>, ) -> Result, GraphError> { - let properties = props - .collect_properties(|name, dtype| self.resolve_node_property(name, dtype, false))?; let ti = time_from_input(self, t)?; - let v_id = self.resolve_node(v)?; - if let Some(node_type) = node_type { - self.set_node_type(v_id, node_type)?; - } - self.internal_add_node(ti, v_id, properties)?; + let properties = props.collect_properties(|name, dtype| { + Ok(self.resolve_node_property(name, dtype, false)?.inner()) + })?; + let v_id = match node_type { + None => self.resolve_node(v)?.inner(), + Some(node_type) => { + let (v_id, _) = self.resolve_node_and_type(v, node_type)?.inner(); + v_id.inner() + } + }; + self.internal_add_node(ti, v_id, &properties)?; Ok(NodeView::new_internal(self.clone(), v_id)) } @@ -128,13 +132,16 @@ impl AdditionOps for G { layer: Option<&str>, ) -> Result, GraphError> { let ti = time_from_input(self, t)?; - let src_id = self.resolve_node(src)?; - let dst_id = self.resolve_node(dst)?; - let layer_id = self.resolve_layer(layer)?; + let src_id = self.resolve_node(src)?.inner(); + let dst_id = self.resolve_node(dst)?.inner(); + let layer_id = self.resolve_layer(layer)?.inner(); - let properties: Vec<(usize, Prop)> = props - .collect_properties(|name, dtype| self.resolve_edge_property(name, dtype, false))?; - let eid = self.internal_add_edge(ti, src_id, dst_id, properties, layer_id)?; + let properties: Vec<(usize, Prop)> = props.collect_properties(|name, dtype| { + Ok(self.resolve_edge_property(name, dtype, false)?.inner()) + })?; + let eid = self + .internal_add_edge(ti, src_id, dst_id, &properties, layer_id)? + .inner(); Ok(EdgeView::new( self.clone(), EdgeRef::new_outgoing(eid, src_id, dst_id).at_layer(layer_id), diff --git a/raphtory/src/db/api/mutation/deletion_ops.rs b/raphtory/src/db/api/mutation/deletion_ops.rs index 007cdf4925..c441b9fd33 100644 --- a/raphtory/src/db/api/mutation/deletion_ops.rs +++ b/raphtory/src/db/api/mutation/deletion_ops.rs @@ -19,10 +19,11 @@ pub trait DeletionOps: InternalDeletionOps + InternalAdditionOps + Sized { layer: Option<&str>, ) -> Result<(), GraphError> { let ti = time_from_input(self, t)?; - let src_id = self.resolve_node(src)?; - let dst_id = self.resolve_node(dst)?; - let layer = self.resolve_layer(layer)?; - self.internal_delete_edge(ti, src_id, dst_id, layer) + let src_id = self.resolve_node(src)?.inner(); + let dst_id = self.resolve_node(dst)?.inner(); + let layer = self.resolve_layer(layer)?.inner(); + self.internal_delete_edge(ti, src_id, dst_id, layer)?; + Ok(()) } fn delete_edge_with_custom_time_format( diff --git a/raphtory/src/db/api/mutation/import_ops.rs b/raphtory/src/db/api/mutation/import_ops.rs index c5f1b32850..02cc94ce42 100644 --- a/raphtory/src/db/api/mutation/import_ops.rs +++ b/raphtory/src/db/api/mutation/import_ops.rs @@ -128,15 +128,17 @@ impl< if !force && self.node(node.id()).is_some() { return Err(NodeExistsError(node.id())); } - - let node_internal = self.resolve_node(node.id())?; - if let Some(node_type) = node.node_type().as_str() { - self.set_node_type(node_internal, node_type)?; - } + let node_internal = match node.node_type().as_str() { + None => self.resolve_node(node.id())?.inner(), + Some(node_type) => { + let (node_internal, _) = self.resolve_node_and_type(node.id(), node_type)?.inner(); + node_internal.inner() + } + }; for h in node.history() { let t = time_from_input(self, h)?; - self.internal_add_node(t, node_internal, vec![])?; + self.internal_add_node(t, node_internal, &[])?; } for (name, prop_view) in node.properties().temporal().iter() { let old_prop_id = node @@ -151,10 +153,10 @@ impl< .temporal_prop_meta() .get_dtype(old_prop_id) .unwrap(); - let new_prop_id = self.resolve_node_property(&name, dtype, false)?; + let new_prop_id = self.resolve_node_property(&name, dtype, false)?.inner(); for (h, prop) in prop_view.iter() { let t = time_from_input(self, h)?; - self.internal_add_node(t, node_internal, vec![(new_prop_id, prop)])?; + self.internal_add_node(t, node_internal, &[(new_prop_id, prop)])?; } } self.node(node.id()) @@ -212,9 +214,9 @@ impl< if self.include_deletions() { for t in edge.graph.edge_deletion_history(edge.edge, &layer_ids) { let ti = time_from_input(self, t)?; - let src_id = self.resolve_node(edge.src().id())?; - let dst_id = self.resolve_node(edge.dst().id())?; - let layer = self.resolve_layer(layer_name)?; + let src_id = self.resolve_node(edge.src().id())?.inner(); + let dst_id = self.resolve_node(edge.dst().id())?.inner(); + let layer = self.resolve_layer(layer_name)?.inner(); self.internal_delete_edge(ti, src_id, dst_id, layer)?; } } diff --git a/raphtory/src/db/api/mutation/internal/internal_addition_ops.rs b/raphtory/src/db/api/mutation/internal/internal_addition_ops.rs index b3a02bf3d3..e5ec60bbda 100644 --- a/raphtory/src/db/api/mutation/internal/internal_addition_ops.rs +++ b/raphtory/src/db/api/mutation/internal/internal_addition_ops.rs @@ -8,6 +8,7 @@ use crate::{ db::api::view::internal::Base, }; use enum_dispatch::enum_dispatch; +use raphtory_api::core::storage::dict_mapper::MaybeNew; #[enum_dispatch] pub trait InternalAdditionOps { @@ -15,12 +16,17 @@ pub trait InternalAdditionOps { fn next_event_id(&self) -> Result; /// map layer name to id and allocate a new layer if needed - fn resolve_layer(&self, layer: Option<&str>) -> Result; - - fn set_node_type(&self, v_id: VID, node_type: &str) -> Result<(), GraphError>; + fn resolve_layer(&self, layer: Option<&str>) -> Result, GraphError>; /// map external node id to internal id, allocating a new empty node if needed - fn resolve_node(&self, id: V) -> Result; + fn resolve_node(&self, id: V) -> Result, GraphError>; + + /// resolve a node and corresponding type, outer MaybeNew tracks whether the type assignment is new for the node even if both node and type already existed. + fn resolve_node_and_type( + &self, + id: V, + node_type: &str, + ) -> Result, MaybeNew)>, GraphError>; /// map property key to internal id, allocating new property if needed fn resolve_graph_property( @@ -28,7 +34,7 @@ pub trait InternalAdditionOps { prop: &str, dtype: PropType, is_static: bool, - ) -> Result; + ) -> Result, GraphError>; /// map property key to internal id, allocating new property if needed and checking property type. /// returns `None` if the type does not match @@ -37,21 +43,21 @@ pub trait InternalAdditionOps { prop: &str, dtype: PropType, is_static: bool, - ) -> Result; + ) -> Result, GraphError>; fn resolve_edge_property( &self, prop: &str, dtype: PropType, is_static: bool, - ) -> Result; + ) -> Result, GraphError>; /// add node update fn internal_add_node( &self, t: TimeIndexEntry, v: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError>; /// add edge update @@ -60,16 +66,16 @@ pub trait InternalAdditionOps { t: TimeIndexEntry, src: VID, dst: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, - ) -> Result; + ) -> Result, GraphError>; /// add update for an existing edge fn internal_add_edge_update( &self, t: TimeIndexEntry, edge: EID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, ) -> Result<(), GraphError>; } @@ -99,18 +105,22 @@ impl InternalAdditionOps for G { } #[inline] - fn resolve_layer(&self, layer: Option<&str>) -> Result { + fn resolve_layer(&self, layer: Option<&str>) -> Result, GraphError> { self.graph().resolve_layer(layer) } #[inline] - fn set_node_type(&self, v_id: VID, node_type: &str) -> Result<(), GraphError> { - self.graph().set_node_type(v_id, node_type) + fn resolve_node(&self, n: V) -> Result, GraphError> { + self.graph().resolve_node(n) } #[inline] - fn resolve_node(&self, n: V) -> Result { - self.graph().resolve_node(n) + fn resolve_node_and_type( + &self, + id: V, + node_type: &str, + ) -> Result, MaybeNew)>, GraphError> { + self.graph().resolve_node_and_type(id, node_type) } #[inline] @@ -119,7 +129,7 @@ impl InternalAdditionOps for G { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.graph().resolve_graph_property(prop, dtype, is_static) } @@ -129,7 +139,7 @@ impl InternalAdditionOps for G { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.graph().resolve_node_property(prop, dtype, is_static) } @@ -139,7 +149,7 @@ impl InternalAdditionOps for G { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.graph().resolve_edge_property(prop, dtype, is_static) } @@ -148,7 +158,7 @@ impl InternalAdditionOps for G { &self, t: TimeIndexEntry, v: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.graph().internal_add_node(t, v, props) } @@ -159,9 +169,9 @@ impl InternalAdditionOps for G { t: TimeIndexEntry, src: VID, dst: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, - ) -> Result { + ) -> Result, GraphError> { self.graph().internal_add_edge(t, src, dst, props, layer) } @@ -170,7 +180,7 @@ impl InternalAdditionOps for G { &self, t: TimeIndexEntry, edge: EID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, ) -> Result<(), GraphError> { self.graph().internal_add_edge_update(t, edge, props, layer) diff --git a/raphtory/src/db/api/mutation/internal/internal_deletion_ops.rs b/raphtory/src/db/api/mutation/internal/internal_deletion_ops.rs index 4585c62668..cd60c40ed5 100644 --- a/raphtory/src/db/api/mutation/internal/internal_deletion_ops.rs +++ b/raphtory/src/db/api/mutation/internal/internal_deletion_ops.rs @@ -2,7 +2,7 @@ use crate::{ core::{entities::VID, storage::timeindex::TimeIndexEntry, utils::errors::GraphError}, db::api::view::internal::Base, }; -use raphtory_api::core::entities::EID; +use raphtory_api::core::{entities::EID, storage::dict_mapper::MaybeNew}; pub trait InternalDeletionOps { fn internal_delete_edge( @@ -11,7 +11,7 @@ pub trait InternalDeletionOps { src: VID, dst: VID, layer: usize, - ) -> Result<(), GraphError>; + ) -> Result, GraphError>; fn internal_delete_existing_edge( &self, @@ -48,7 +48,7 @@ impl InternalDeletionOps for G { src: VID, dst: VID, layer: usize, - ) -> Result<(), GraphError> { + ) -> Result, GraphError> { self.graph().internal_delete_edge(t, src, dst, layer) } diff --git a/raphtory/src/db/api/mutation/internal/internal_property_additions_ops.rs b/raphtory/src/db/api/mutation/internal/internal_property_additions_ops.rs index f688fb612b..c5916ed316 100644 --- a/raphtory/src/db/api/mutation/internal/internal_property_additions_ops.rs +++ b/raphtory/src/db/api/mutation/internal/internal_property_additions_ops.rs @@ -15,41 +15,40 @@ pub trait InternalPropertyAdditionOps { fn internal_add_properties( &self, t: TimeIndexEntry, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError>; - fn internal_add_constant_properties(&self, props: Vec<(usize, Prop)>) - -> Result<(), GraphError>; + fn internal_add_constant_properties(&self, props: &[(usize, Prop)]) -> Result<(), GraphError>; fn internal_update_constant_properties( &self, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError>; fn internal_add_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError>; fn internal_update_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError>; fn internal_add_constant_edge_properties( &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError>; fn internal_update_constant_edge_properties( &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError>; } @@ -77,23 +76,20 @@ impl InternalPropertyAdditionOps for G { fn internal_add_properties( &self, t: TimeIndexEntry, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.graph().internal_add_properties(t, props) } #[inline(always)] - fn internal_add_constant_properties( - &self, - props: Vec<(usize, Prop)>, - ) -> Result<(), GraphError> { + fn internal_add_constant_properties(&self, props: &[(usize, Prop)]) -> Result<(), GraphError> { self.graph().internal_add_constant_properties(props) } #[inline(always)] fn internal_update_constant_properties( &self, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.graph().internal_update_constant_properties(props) } @@ -102,7 +98,7 @@ impl InternalPropertyAdditionOps for G { fn internal_add_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.graph() .internal_add_constant_node_properties(vid, props) @@ -112,7 +108,7 @@ impl InternalPropertyAdditionOps for G { fn internal_update_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.graph() .internal_update_constant_node_properties(vid, props) @@ -123,7 +119,7 @@ impl InternalPropertyAdditionOps for G { &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.graph() .internal_add_constant_edge_properties(eid, layer, props) @@ -134,7 +130,7 @@ impl InternalPropertyAdditionOps for G { &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.graph() .internal_update_constant_edge_properties(eid, layer, props) diff --git a/raphtory/src/db/api/mutation/property_addition_ops.rs b/raphtory/src/db/api/mutation/property_addition_ops.rs index abd6f0a1bb..7aec6632f6 100644 --- a/raphtory/src/db/api/mutation/property_addition_ops.rs +++ b/raphtory/src/db/api/mutation/property_addition_ops.rs @@ -29,23 +29,26 @@ impl PropertyAdditionOps f props: PI, ) -> Result<(), GraphError> { let ti = time_from_input(self, t)?; - let properties: Vec<_> = props - .collect_properties(|name, dtype| self.resolve_graph_property(name, dtype, false))?; - self.internal_add_properties(ti, properties) + let properties: Vec<_> = props.collect_properties(|name, dtype| { + Ok(self.resolve_graph_property(name, dtype, false)?.inner()) + })?; + self.internal_add_properties(ti, &properties) } fn add_constant_properties(&self, props: PI) -> Result<(), GraphError> { - let properties: Vec<_> = props - .collect_properties(|name, dtype| self.resolve_graph_property(name, dtype, true))?; - self.internal_add_constant_properties(properties) + let properties: Vec<_> = props.collect_properties(|name, dtype| { + Ok(self.resolve_graph_property(name, dtype, true)?.inner()) + })?; + self.internal_add_constant_properties(&properties) } fn update_constant_properties( &self, props: PI, ) -> Result<(), GraphError> { - let properties: Vec<_> = props - .collect_properties(|name, dtype| self.resolve_graph_property(name, dtype, true))?; - self.internal_update_constant_properties(properties) + let properties: Vec<_> = props.collect_properties(|name, dtype| { + Ok(self.resolve_graph_property(name, dtype, true)?.inner()) + })?; + self.internal_update_constant_properties(&properties) } } diff --git a/raphtory/src/db/api/state/lazy_node_state.rs b/raphtory/src/db/api/state/lazy_node_state.rs index 3470473fda..713fd20901 100644 --- a/raphtory/src/db/api/state/lazy_node_state.rs +++ b/raphtory/src/db/api/state/lazy_node_state.rs @@ -3,7 +3,7 @@ use crate::{ db::{ api::{ state::{NodeState, NodeStateOps}, - storage::{nodes::node_storage_ops::NodeStorageOps, storage_ops::GraphStorage}, + storage::graph::{nodes::node_storage_ops::NodeStorageOps, storage_ops::GraphStorage}, view::{internal::NodeList, IntoDynBoxed}, }, graph::node::NodeView, diff --git a/raphtory/src/db/api/storage/edges/edge_entry.rs b/raphtory/src/db/api/storage/graph/edges/edge_entry.rs similarity index 99% rename from raphtory/src/db/api/storage/edges/edge_entry.rs rename to raphtory/src/db/api/storage/graph/edges/edge_entry.rs index ccce6d6ae2..19949f36ca 100644 --- a/raphtory/src/db/api/storage/edges/edge_entry.rs +++ b/raphtory/src/db/api/storage/graph/edges/edge_entry.rs @@ -11,7 +11,7 @@ use crate::{ storage::raw_edges::EdgeRGuard, Prop, }, - db::api::storage::{ + db::api::storage::graph::{ edges::{ edge_ref::EdgeStorageRef, edge_storage_ops::{EdgeStorageOps, TimeIndexRef}, diff --git a/raphtory/src/db/api/storage/edges/edge_owned_entry.rs b/raphtory/src/db/api/storage/graph/edges/edge_owned_entry.rs similarity index 98% rename from raphtory/src/db/api/storage/edges/edge_owned_entry.rs rename to raphtory/src/db/api/storage/graph/edges/edge_owned_entry.rs index 51ab8b8c7e..fdd61040f7 100644 --- a/raphtory/src/db/api/storage/edges/edge_owned_entry.rs +++ b/raphtory/src/db/api/storage/graph/edges/edge_owned_entry.rs @@ -1,20 +1,10 @@ -use std::ops::Range; - -use rayon::iter::ParallelIterator; - -use raphtory_api::core::storage::timeindex::TimeIndexEntry; - -#[cfg(feature = "storage")] -use crate::db::api::storage::variants::storage_variants::StorageVariants; -#[cfg(feature = "storage")] -use crate::disk_graph::storage_interface::edge::DiskOwnedEdge; use crate::{ core::{ entities::{edges::edge_ref::EdgeRef, LayerIds, VID}, storage::raw_edges::EdgeArcGuard, Prop, }, - db::api::storage::{ + db::api::storage::graph::{ edges::{ edge_ref::EdgeStorageRef, edge_storage_ops::{EdgeStorageIntoOps, EdgeStorageOps, TimeIndexRef}, @@ -22,6 +12,14 @@ use crate::{ tprop_storage_ops::TPropOps, }, }; +use raphtory_api::core::storage::timeindex::TimeIndexEntry; +use rayon::iter::ParallelIterator; +use std::ops::Range; + +#[cfg(feature = "storage")] +use crate::db::api::storage::graph::variants::storage_variants::StorageVariants; +#[cfg(feature = "storage")] +use crate::disk_graph::storage_interface::edge::DiskOwnedEdge; #[derive(Debug, Clone)] pub enum EdgeOwnedEntry { diff --git a/raphtory/src/db/api/storage/edges/edge_ref.rs b/raphtory/src/db/api/storage/graph/edges/edge_ref.rs similarity index 97% rename from raphtory/src/db/api/storage/edges/edge_ref.rs rename to raphtory/src/db/api/storage/graph/edges/edge_ref.rs index b5c7ce4b0e..37e4047228 100644 --- a/raphtory/src/db/api/storage/edges/edge_ref.rs +++ b/raphtory/src/db/api/storage/graph/edges/edge_ref.rs @@ -1,22 +1,21 @@ -use std::ops::Range; - -use rayon::prelude::*; - use super::edge_storage_ops::MemEdge; -#[cfg(feature = "storage")] -use crate::db::api::storage::variants::storage_variants::StorageVariants; -#[cfg(feature = "storage")] -use crate::disk_graph::storage_interface::edge::DiskEdge; use crate::{ core::{ entities::{edges::edge_ref::EdgeRef, LayerIds, EID, VID}, Prop, }, - db::api::storage::{ + db::api::storage::graph::{ edges::edge_storage_ops::{EdgeStorageOps, TimeIndexRef}, tprop_storage_ops::TPropOps, }, }; +use rayon::prelude::*; +use std::ops::Range; + +#[cfg(feature = "storage")] +use crate::db::api::storage::graph::variants::storage_variants::StorageVariants; +#[cfg(feature = "storage")] +use crate::disk_graph::storage_interface::edge::DiskEdge; macro_rules! for_all { ($value:expr, $pattern:pat => $result:expr) => { diff --git a/raphtory/src/db/api/storage/edges/edge_storage_ops.rs b/raphtory/src/db/api/storage/graph/edges/edge_storage_ops.rs similarity index 99% rename from raphtory/src/db/api/storage/edges/edge_storage_ops.rs rename to raphtory/src/db/api/storage/graph/edges/edge_storage_ops.rs index 178748ac2b..81e134caad 100644 --- a/raphtory/src/db/api/storage/edges/edge_storage_ops.rs +++ b/raphtory/src/db/api/storage/graph/edges/edge_storage_ops.rs @@ -12,7 +12,7 @@ use crate::{ Prop, }, db::api::{ - storage::{tprop_storage_ops::TPropOps, variants::layer_variants::LayerVariants}, + storage::graph::{tprop_storage_ops::TPropOps, variants::layer_variants::LayerVariants}, view::IntoDynBoxed, }, }; diff --git a/raphtory/src/db/api/storage/edges/edges.rs b/raphtory/src/db/api/storage/graph/edges/edges.rs similarity index 99% rename from raphtory/src/db/api/storage/edges/edges.rs rename to raphtory/src/db/api/storage/graph/edges/edges.rs index ef56ea2778..01d08a4bf9 100644 --- a/raphtory/src/db/api/storage/edges/edges.rs +++ b/raphtory/src/db/api/storage/graph/edges/edges.rs @@ -3,7 +3,7 @@ use super::{edge_entry::EdgeStorageEntry, unlocked::UnlockedEdges}; use crate::disk_graph::storage_interface::{edges::DiskEdges, edges_ref::DiskEdgesRef}; use crate::{ core::{entities::LayerIds, storage::raw_edges::LockedEdges}, - db::api::storage::{ + db::api::storage::graph::{ edges::edge_storage_ops::EdgeStorageOps, variants::storage_variants3::StorageVariants, }, }; diff --git a/raphtory/src/db/api/storage/edges/mod.rs b/raphtory/src/db/api/storage/graph/edges/mod.rs similarity index 100% rename from raphtory/src/db/api/storage/edges/mod.rs rename to raphtory/src/db/api/storage/graph/edges/mod.rs diff --git a/raphtory/src/db/api/storage/edges/unlocked.rs b/raphtory/src/db/api/storage/graph/edges/unlocked.rs similarity index 100% rename from raphtory/src/db/api/storage/edges/unlocked.rs rename to raphtory/src/db/api/storage/graph/edges/unlocked.rs diff --git a/raphtory/src/db/api/storage/locked.rs b/raphtory/src/db/api/storage/graph/locked.rs similarity index 100% rename from raphtory/src/db/api/storage/locked.rs rename to raphtory/src/db/api/storage/graph/locked.rs diff --git a/raphtory/src/db/api/storage/graph/mod.rs b/raphtory/src/db/api/storage/graph/mod.rs new file mode 100644 index 0000000000..7f8f256ae9 --- /dev/null +++ b/raphtory/src/db/api/storage/graph/mod.rs @@ -0,0 +1,6 @@ +pub mod edges; +pub mod locked; +pub mod nodes; +pub mod storage_ops; +pub mod tprop_storage_ops; +pub mod variants; diff --git a/raphtory/src/db/api/storage/nodes/mod.rs b/raphtory/src/db/api/storage/graph/nodes/mod.rs similarity index 100% rename from raphtory/src/db/api/storage/nodes/mod.rs rename to raphtory/src/db/api/storage/graph/nodes/mod.rs diff --git a/raphtory/src/db/api/storage/nodes/node_entry.rs b/raphtory/src/db/api/storage/graph/nodes/node_entry.rs similarity index 99% rename from raphtory/src/db/api/storage/nodes/node_entry.rs rename to raphtory/src/db/api/storage/graph/nodes/node_entry.rs index 4c1401185b..4210dd7161 100644 --- a/raphtory/src/db/api/storage/nodes/node_entry.rs +++ b/raphtory/src/db/api/storage/graph/nodes/node_entry.rs @@ -10,7 +10,7 @@ use crate::{ Direction, }, db::api::{ - storage::{ + storage::graph::{ nodes::{node_ref::NodeStorageRef, node_storage_ops::NodeStorageOps}, tprop_storage_ops::TPropOps, variants::storage_variants3::StorageVariants, diff --git a/raphtory/src/db/api/storage/nodes/node_owned_entry.rs b/raphtory/src/db/api/storage/graph/nodes/node_owned_entry.rs similarity index 93% rename from raphtory/src/db/api/storage/nodes/node_owned_entry.rs rename to raphtory/src/db/api/storage/graph/nodes/node_owned_entry.rs index 70b16b6b27..b223e29217 100644 --- a/raphtory/src/db/api/storage/nodes/node_owned_entry.rs +++ b/raphtory/src/db/api/storage/graph/nodes/node_owned_entry.rs @@ -10,7 +10,7 @@ use crate::{ storage::ArcEntry, Direction, }, - db::api::storage::nodes::node_storage_ops::NodeStorageIntoOps, + db::api::storage::graph::nodes::node_storage_ops::NodeStorageIntoOps, }; pub enum NodeOwnedEntry { diff --git a/raphtory/src/db/api/storage/nodes/node_ref.rs b/raphtory/src/db/api/storage/graph/nodes/node_ref.rs similarity index 94% rename from raphtory/src/db/api/storage/nodes/node_ref.rs rename to raphtory/src/db/api/storage/graph/nodes/node_ref.rs index 4e094a6691..e4cf012ef4 100644 --- a/raphtory/src/db/api/storage/nodes/node_ref.rs +++ b/raphtory/src/db/api/storage/graph/nodes/node_ref.rs @@ -1,23 +1,21 @@ -use std::borrow::Cow; - -use raphtory_api::core::entities::GidRef; - -#[cfg(feature = "storage")] -use crate::db::api::storage::variants::storage_variants::StorageVariants; - -#[cfg(feature = "storage")] -use crate::disk_graph::storage_interface::node::DiskNode; use crate::{ core::{ entities::{edges::edge_ref::EdgeRef, nodes::node_store::NodeStore, LayerIds, VID}, Direction, }, db::api::{ - storage::{nodes::node_storage_ops::NodeStorageOps, tprop_storage_ops::TPropOps}, + storage::graph::{nodes::node_storage_ops::NodeStorageOps, tprop_storage_ops::TPropOps}, view::internal::NodeAdditions, }, prelude::Prop, }; +use raphtory_api::core::entities::GidRef; +use std::borrow::Cow; + +#[cfg(feature = "storage")] +use crate::db::api::storage::graph::variants::storage_variants::StorageVariants; +#[cfg(feature = "storage")] +use crate::disk_graph::storage_interface::node::DiskNode; #[derive(Copy, Clone, Debug)] pub enum NodeStorageRef<'a> { diff --git a/raphtory/src/db/api/storage/nodes/node_storage_ops.rs b/raphtory/src/db/api/storage/graph/nodes/node_storage_ops.rs similarity index 96% rename from raphtory/src/db/api/storage/nodes/node_storage_ops.rs rename to raphtory/src/db/api/storage/graph/nodes/node_storage_ops.rs index b997680135..eca6e956c2 100644 --- a/raphtory/src/db/api/storage/nodes/node_storage_ops.rs +++ b/raphtory/src/db/api/storage/graph/nodes/node_storage_ops.rs @@ -9,7 +9,7 @@ use crate::{ storage::ArcEntry, Direction, }, - db::api::{storage::tprop_storage_ops::TPropOps, view::internal::NodeAdditions}, + db::api::{storage::graph::tprop_storage_ops::TPropOps, view::internal::NodeAdditions}, prelude::Prop, }; use itertools::Itertools; diff --git a/raphtory/src/db/api/storage/nodes/nodes.rs b/raphtory/src/db/api/storage/graph/nodes/nodes.rs similarity index 94% rename from raphtory/src/db/api/storage/nodes/nodes.rs rename to raphtory/src/db/api/storage/graph/nodes/nodes.rs index 9699297401..d545e983a6 100644 --- a/raphtory/src/db/api/storage/nodes/nodes.rs +++ b/raphtory/src/db/api/storage/graph/nodes/nodes.rs @@ -1,15 +1,15 @@ -#[cfg(feature = "storage")] -use crate::disk_graph::storage_interface::nodes::DiskNodesOwned; +use super::node_ref::NodeStorageRef; use crate::{ core::{ entities::{nodes::node_store::NodeStore, VID}, storage::ReadLockedStorage, }, - db::api::storage::nodes::nodes_ref::NodesStorageEntry, + db::api::storage::graph::nodes::nodes_ref::NodesStorageEntry, }; use std::sync::Arc; -use super::node_ref::NodeStorageRef; +#[cfg(feature = "storage")] +use crate::disk_graph::storage_interface::nodes::DiskNodesOwned; pub enum NodesStorage { Mem(Arc>), diff --git a/raphtory/src/db/api/storage/nodes/nodes_ref.rs b/raphtory/src/db/api/storage/graph/nodes/nodes_ref.rs similarity index 96% rename from raphtory/src/db/api/storage/nodes/nodes_ref.rs rename to raphtory/src/db/api/storage/graph/nodes/nodes_ref.rs index 47abbadf3f..2fe0928a85 100644 --- a/raphtory/src/db/api/storage/nodes/nodes_ref.rs +++ b/raphtory/src/db/api/storage/graph/nodes/nodes_ref.rs @@ -1,18 +1,17 @@ -#[cfg(feature = "storage")] -use crate::db::api::storage::variants::storage_variants3::StorageVariants; -#[cfg(feature = "storage")] -use crate::disk_graph::storage_interface::nodes_ref::DiskNodesRef; - -#[cfg(not(feature = "storage"))] -use either::Either; - +use super::node_ref::NodeStorageRef; use crate::core::{ entities::{nodes::node_store::NodeStore, VID}, storage::ReadLockedStorage, }; use rayon::iter::ParallelIterator; -use super::node_ref::NodeStorageRef; +#[cfg(feature = "storage")] +use crate::db::api::storage::graph::variants::storage_variants3::StorageVariants; +#[cfg(feature = "storage")] +use crate::disk_graph::storage_interface::nodes_ref::DiskNodesRef; + +#[cfg(not(feature = "storage"))] +use either::Either; #[derive(Debug)] pub enum NodesStorageEntry<'a> { diff --git a/raphtory/src/db/api/storage/storage_ops/additions.rs b/raphtory/src/db/api/storage/graph/storage_ops/additions.rs similarity index 71% rename from raphtory/src/db/api/storage/storage_ops/additions.rs rename to raphtory/src/db/api/storage/graph/storage_ops/additions.rs index 42c81a6796..4372e6ab85 100644 --- a/raphtory/src/db/api/storage/storage_ops/additions.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/additions.rs @@ -1,10 +1,3 @@ -use either::Either; -use raphtory_api::core::{ - entities::{EID, VID}, - storage::timeindex::TimeIndexEntry, -}; -use std::sync::atomic::Ordering; - use super::GraphStorage; use crate::{ core::{ @@ -18,40 +11,25 @@ use crate::{ db::api::mutation::internal::InternalAdditionOps, prelude::Prop, }; +use either::Either; +use raphtory_api::core::{ + entities::{EID, VID}, + storage::{dict_mapper::MaybeNew, timeindex::TimeIndexEntry}, +}; +use std::sync::atomic::Ordering; impl InternalAdditionOps for TemporalGraph { fn next_event_id(&self) -> Result { Ok(self.event_counter.fetch_add(1, Ordering::Relaxed)) } - fn resolve_layer(&self, layer: Option<&str>) -> Result { + fn resolve_layer(&self, layer: Option<&str>) -> Result, GraphError> { Ok(layer .map(|name| self.edge_meta.get_or_create_layer_id(name)) - .unwrap_or(0)) - } - - fn set_node_type(&self, v_id: VID, node_type: &str) -> Result<(), GraphError> { - let mut node = self.storage.get_node_mut(v_id); - if node_type == "_default" { - return Err(GraphError::NodeTypeError( - "_default type is not allowed to be used on nodes".to_string(), - )); - } - if node.node_type == 0 { - let node_type_id = self.node_meta.get_or_create_node_type_id(node_type); - node.update_node_type(node_type_id); - } else { - let new_node_type_id = self.node_meta.get_node_type_id(node_type).unwrap_or(0); - if node.node_type != new_node_type_id { - return Err(GraphError::NodeTypeError( - "Node already has a non-default type".to_string(), - )); - } - } - Ok(()) + .unwrap_or(MaybeNew::Existing(0))) } - fn resolve_node(&self, n: V) -> Result { + fn resolve_node(&self, n: V) -> Result, GraphError> { match n.as_gid_ref() { Either::Left(id) => { let ref_mut = self.logical_to_physical.get_or_init(id, || { @@ -60,7 +38,38 @@ impl InternalAdditionOps for TemporalGraph { })?; Ok(ref_mut) } - Either::Right(vid) => Ok(vid), + Either::Right(vid) => Ok(MaybeNew::Existing(vid)), + } + } + + fn resolve_node_and_type( + &self, + id: V, + node_type: &str, + ) -> Result, MaybeNew)>, GraphError> { + if node_type == "_default" { + return Err(GraphError::NodeTypeError( + "_default type is not allowed to be used on nodes".to_string(), + )); + } + let vid = self.resolve_node(id)?; + let mut node_store = self.storage.get_node_mut(vid.inner()); + if node_store.node_type == 0 { + let node_type_id = self.node_meta.get_or_create_node_type_id(node_type); + node_store.update_node_type(node_type_id.inner()); + Ok(MaybeNew::New((vid, node_type_id))) + } else { + let node_type_id = self + .node_meta + .get_node_type_id(node_type) + .ok_or_else(|| GraphError::NodeTypeError("Cannot change node type".to_string()))?; + if node_type_id == node_store.node_type { + Ok(MaybeNew::Existing((vid, MaybeNew::Existing(node_type_id)))) + } else { + Err(GraphError::NodeTypeError( + "Cannot change node type".to_string(), + )) + } } } @@ -69,7 +78,7 @@ impl InternalAdditionOps for TemporalGraph { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.graph_meta.resolve_property(prop, dtype, is_static) } @@ -78,7 +87,7 @@ impl InternalAdditionOps for TemporalGraph { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.node_meta.resolve_prop_id(prop, dtype, is_static) } @@ -87,7 +96,7 @@ impl InternalAdditionOps for TemporalGraph { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.edge_meta.resolve_prop_id(prop, dtype, is_static) } @@ -95,7 +104,7 @@ impl InternalAdditionOps for TemporalGraph { &self, t: TimeIndexEntry, v: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { self.update_time(t); // get the node and update the time index @@ -103,7 +112,7 @@ impl InternalAdditionOps for TemporalGraph { node.update_time(t); for (id, prop) in props { let prop = self.process_prop_value(prop); - node.add_prop(t, id, prop)?; + node.add_prop(t, *id, prop)?; } Ok(()) } @@ -113,16 +122,16 @@ impl InternalAdditionOps for TemporalGraph { t: TimeIndexEntry, src: VID, dst: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, - ) -> Result { + ) -> Result, GraphError> { self.link_nodes(src, dst, t, layer, move |edge| { edge.additions_mut(layer).insert(t); if !props.is_empty() { let edge_layer = edge.layer_mut(layer); for (prop_id, prop) in props { let prop = self.process_prop_value(prop); - edge_layer.add_prop(t, prop_id, prop)?; + edge_layer.add_prop(t, *prop_id, prop)?; } } Ok(()) @@ -133,7 +142,7 @@ impl InternalAdditionOps for TemporalGraph { &self, t: TimeIndexEntry, edge: EID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, ) -> Result<(), GraphError> { self.link_edge(edge, t, layer, |edge| { @@ -142,7 +151,7 @@ impl InternalAdditionOps for TemporalGraph { let edge_layer = edge.layer_mut(layer); for (prop_id, prop) in props { let prop = self.process_prop_value(prop); - edge_layer.add_prop(t, prop_id, prop)?; + edge_layer.add_prop(t, *prop_id, prop)?; } } Ok(()) @@ -158,23 +167,27 @@ impl InternalAdditionOps for GraphStorage { } } - fn resolve_layer(&self, layer: Option<&str>) -> Result { + fn resolve_layer(&self, layer: Option<&str>) -> Result, GraphError> { match self { GraphStorage::Unlocked(storage) => storage.resolve_layer(layer), _ => Err(GraphError::AttemptToMutateImmutableGraph), } } - fn set_node_type(&self, v_id: VID, node_type: &str) -> Result<(), GraphError> { + fn resolve_node(&self, n: V) -> Result, GraphError> { match self { - GraphStorage::Unlocked(storage) => storage.set_node_type(v_id, node_type), + GraphStorage::Unlocked(storage) => storage.resolve_node(n), _ => Err(GraphError::AttemptToMutateImmutableGraph), } } - fn resolve_node(&self, n: V) -> Result { + fn resolve_node_and_type( + &self, + id: V, + node_type: &str, + ) -> Result, MaybeNew)>, GraphError> { match self { - GraphStorage::Unlocked(storage) => storage.resolve_node(n), + GraphStorage::Unlocked(storage) => storage.resolve_node_and_type(id, node_type), _ => Err(GraphError::AttemptToMutateImmutableGraph), } } @@ -184,7 +197,7 @@ impl InternalAdditionOps for GraphStorage { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { match self { GraphStorage::Unlocked(storage) => { storage.resolve_graph_property(prop, dtype, is_static) @@ -198,7 +211,7 @@ impl InternalAdditionOps for GraphStorage { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { match self { GraphStorage::Unlocked(storage) => { storage.resolve_node_property(prop, dtype, is_static) @@ -212,7 +225,7 @@ impl InternalAdditionOps for GraphStorage { prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { match self { GraphStorage::Unlocked(storage) => { storage.resolve_edge_property(prop, dtype, is_static) @@ -225,7 +238,7 @@ impl InternalAdditionOps for GraphStorage { &self, t: TimeIndexEntry, v: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => storage.internal_add_node(t, v, props), @@ -238,9 +251,9 @@ impl InternalAdditionOps for GraphStorage { t: TimeIndexEntry, src: VID, dst: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, - ) -> Result { + ) -> Result, GraphError> { match self { GraphStorage::Unlocked(storage) => storage.internal_add_edge(t, src, dst, props, layer), _ => Err(GraphError::AttemptToMutateImmutableGraph), @@ -251,7 +264,7 @@ impl InternalAdditionOps for GraphStorage { &self, t: TimeIndexEntry, edge: EID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], layer: usize, ) -> Result<(), GraphError> { match self { diff --git a/raphtory/src/db/api/storage/storage_ops/const_props.rs b/raphtory/src/db/api/storage/graph/storage_ops/const_props.rs similarity index 100% rename from raphtory/src/db/api/storage/storage_ops/const_props.rs rename to raphtory/src/db/api/storage/graph/storage_ops/const_props.rs diff --git a/raphtory/src/db/api/storage/storage_ops/deletions.rs b/raphtory/src/db/api/storage/graph/storage_ops/deletions.rs similarity index 90% rename from raphtory/src/db/api/storage/storage_ops/deletions.rs rename to raphtory/src/db/api/storage/graph/storage_ops/deletions.rs index f9e4ef7f93..febabaef18 100644 --- a/raphtory/src/db/api/storage/storage_ops/deletions.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/deletions.rs @@ -5,7 +5,7 @@ use crate::{ }; use raphtory_api::core::{ entities::{EID, VID}, - storage::timeindex::TimeIndexEntry, + storage::{dict_mapper::MaybeNew, timeindex::TimeIndexEntry}, }; impl InternalDeletionOps for TemporalGraph { @@ -15,12 +15,11 @@ impl InternalDeletionOps for TemporalGraph { src: VID, dst: VID, layer: usize, - ) -> Result<(), GraphError> { + ) -> Result, GraphError> { self.link_nodes(src, dst, t, layer, |new_edge| { new_edge.deletions_mut(layer).insert(t); Ok(()) - })?; - Ok(()) + }) } fn internal_delete_existing_edge( @@ -44,7 +43,7 @@ impl InternalDeletionOps for GraphStorage { src: VID, dst: VID, layer: usize, - ) -> Result<(), GraphError> { + ) -> Result, GraphError> { match self { GraphStorage::Unlocked(storage) => storage.internal_delete_edge(t, src, dst, layer), _ => Err(GraphError::AttemptToMutateImmutableGraph), diff --git a/raphtory/src/db/api/storage/storage_ops/edge_filter.rs b/raphtory/src/db/api/storage/graph/storage_ops/edge_filter.rs similarity index 81% rename from raphtory/src/db/api/storage/storage_ops/edge_filter.rs rename to raphtory/src/db/api/storage/graph/storage_ops/edge_filter.rs index 4b0221b3b6..2eaf9ab08b 100644 --- a/raphtory/src/db/api/storage/storage_ops/edge_filter.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/edge_filter.rs @@ -1,10 +1,9 @@ +use super::GraphStorage; use crate::{ core::entities::LayerIds, - db::api::{storage::edges::edge_ref::EdgeStorageRef, view::internal::EdgeFilterOps}, + db::api::{storage::graph::edges::edge_ref::EdgeStorageRef, view::internal::EdgeFilterOps}, }; -use super::GraphStorage; - impl EdgeFilterOps for GraphStorage { fn edges_filtered(&self) -> bool { false diff --git a/raphtory/src/db/api/storage/storage_ops/layer_ops.rs b/raphtory/src/db/api/storage/graph/storage_ops/layer_ops.rs similarity index 100% rename from raphtory/src/db/api/storage/storage_ops/layer_ops.rs rename to raphtory/src/db/api/storage/graph/storage_ops/layer_ops.rs diff --git a/raphtory/src/db/api/storage/storage_ops/list_ops.rs b/raphtory/src/db/api/storage/graph/storage_ops/list_ops.rs similarity index 100% rename from raphtory/src/db/api/storage/storage_ops/list_ops.rs rename to raphtory/src/db/api/storage/graph/storage_ops/list_ops.rs diff --git a/raphtory/src/db/api/storage/storage_ops/materialize.rs b/raphtory/src/db/api/storage/graph/storage_ops/materialize.rs similarity index 100% rename from raphtory/src/db/api/storage/storage_ops/materialize.rs rename to raphtory/src/db/api/storage/graph/storage_ops/materialize.rs diff --git a/raphtory/src/db/api/storage/storage_ops/mod.rs b/raphtory/src/db/api/storage/graph/storage_ops/mod.rs similarity index 98% rename from raphtory/src/db/api/storage/storage_ops/mod.rs rename to raphtory/src/db/api/storage/graph/storage_ops/mod.rs index 693e2c6b3a..668686efbe 100644 --- a/raphtory/src/db/api/storage/storage_ops/mod.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/mod.rs @@ -1,5 +1,9 @@ -#[cfg(feature = "storage")] -use crate::disk_graph::{storage_interface::edge::DiskOwnedEdge, DiskGraphStorage}; +use super::{ + edges::{ + edge_entry::EdgeStorageEntry, edge_owned_entry::EdgeOwnedEntry, unlocked::UnlockedEdges, + }, + nodes::node_entry::NodeStorageEntry, +}; use crate::{ core::{ entities::{ @@ -12,7 +16,7 @@ use crate::{ Direction, }, db::api::{ - storage::{ + storage::graph::{ edges::{ edge_ref::EdgeStorageRef, edge_storage_ops::EdgeStorageOps, @@ -39,21 +43,18 @@ use std::{iter, sync::Arc}; #[cfg(feature = "storage")] use crate::{ - db::api::storage::variants::storage_variants::StorageVariants, - disk_graph::storage_interface::{ - edges::DiskEdges, - edges_ref::DiskEdgesRef, - node::{DiskNode, DiskOwnedNode}, - nodes::DiskNodesOwned, - nodes_ref::DiskNodesRef, - }, -}; - -use super::{ - edges::{ - edge_entry::EdgeStorageEntry, edge_owned_entry::EdgeOwnedEntry, unlocked::UnlockedEdges, + db::api::storage::graph::variants::storage_variants::StorageVariants, + disk_graph::{ + storage_interface::{ + edge::DiskOwnedEdge, + edges::DiskEdges, + edges_ref::DiskEdgesRef, + node::{DiskNode, DiskOwnedNode}, + nodes::DiskNodesOwned, + nodes_ref::DiskNodesRef, + }, + DiskGraphStorage, }, - nodes::node_entry::NodeStorageEntry, }; pub mod additions; diff --git a/raphtory/src/db/api/storage/storage_ops/node_filter.rs b/raphtory/src/db/api/storage/graph/storage_ops/node_filter.rs similarity index 79% rename from raphtory/src/db/api/storage/storage_ops/node_filter.rs rename to raphtory/src/db/api/storage/graph/storage_ops/node_filter.rs index 8e9f240b73..c7e7ca3a21 100644 --- a/raphtory/src/db/api/storage/storage_ops/node_filter.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/node_filter.rs @@ -1,10 +1,9 @@ +use super::GraphStorage; use crate::{ core::entities::LayerIds, - db::api::{storage::nodes::node_ref::NodeStorageRef, view::internal::NodeFilterOps}, + db::api::{storage::graph::nodes::node_ref::NodeStorageRef, view::internal::NodeFilterOps}, }; -use super::GraphStorage; - impl NodeFilterOps for GraphStorage { #[inline] fn node_list_trusted(&self) -> bool { diff --git a/raphtory/src/db/api/storage/storage_ops/prop_add.rs b/raphtory/src/db/api/storage/graph/storage_ops/prop_add.rs similarity index 72% rename from raphtory/src/db/api/storage/storage_ops/prop_add.rs rename to raphtory/src/db/api/storage/graph/storage_ops/prop_add.rs index d59cfb52dd..c042e14593 100644 --- a/raphtory/src/db/api/storage/storage_ops/prop_add.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/prop_add.rs @@ -14,33 +14,30 @@ impl InternalPropertyAdditionOps for TemporalGraph { fn internal_add_properties( &self, t: TimeIndexEntry, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { for (prop_id, prop) in props { let prop = self.process_prop_value(prop); - self.graph_meta.add_prop(t, prop_id, prop)?; + self.graph_meta.add_prop(t, *prop_id, prop)?; } Ok(()) } - fn internal_add_constant_properties( - &self, - props: Vec<(usize, Prop)>, - ) -> Result<(), GraphError> { + fn internal_add_constant_properties(&self, props: &[(usize, Prop)]) -> Result<(), GraphError> { for (id, prop) in props { let prop = self.process_prop_value(prop); - self.graph_meta.add_constant_prop(id, prop)?; + self.graph_meta.add_constant_prop(*id, prop)?; } Ok(()) } fn internal_update_constant_properties( &self, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { for (id, prop) in props { let prop = self.process_prop_value(prop); - self.graph_meta.update_constant_prop(id, prop)?; + self.graph_meta.update_constant_prop(*id, prop)?; } Ok(()) } @@ -48,13 +45,13 @@ impl InternalPropertyAdditionOps for TemporalGraph { fn internal_add_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { let mut node = self.storage.get_node_mut(vid); for (prop_id, prop) in props { let prop = self.process_prop_value(prop); - node.add_constant_prop(prop_id, prop).map_err(|err| { - let name = self.node_meta.get_prop_name(prop_id, true); + node.add_constant_prop(*prop_id, prop).map_err(|err| { + let name = self.node_meta.get_prop_name(*prop_id, true); GraphError::ConstantPropertyMutationError { name, new: err.new_value.expect("new value exists"), @@ -70,12 +67,12 @@ impl InternalPropertyAdditionOps for TemporalGraph { fn internal_update_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { let mut node = self.storage.get_node_mut(vid); for (prop_id, prop) in props { let prop = self.process_prop_value(prop); - node.update_constant_prop(prop_id, prop)?; + node.update_constant_prop(*prop_id, prop)?; } Ok(()) } @@ -84,22 +81,24 @@ impl InternalPropertyAdditionOps for TemporalGraph { &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { let mut edge = self.storage.get_edge_mut(eid); let edge_layer = edge.layer_mut(layer); for (prop_id, prop) in props { let prop = self.process_prop_value(prop); - edge_layer.add_constant_prop(prop_id, prop).map_err(|err| { - let name = self.edge_meta.get_prop_name(prop_id, true); - GraphError::ConstantPropertyMutationError { - name, - new: err.new_value.expect("new value exists"), - old: err - .previous_value - .expect("previous value exists if set failed"), - } - })?; + edge_layer + .add_constant_prop(*prop_id, prop) + .map_err(|err| { + let name = self.edge_meta.get_prop_name(*prop_id, true); + GraphError::ConstantPropertyMutationError { + name, + new: err.new_value.expect("new value exists"), + old: err + .previous_value + .expect("previous value exists if set failed"), + } + })?; } Ok(()) } @@ -108,13 +107,13 @@ impl InternalPropertyAdditionOps for TemporalGraph { &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { let mut edge = self.storage.get_edge_mut(eid); let edge_layer = edge.layer_mut(layer); for (prop_id, prop) in props { let prop = self.process_prop_value(prop); - edge_layer.update_constant_prop(prop_id, prop)?; + edge_layer.update_constant_prop(*prop_id, prop)?; } Ok(()) } @@ -124,7 +123,7 @@ impl InternalPropertyAdditionOps for GraphStorage { fn internal_add_properties( &self, t: TimeIndexEntry, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => storage.internal_add_properties(t, props), @@ -132,10 +131,7 @@ impl InternalPropertyAdditionOps for GraphStorage { } } - fn internal_add_constant_properties( - &self, - props: Vec<(usize, Prop)>, - ) -> Result<(), GraphError> { + fn internal_add_constant_properties(&self, props: &[(usize, Prop)]) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => storage.internal_add_constant_properties(props), _ => Err(GraphError::AttemptToMutateImmutableGraph), @@ -144,7 +140,7 @@ impl InternalPropertyAdditionOps for GraphStorage { fn internal_update_constant_properties( &self, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => storage.internal_update_constant_properties(props), @@ -155,7 +151,7 @@ impl InternalPropertyAdditionOps for GraphStorage { fn internal_add_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => { @@ -168,7 +164,7 @@ impl InternalPropertyAdditionOps for GraphStorage { fn internal_update_constant_node_properties( &self, vid: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => { @@ -182,7 +178,7 @@ impl InternalPropertyAdditionOps for GraphStorage { &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => { @@ -196,16 +192,11 @@ impl InternalPropertyAdditionOps for GraphStorage { &self, eid: EID, layer: usize, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { match self { GraphStorage::Unlocked(storage) => { - let mut edge = storage.storage.get_edge_mut(eid); - let edge_layer = edge.layer_mut(layer); - for (prop_id, value) in props { - edge_layer.update_constant_prop(prop_id, value)?; - } - Ok(()) + storage.internal_update_constant_edge_properties(eid, layer, props) } _ => Err(GraphError::AttemptToMutateImmutableGraph), } diff --git a/raphtory/src/db/api/storage/storage_ops/time_props.rs b/raphtory/src/db/api/storage/graph/storage_ops/time_props.rs similarity index 97% rename from raphtory/src/db/api/storage/storage_ops/time_props.rs rename to raphtory/src/db/api/storage/graph/storage_ops/time_props.rs index f62d8bfa23..1a05f74d43 100644 --- a/raphtory/src/db/api/storage/storage_ops/time_props.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/time_props.rs @@ -5,7 +5,7 @@ use raphtory_api::core::storage::{arc_str::ArcStr, timeindex::AsTime}; use crate::{ db::api::{ properties::internal::{TemporalPropertiesOps, TemporalPropertyViewOps}, - storage::tprop_storage_ops::TPropOps, + storage::graph::tprop_storage_ops::TPropOps, }, prelude::Prop, }; diff --git a/raphtory/src/db/api/storage/storage_ops/time_semantics.rs b/raphtory/src/db/api/storage/graph/storage_ops/time_semantics.rs similarity index 99% rename from raphtory/src/db/api/storage/storage_ops/time_semantics.rs rename to raphtory/src/db/api/storage/graph/storage_ops/time_semantics.rs index e9252959cf..00fc9d0032 100644 --- a/raphtory/src/db/api/storage/storage_ops/time_semantics.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/time_semantics.rs @@ -13,7 +13,7 @@ use crate::{ storage::timeindex::{TimeIndexIntoOps, TimeIndexOps}, }, db::api::{ - storage::{ + storage::graph::{ edges::{ edge_ref::EdgeStorageRef, edge_storage_ops::{EdgeStorageIntoOps, EdgeStorageOps}, diff --git a/raphtory/src/db/api/storage/tprop_storage_ops.rs b/raphtory/src/db/api/storage/graph/tprop_storage_ops.rs similarity index 97% rename from raphtory/src/db/api/storage/tprop_storage_ops.rs rename to raphtory/src/db/api/storage/graph/tprop_storage_ops.rs index 3e98686ff2..1082bbf0ba 100644 --- a/raphtory/src/db/api/storage/tprop_storage_ops.rs +++ b/raphtory/src/db/api/storage/graph/tprop_storage_ops.rs @@ -1,6 +1,6 @@ use crate::core::{entities::properties::tprop::TProp, storage::timeindex::AsTime, Prop}; #[cfg(feature = "storage")] -use crate::db::api::storage::variants::storage_variants::StorageVariants; +use crate::db::api::storage::graph::variants::storage_variants::StorageVariants; #[cfg(feature = "storage")] use pometry_storage::tprops::DiskTProp; use raphtory_api::core::storage::timeindex::TimeIndexEntry; diff --git a/raphtory/src/db/api/storage/variants/direction_variants.rs b/raphtory/src/db/api/storage/graph/variants/direction_variants.rs similarity index 100% rename from raphtory/src/db/api/storage/variants/direction_variants.rs rename to raphtory/src/db/api/storage/graph/variants/direction_variants.rs diff --git a/raphtory/src/db/api/storage/variants/filter_variants.rs b/raphtory/src/db/api/storage/graph/variants/filter_variants.rs similarity index 100% rename from raphtory/src/db/api/storage/variants/filter_variants.rs rename to raphtory/src/db/api/storage/graph/variants/filter_variants.rs diff --git a/raphtory/src/db/api/storage/variants/layer_variants.rs b/raphtory/src/db/api/storage/graph/variants/layer_variants.rs similarity index 100% rename from raphtory/src/db/api/storage/variants/layer_variants.rs rename to raphtory/src/db/api/storage/graph/variants/layer_variants.rs diff --git a/raphtory/src/db/api/storage/variants/mod.rs b/raphtory/src/db/api/storage/graph/variants/mod.rs similarity index 100% rename from raphtory/src/db/api/storage/variants/mod.rs rename to raphtory/src/db/api/storage/graph/variants/mod.rs diff --git a/raphtory/src/db/api/storage/variants/storage_variants.rs b/raphtory/src/db/api/storage/graph/variants/storage_variants.rs similarity index 98% rename from raphtory/src/db/api/storage/variants/storage_variants.rs rename to raphtory/src/db/api/storage/graph/variants/storage_variants.rs index dfb41a855b..c63f6713a5 100644 --- a/raphtory/src/db/api/storage/variants/storage_variants.rs +++ b/raphtory/src/db/api/storage/graph/variants/storage_variants.rs @@ -1,4 +1,4 @@ -use crate::{core::Prop, db::api::storage::tprop_storage_ops::TPropOps}; +use crate::{core::Prop, db::api::storage::graph::tprop_storage_ops::TPropOps}; use raphtory_api::core::storage::timeindex::TimeIndexEntry; use rayon::iter::{ plumbing::{Consumer, ProducerCallback, UnindexedConsumer}, diff --git a/raphtory/src/db/api/storage/variants/storage_variants3.rs b/raphtory/src/db/api/storage/graph/variants/storage_variants3.rs similarity index 99% rename from raphtory/src/db/api/storage/variants/storage_variants3.rs rename to raphtory/src/db/api/storage/graph/variants/storage_variants3.rs index b2bfcb92b1..5e706a2f0f 100644 --- a/raphtory/src/db/api/storage/variants/storage_variants3.rs +++ b/raphtory/src/db/api/storage/graph/variants/storage_variants3.rs @@ -1,4 +1,4 @@ -use crate::{core::Prop, db::api::storage::tprop_storage_ops::TPropOps}; +use crate::{core::Prop, db::api::storage::graph::tprop_storage_ops::TPropOps}; use raphtory_api::core::storage::timeindex::TimeIndexEntry; use rayon::iter::{ plumbing::{Consumer, ProducerCallback, UnindexedConsumer}, diff --git a/raphtory/src/db/api/storage/mod.rs b/raphtory/src/db/api/storage/mod.rs index 7f8f256ae9..f05d171044 100644 --- a/raphtory/src/db/api/storage/mod.rs +++ b/raphtory/src/db/api/storage/mod.rs @@ -1,6 +1,2 @@ -pub mod edges; -pub mod locked; -pub mod nodes; -pub mod storage_ops; -pub mod tprop_storage_ops; -pub mod variants; +pub mod graph; +pub mod storage; diff --git a/raphtory/src/db/api/storage/storage.rs b/raphtory/src/db/api/storage/storage.rs new file mode 100644 index 0000000000..8c289f5827 --- /dev/null +++ b/raphtory/src/db/api/storage/storage.rs @@ -0,0 +1,345 @@ +use crate::{ + core::{ + entities::{ + graph::tgraph::TemporalGraph, + nodes::node_ref::{AsNodeRef, NodeRef}, + }, + utils::errors::GraphError, + Prop, PropType, + }, + db::api::{ + mutation::internal::{ + InternalAdditionOps, InternalDeletionOps, InternalPropertyAdditionOps, + }, + storage::graph::{nodes::node_storage_ops::NodeStorageOps, storage_ops::GraphStorage}, + view::{Base, InheritViewOps}, + }, +}; +use once_cell::sync::OnceCell; +use raphtory_api::core::{ + entities::{EID, VID}, + storage::{dict_mapper::MaybeNew, timeindex::TimeIndexEntry}, +}; +use serde::{Deserialize, Serialize}; +use std::{ + fmt::{Display, Formatter}, + sync::Arc, +}; + +#[cfg(feature = "proto")] +use crate::serialise::incremental::GraphWriter; + +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct Storage { + graph: GraphStorage, + #[cfg(feature = "proto")] + #[serde(skip)] + pub(crate) cache: OnceCell, +} + +impl Display for Storage { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + Display::fmt(&self.graph, f) + } +} + +impl Base for Storage { + type Base = GraphStorage; + + #[inline] + fn base(&self) -> &Self::Base { + &self.graph + } +} + +impl Storage { + pub(crate) fn new(num_locks: usize) -> Self { + Self { + graph: GraphStorage::Unlocked(Arc::new(TemporalGraph::new(num_locks))), + #[cfg(feature = "proto")] + cache: OnceCell::new(), + } + } + + pub(crate) fn from_inner(graph: GraphStorage) -> Self { + Self { + graph, + #[cfg(feature = "proto")] + cache: OnceCell::new(), + } + } + + #[cfg(feature = "proto")] + #[inline] + fn if_cache(&self, map_fn: impl FnOnce(&GraphWriter)) { + if let Some(cache) = self.cache.get() { + map_fn(cache) + } + } +} +impl InheritViewOps for Storage {} + +impl InternalAdditionOps for Storage { + #[inline] + fn next_event_id(&self) -> Result { + self.graph.next_event_id() + } + + fn resolve_layer(&self, layer: Option<&str>) -> Result, GraphError> { + let id = self.graph.resolve_layer(layer)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.resolve_layer(layer, id)); + + Ok(id) + } + + fn resolve_node(&self, id: V) -> Result, GraphError> { + match id.as_node_ref() { + NodeRef::Internal(id) => Ok(MaybeNew::Existing(id)), + NodeRef::External(gid) => { + let id = self.graph.resolve_node(gid)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.resolve_node(id, gid)); + + Ok(id) + } + } + } + + fn resolve_node_and_type( + &self, + id: V, + node_type: &str, + ) -> Result, MaybeNew)>, GraphError> { + let node_and_type = self.graph.resolve_node_and_type(id, node_type)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| { + let (vid, _) = node_and_type.inner(); + let node_entry = self.graph.node_entry(vid.inner()); + cache.resolve_node_and_type(node_and_type, node_type, node_entry.id()) + }); + + Ok(node_and_type) + } + + fn resolve_graph_property( + &self, + prop: &str, + dtype: PropType, + is_static: bool, + ) -> Result, GraphError> { + let id = self.graph.resolve_graph_property(prop, dtype, is_static)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.resolve_graph_property(prop, id, dtype, is_static)); + + Ok(id) + } + + fn resolve_node_property( + &self, + prop: &str, + dtype: PropType, + is_static: bool, + ) -> Result, GraphError> { + let id = self.graph.resolve_node_property(prop, dtype, is_static)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.resolve_node_property(prop, id, dtype, is_static)); + + Ok(id) + } + + fn resolve_edge_property( + &self, + prop: &str, + dtype: PropType, + is_static: bool, + ) -> Result, GraphError> { + let id = self.graph.resolve_edge_property(prop, dtype, is_static)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.resolve_edge_property(prop, id, dtype, is_static)); + + Ok(id) + } + + fn internal_add_node( + &self, + t: TimeIndexEntry, + v: VID, + props: &[(usize, Prop)], + ) -> Result<(), GraphError> { + self.graph.internal_add_node(t, v, props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_node_update(t, v, props)); + + Ok(()) + } + + fn internal_add_edge( + &self, + t: TimeIndexEntry, + src: VID, + dst: VID, + props: &[(usize, Prop)], + layer: usize, + ) -> Result, GraphError> { + let id = self.graph.internal_add_edge(t, src, dst, props, layer)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| { + cache.resolve_edge(id, src, dst); + cache.add_edge_update(t, id.inner(), props, layer); + }); + + Ok(id) + } + + fn internal_add_edge_update( + &self, + t: TimeIndexEntry, + edge: EID, + props: &[(usize, Prop)], + layer: usize, + ) -> Result<(), GraphError> { + self.graph.internal_add_edge_update(t, edge, props, layer)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_edge_update(t, edge, props, layer)); + + Ok(()) + } +} + +impl InternalPropertyAdditionOps for Storage { + fn internal_add_properties( + &self, + t: TimeIndexEntry, + props: &[(usize, Prop)], + ) -> Result<(), GraphError> { + self.graph.internal_add_properties(t, props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_graph_tprops(t, props)); + + Ok(()) + } + + fn internal_add_constant_properties(&self, props: &[(usize, Prop)]) -> Result<(), GraphError> { + self.graph.internal_add_constant_properties(props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_graph_cprops(props)); + + Ok(()) + } + + fn internal_update_constant_properties( + &self, + props: &[(usize, Prop)], + ) -> Result<(), GraphError> { + self.graph.internal_update_constant_properties(props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_graph_cprops(props)); + + Ok(()) + } + + fn internal_add_constant_node_properties( + &self, + vid: VID, + props: &[(usize, Prop)], + ) -> Result<(), GraphError> { + self.graph + .internal_add_constant_node_properties(vid, props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_node_cprops(vid, props)); + + Ok(()) + } + + fn internal_update_constant_node_properties( + &self, + vid: VID, + props: &[(usize, Prop)], + ) -> Result<(), GraphError> { + self.graph + .internal_update_constant_node_properties(vid, props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_node_cprops(vid, props)); + + Ok(()) + } + + fn internal_add_constant_edge_properties( + &self, + eid: EID, + layer: usize, + props: &[(usize, Prop)], + ) -> Result<(), GraphError> { + self.graph + .internal_add_constant_edge_properties(eid, layer, props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_edge_cprops(eid, layer, props)); + + Ok(()) + } + + fn internal_update_constant_edge_properties( + &self, + eid: EID, + layer: usize, + props: &[(usize, Prop)], + ) -> Result<(), GraphError> { + self.graph + .internal_update_constant_edge_properties(eid, layer, props)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.add_edge_cprops(eid, layer, props)); + + Ok(()) + } +} + +impl InternalDeletionOps for Storage { + fn internal_delete_edge( + &self, + t: TimeIndexEntry, + src: VID, + dst: VID, + layer: usize, + ) -> Result, GraphError> { + let eid = self.graph.internal_delete_edge(t, src, dst, layer)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| { + cache.resolve_edge(eid, src, dst); + cache.delete_edge(eid.inner(), t, layer); + }); + + Ok(eid) + } + + fn internal_delete_existing_edge( + &self, + t: TimeIndexEntry, + eid: EID, + layer: usize, + ) -> Result<(), GraphError> { + self.graph.internal_delete_existing_edge(t, eid, layer)?; + + #[cfg(feature = "proto")] + self.if_cache(|cache| cache.delete_edge(eid, t, layer)); + + Ok(()) + } +} diff --git a/raphtory/src/db/api/view/graph.rs b/raphtory/src/db/api/view/graph.rs index 577c143869..a77c3373d7 100644 --- a/raphtory/src/db/api/view/graph.rs +++ b/raphtory/src/db/api/view/graph.rs @@ -8,7 +8,7 @@ use crate::{ api::{ mutation::{internal::InternalAdditionOps, AdditionOps, PropertyAdditionOps}, properties::Properties, - storage::{ + storage::graph::{ edges::edge_storage_ops::EdgeStorageOps, nodes::node_storage_ops::NodeStorageOps, storage_ops::GraphStorage, }, diff --git a/raphtory/src/db/api/view/internal/core_ops.rs b/raphtory/src/db/api/view/internal/core_ops.rs index 0cc13c88fc..c8f7496476 100644 --- a/raphtory/src/db/api/view/internal/core_ops.rs +++ b/raphtory/src/db/api/view/internal/core_ops.rs @@ -13,7 +13,7 @@ use crate::{ Prop, }, db::api::{ - storage::{ + storage::graph::{ edges::{ edge_entry::EdgeStorageEntry, edge_owned_entry::EdgeOwnedEntry, edges::EdgesStorage, }, @@ -27,11 +27,11 @@ use crate::{ }, }; use enum_dispatch::enum_dispatch; +use raphtory_api::core::{entities::GID, storage::arc_str::ArcStr}; use std::{iter, ops::Range}; #[cfg(feature = "storage")] use pometry_storage::timestamps::TimeStamps; -use raphtory_api::core::{entities::GID, storage::arc_str::ArcStr}; #[cfg(feature = "storage")] use rayon::prelude::*; diff --git a/raphtory/src/db/api/view/internal/edge_filter_ops.rs b/raphtory/src/db/api/view/internal/edge_filter_ops.rs index d55750f325..ed7dd36811 100644 --- a/raphtory/src/db/api/view/internal/edge_filter_ops.rs +++ b/raphtory/src/db/api/view/internal/edge_filter_ops.rs @@ -1,6 +1,6 @@ use crate::{ core::entities::LayerIds, - db::api::{storage::edges::edge_ref::EdgeStorageRef, view::internal::Base}, + db::api::{storage::graph::edges::edge_ref::EdgeStorageRef, view::internal::Base}, }; use enum_dispatch::enum_dispatch; diff --git a/raphtory/src/db/api/view/internal/materialize.rs b/raphtory/src/db/api/view/internal/materialize.rs index 0a383f3003..52ddfca383 100644 --- a/raphtory/src/db/api/view/internal/materialize.rs +++ b/raphtory/src/db/api/view/internal/materialize.rs @@ -18,7 +18,7 @@ use crate::{ properties::internal::{ ConstPropertiesOps, TemporalPropertiesOps, TemporalPropertyViewOps, }, - storage::{ + storage::graph::{ edges::{ edge_entry::EdgeStorageEntry, edge_owned_entry::EdgeOwnedEntry, edge_ref::EdgeStorageRef, edges::EdgesStorage, @@ -34,13 +34,11 @@ use crate::{ graph::{graph::Graph, views::deletion_graph::PersistentGraph}, }, prelude::*, - BINCODE_VERSION, }; use chrono::{DateTime, Utc}; use enum_dispatch::enum_dispatch; -use raphtory_api::core::storage::arc_str::ArcStr; -use serde::{de::Error, Deserialize, Deserializer, Serialize}; -use std::{fs, io, path::Path}; +use raphtory_api::core::storage::{arc_str::ArcStr, dict_mapper::MaybeNew}; +use serde::{Deserialize, Serialize}; #[enum_dispatch(CoreGraphOps)] #[enum_dispatch(InternalLayerOps)] @@ -65,27 +63,6 @@ pub enum GraphType { PersistentGraph, } -fn version_deserialize<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - let version = u32::deserialize(deserializer)?; - if version != BINCODE_VERSION { - return Err(Error::custom(GraphError::BincodeVersionError( - version, - BINCODE_VERSION, - ))); - }; - Ok(version) -} - -#[derive(Serialize, Deserialize)] -struct VersionedGraph { - #[serde(deserialize_with = "version_deserialize")] - version: u32, - graph: T, -} - impl Static for MaterializedGraph {} impl MaterializedGraph { @@ -108,60 +85,6 @@ impl MaterializedGraph { MaterializedGraph::PersistentGraph(g) => Some(g), } } - - pub fn load_from_file>(path: P, force: bool) -> Result { - let f = std::fs::File::open(path)?; - let mut reader = std::io::BufReader::new(f); - if force { - let _: String = bincode::deserialize_from(&mut reader)?; - let data: Self = bincode::deserialize_from(&mut reader)?; - Ok(data) - } else { - let version: u32 = bincode::deserialize_from(&mut reader)?; - if version != BINCODE_VERSION { - return Err(GraphError::BincodeVersionError(version, BINCODE_VERSION)); - } - let data: Self = bincode::deserialize_from(&mut reader)?; - Ok(data) - } - } - - pub fn save_to_file>(&self, path: P) -> Result<(), GraphError> { - let f = fs::File::create(path)?; - let mut writer = io::BufWriter::new(f); - let versioned_data = VersionedGraph { - version: BINCODE_VERSION, - graph: self.clone(), - }; - Ok(bincode::serialize_into(&mut writer, &versioned_data)?) - } - - pub fn save_to_path(&self, path: &Path) -> Result<(), GraphError> { - match self { - MaterializedGraph::EventGraph(g) => g.save_to_file(&path)?, - MaterializedGraph::PersistentGraph(g) => g.save_to_file(&path)?, - }; - - Ok(()) - } - - pub fn bincode(&self) -> Result, GraphError> { - let versioned_data = VersionedGraph { - version: BINCODE_VERSION, - graph: self.clone(), - }; - let encoded = bincode::serialize(&versioned_data)?; - Ok(encoded) - } - - pub fn from_bincode(b: &[u8]) -> Result { - let version: u32 = bincode::deserialize(b)?; - if version != BINCODE_VERSION { - return Err(GraphError::BincodeVersionError(version, BINCODE_VERSION)); - } - let g: VersionedGraph = bincode::deserialize(b)?; - Ok(g.graph) - } } impl InternalDeletionOps for MaterializedGraph { @@ -171,7 +94,7 @@ impl InternalDeletionOps for MaterializedGraph { src: VID, dst: VID, layer: usize, - ) -> Result<(), GraphError> { + ) -> Result, GraphError> { match self { MaterializedGraph::EventGraph(_) => Err(EventGraphDeletionsNotSupported), MaterializedGraph::PersistentGraph(g) => g.internal_delete_edge(t, src, dst, layer), diff --git a/raphtory/src/db/api/view/internal/node_filter_ops.rs b/raphtory/src/db/api/view/internal/node_filter_ops.rs index a6d27346e4..505418ac05 100644 --- a/raphtory/src/db/api/view/internal/node_filter_ops.rs +++ b/raphtory/src/db/api/view/internal/node_filter_ops.rs @@ -1,7 +1,7 @@ use crate::{ core::entities::LayerIds, db::api::{ - storage::nodes::node_ref::NodeStorageRef, + storage::graph::nodes::node_ref::NodeStorageRef, view::{Base, MaterializedGraph}, }, }; diff --git a/raphtory/src/db/api/view/internal/time_semantics.rs b/raphtory/src/db/api/view/internal/time_semantics.rs index 27a5ab9509..be398c5b20 100644 --- a/raphtory/src/db/api/view/internal/time_semantics.rs +++ b/raphtory/src/db/api/view/internal/time_semantics.rs @@ -4,7 +4,7 @@ use crate::{ Prop, }, db::api::{ - storage::{edges::edge_ref::EdgeStorageRef, nodes::node_ref::NodeStorageRef}, + storage::graph::{edges::edge_ref::EdgeStorageRef, nodes::node_ref::NodeStorageRef}, view::{internal::Base, BoxedIter, MaterializedGraph}, }, }; diff --git a/raphtory/src/db/api/view/mod.rs b/raphtory/src/db/api/view/mod.rs index e560d9c99d..8bb2bf5cd2 100644 --- a/raphtory/src/db/api/view/mod.rs +++ b/raphtory/src/db/api/view/mod.rs @@ -6,8 +6,6 @@ pub mod internal; mod layer; pub(crate) mod node; mod reset_filter; -#[cfg(feature = "proto")] -pub mod serialise; pub(crate) mod time; pub(crate) use edge::BaseEdgeViewOps; diff --git a/raphtory/src/db/api/view/node.rs b/raphtory/src/db/api/view/node.rs index 79fa63289b..11e9ba0cf2 100644 --- a/raphtory/src/db/api/view/node.rs +++ b/raphtory/src/db/api/view/node.rs @@ -6,7 +6,7 @@ use crate::{ }, db::api::{ properties::{internal::PropertiesOps, Properties}, - storage::{nodes::node_storage_ops::NodeStorageOps, storage_ops::GraphStorage}, + storage::graph::{nodes::node_storage_ops::NodeStorageOps, storage_ops::GraphStorage}, view::{ internal::{CoreGraphOps, OneHopFilter, TimeSemantics}, reset_filter::ResetFilter, diff --git a/raphtory/src/db/graph/edge.rs b/raphtory/src/db/graph/edge.rs index b39995d07f..4d2f67beb3 100644 --- a/raphtory/src/db/graph/edge.rs +++ b/raphtory/src/db/graph/edge.rs @@ -23,7 +23,7 @@ use crate::{ internal::{ConstPropertiesOps, TemporalPropertiesOps, TemporalPropertyViewOps}, Properties, }, - storage::edges::edge_storage_ops::EdgeStorageOps, + storage::graph::edges::edge_storage_ops::EdgeStorageOps, view::{ internal::{OneHopFilter, Static}, BaseEdgeViewOps, IntoDynBoxed, StaticGraphViewOps, @@ -183,7 +183,7 @@ impl }), None => { if create { - self.graph.resolve_layer(layer) + Ok(self.graph.resolve_layer(layer)?.inner()) } else { self.graph .get_layer_id(name) @@ -228,13 +228,13 @@ impl }); } let properties: Vec<(usize, Prop)> = props.collect_properties(|name, dtype| { - self.graph.resolve_edge_property(name, dtype, true) + Ok(self.graph.resolve_edge_property(name, dtype, true)?.inner()) })?; self.graph.internal_add_constant_edge_properties( self.edge.pid(), input_layer_id, - properties, + &properties, ) } @@ -245,13 +245,13 @@ impl ) -> Result<(), GraphError> { let input_layer_id = self.resolve_layer(layer, false)?; let properties: Vec<(usize, Prop)> = props.collect_properties(|name, dtype| { - self.graph.resolve_edge_property(name, dtype, true) + Ok(self.graph.resolve_edge_property(name, dtype, true)?.inner()) })?; self.graph.internal_update_constant_edge_properties( self.edge.pid(), input_layer_id, - properties, + &properties, ) } @@ -264,11 +264,14 @@ impl let t = time_from_input(&self.graph, time)?; let layer_id = self.resolve_layer(layer, true)?; let properties: Vec<(usize, Prop)> = props.collect_properties(|name, dtype| { - self.graph.resolve_edge_property(name, dtype, false) + Ok(self + .graph + .resolve_edge_property(name, dtype, false)? + .inner()) })?; self.graph - .internal_add_edge_update(t, self.edge.pid(), properties, layer_id)?; + .internal_add_edge_update(t, self.edge.pid(), &properties, layer_id)?; Ok(()) } } diff --git a/raphtory/src/db/graph/graph.rs b/raphtory/src/db/graph/graph.rs index 06efa25a18..20c7c699db 100644 --- a/raphtory/src/db/graph/graph.rs +++ b/raphtory/src/db/graph/graph.rs @@ -18,11 +18,10 @@ use super::views::deletion_graph::PersistentGraph; use crate::{ - core::{entities::graph::tgraph::TemporalGraph, utils::errors::GraphError}, db::api::{ mutation::internal::InheritMutationOps, - storage::storage_ops::GraphStorage, - view::internal::{Base, InheritViewOps, MaterializedGraph, Static}, + storage::{graph::storage_ops::GraphStorage, storage::Storage}, + view::internal::{Base, InheritViewOps, Static}, }, prelude::*, }; @@ -31,14 +30,13 @@ use rayon::prelude::*; use serde::{Deserialize, Serialize}; use std::{ fmt::{Display, Formatter}, - path::Path, sync::Arc, }; #[repr(transparent)] #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct Graph { - inner: GraphStorage, + pub(crate) inner: Arc, } impl Static for Graph {} @@ -150,7 +148,7 @@ where } impl Base for Graph { - type Base = GraphStorage; + type Base = Storage; #[inline(always)] fn base(&self) -> &Self::Base { @@ -177,7 +175,7 @@ impl Graph { /// ``` pub fn new() -> Self { Self { - inner: GraphStorage::Unlocked(Arc::new(TemporalGraph::default())), + inner: Arc::new(Storage::default()), } } @@ -188,45 +186,22 @@ impl Graph { /// A raphtory graph pub fn new_with_shards(num_shards: usize) -> Self { Self { - inner: GraphStorage::Unlocked(Arc::new(TemporalGraph::new(num_shards))), + inner: Arc::new(Storage::new(num_shards)), } } - pub(crate) fn from_internal_graph(internal_graph: GraphStorage) -> Self { - Self { - inner: internal_graph, - } - } - - /// Load a graph from a directory - /// - /// # Arguments - /// - /// * `path` - The path to the directory - /// - /// Returns: - /// - /// A raphtory graph - /// - /// # Example - /// - /// ```no_run - /// use raphtory::prelude::Graph; - /// let g = Graph::load_from_file("path/to/graph", false); - /// ``` - pub fn load_from_file>(path: P, force: bool) -> Result { - let g = MaterializedGraph::load_from_file(path, force)?; - g.into_events().ok_or(GraphError::GraphLoadError) + pub(crate) fn from_storage(inner: Arc) -> Self { + Self { inner } } - /// Save a graph to a directory - pub fn save_to_file>(&self, path: P) -> Result<(), GraphError> { - MaterializedGraph::from(self.clone()).save_to_file(path) + pub(crate) fn from_internal_graph(graph_storage: GraphStorage) -> Self { + let inner = Arc::new(Storage::from_inner(graph_storage)); + Self { inner } } /// Get persistent graph pub fn persistent_graph(&self) -> PersistentGraph { - PersistentGraph::from_internal_graph(self.inner.clone()) + PersistentGraph::from_storage(self.inner.clone()) } } @@ -236,7 +211,10 @@ mod db_tests { use crate::{ algorithms::components::weakly_connected_components, core::{ - utils::time::{error::ParseTimeError, TryIntoTime}, + utils::{ + errors::GraphError, + time::{error::ParseTimeError, TryIntoTime}, + }, Prop, }, db::{ @@ -551,6 +529,7 @@ mod db_tests { } #[test] + #[cfg(feature = "proto")] fn graph_save_to_load_from_file() { let vs = vec![ (1, 1, 2), @@ -570,10 +549,10 @@ mod db_tests { let tmp_raphtory_path: TempDir = TempDir::new().expect("Failed to create tempdir"); let graph_path = format!("{}/graph.bin", tmp_raphtory_path.path().display()); - g.save_to_file(&graph_path).expect("Failed to save graph"); + g.encode(&graph_path).expect("Failed to save graph"); // Load from files - let g2 = Graph::load_from_file(&graph_path, false).expect("Failed to load graph"); + let g2 = Graph::decode(&graph_path).expect("Failed to load graph"); assert_eq!(g, g2); @@ -2326,12 +2305,15 @@ mod db_tests { } #[test] + #[cfg(feature = "proto")] fn save_load_serial() { let g = Graph::new(); g.add_edge(0, 0, 1, NO_PROPS, None).unwrap(); let dir = tempfile::tempdir().unwrap(); let file_path = dir.path().join("abcd11"); - g.save_to_file(file_path).unwrap(); + g.encode(&file_path).unwrap(); + let gg = Graph::decode(file_path).unwrap(); + assert_graph_equal(&g, &gg); } #[test] diff --git a/raphtory/src/db/graph/node.rs b/raphtory/src/db/graph/node.rs index 423a621092..dc7d9f708c 100644 --- a/raphtory/src/db/graph/node.rs +++ b/raphtory/src/db/graph/node.rs @@ -27,7 +27,7 @@ use crate::{ use crate::{ core::{entities::nodes::node_ref::AsNodeRef, storage::timeindex::AsTime}, - db::{api::storage::storage_ops::GraphStorage, graph::edges::Edges}, + db::{api::storage::graph::storage_ops::GraphStorage, graph::edges::Edges}, }; use chrono::{DateTime, Utc}; use raphtory_api::core::storage::arc_str::ArcStr; @@ -329,14 +329,15 @@ impl props: C, ) -> Result<(), GraphError> { let properties: Vec<(usize, Prop)> = props.collect_properties(|name, dtype| { - self.graph.resolve_node_property(name, dtype, true) + Ok(self.graph.resolve_node_property(name, dtype, true)?.inner()) })?; self.graph - .internal_add_constant_node_properties(self.node, properties) + .internal_add_constant_node_properties(self.node, &properties) } pub fn set_node_type(&self, new_type: &str) -> Result<(), GraphError> { - self.graph.set_node_type(self.node, new_type) + self.graph.resolve_node_and_type(self.node, new_type)?; + Ok(()) } pub fn update_constant_properties( @@ -344,10 +345,10 @@ impl props: C, ) -> Result<(), GraphError> { let properties: Vec<(usize, Prop)> = props.collect_properties(|name, dtype| { - self.graph.resolve_node_property(name, dtype, true) + Ok(self.graph.resolve_node_property(name, dtype, true)?.inner()) })?; self.graph - .internal_update_constant_node_properties(self.node, properties) + .internal_update_constant_node_properties(self.node, &properties) } pub fn add_updates( @@ -357,9 +358,12 @@ impl ) -> Result<(), GraphError> { let t = time_from_input(&self.graph, time)?; let properties: Vec<(usize, Prop)> = props.collect_properties(|name, dtype| { - self.graph.resolve_node_property(name, dtype, false) + Ok(self + .graph + .resolve_node_property(name, dtype, false)? + .inner()) })?; - self.graph.internal_add_node(t, self.node, properties) + self.graph.internal_add_node(t, self.node, &properties) } } diff --git a/raphtory/src/db/graph/nodes.rs b/raphtory/src/db/graph/nodes.rs index 749738113f..cda9a41c2d 100644 --- a/raphtory/src/db/graph/nodes.rs +++ b/raphtory/src/db/graph/nodes.rs @@ -4,7 +4,7 @@ use crate::{ api::{ properties::Properties, state::LazyNodeState, - storage::storage_ops::GraphStorage, + storage::graph::storage_ops::GraphStorage, view::{ internal::{OneHopFilter, Static}, BaseNodeViewOps, BoxedLIter, DynamicGraph, IntoDynBoxed, IntoDynamic, diff --git a/raphtory/src/db/graph/path.rs b/raphtory/src/db/graph/path.rs index ce74e80330..0393f3f0be 100644 --- a/raphtory/src/db/graph/path.rs +++ b/raphtory/src/db/graph/path.rs @@ -3,7 +3,7 @@ use crate::{ db::{ api::{ properties::Properties, - storage::storage_ops::GraphStorage, + storage::graph::storage_ops::GraphStorage, view::{ internal::OneHopFilter, BaseNodeViewOps, BoxedLIter, DynamicGraph, IntoDynBoxed, }, diff --git a/raphtory/src/db/graph/views/deletion_graph.rs b/raphtory/src/db/graph/views/deletion_graph.rs index 6458465bfd..a64ba4e474 100644 --- a/raphtory/src/db/graph/views/deletion_graph.rs +++ b/raphtory/src/db/graph/views/deletion_graph.rs @@ -2,7 +2,6 @@ use crate::{ core::{ entities::{edges::edge_ref::EdgeRef, LayerIds, VID}, storage::timeindex::{AsTime, TimeIndexEntry, TimeIndexIntoOps, TimeIndexOps}, - utils::errors::GraphError, Prop, }, db::{ @@ -10,10 +9,13 @@ use crate::{ mutation::internal::InheritMutationOps, properties::internal::InheritPropertiesOps, storage::{ - edges::{edge_ref::EdgeStorageRef, edge_storage_ops::EdgeStorageOps}, - nodes::{node_ref::NodeStorageRef, node_storage_ops::NodeStorageOps}, - storage_ops::GraphStorage, - tprop_storage_ops::TPropOps, + graph::{ + edges::{edge_ref::EdgeStorageRef, edge_storage_ops::EdgeStorageOps}, + nodes::{node_ref::NodeStorageRef, node_storage_ops::NodeStorageOps}, + storage_ops::GraphStorage, + tprop_storage_ops::TPropOps, + }, + storage::Storage, }, view::{internal::*, BoxedIter, IntoDynBoxed}, }, @@ -28,8 +30,8 @@ use std::{ cmp::min, fmt::{Display, Formatter}, iter, - ops::Range, - path::Path, + ops::{Deref, Range}, + sync::Arc, }; /// A graph view where an edge remains active from the time it is added until it is explicitly marked as deleted. @@ -37,14 +39,14 @@ use std::{ /// Note that the graph will give you access to all edges that were added at any point in time, even those that are marked as deleted. /// The deletion only has an effect on the exploded edge view that are returned. An edge is included in a windowed view of the graph if /// it is considered active at any point in the window. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct PersistentGraph(pub GraphStorage); +#[derive(Clone, Debug, Serialize, Deserialize, Default)] +pub struct PersistentGraph(pub(crate) Arc); impl Static for PersistentGraph {} impl From for PersistentGraph { fn from(value: GraphStorage) -> Self { - Self(value) + Self(Arc::new(Storage::from_inner(value))) } } @@ -107,68 +109,22 @@ fn edge_alive_at_start(e: EdgeStorageRef, t: i64, layer_ids: &LayerIds) -> bool .any(|(_, additions, deletions)| alive_at(&additions, &deletions, t)) } -impl Default for PersistentGraph { - fn default() -> Self { - Self::new() - } -} - impl PersistentGraph { pub fn new() -> Self { - Self(GraphStorage::default()) + Self::default() + } + + pub fn from_storage(storage: Arc) -> Self { + Self(storage) } pub fn from_internal_graph(internal_graph: GraphStorage) -> Self { - Self(internal_graph) - } - - /// Save a graph to a directory - /// - /// # Arguments - /// - /// * `path` - The path to the directory - /// - /// Returns: - /// - /// A raphtory graph - /// - /// # Example - /// - /// ```no_run - /// use std::fs::File; - /// use raphtory::prelude::*; - /// let g = Graph::new(); - /// g.add_node(1, 1, NO_PROPS, None).unwrap(); - /// g.save_to_file("path_str").expect("failed to save file"); - /// ``` - pub fn save_to_file>(&self, path: P) -> Result<(), GraphError> { - MaterializedGraph::from(self.clone()).save_to_file(path) - } - - /// Load a graph from a directory - /// - /// # Arguments - /// - /// * `path` - The path to the directory - /// - /// Returns: - /// - /// A raphtory graph - /// - /// # Example - /// - /// ```no_run - /// use raphtory::prelude::*; - /// let g = Graph::load_from_file("path/to/graph", false); - /// ``` - pub fn load_from_file>(path: P, force: bool) -> Result { - let g = MaterializedGraph::load_from_file(path, force)?; - g.into_persistent().ok_or(GraphError::GraphLoadError) + Self(Arc::new(Storage::from_inner(internal_graph))) } /// Get event graph pub fn event_graph(&self) -> Graph { - Graph::from_internal_graph(self.0.clone()) + Graph::from_storage(self.0.clone()) } } @@ -179,7 +135,7 @@ impl<'graph, G: GraphViewOps<'graph>> PartialEq for PersistentGraph { } impl Base for PersistentGraph { - type Base = GraphStorage; + type Base = Storage; #[inline(always)] fn base(&self) -> &Self::Base { &self.0 diff --git a/raphtory/src/db/graph/views/layer_graph.rs b/raphtory/src/db/graph/views/layer_graph.rs index dc08caefce..572dd95ed3 100644 --- a/raphtory/src/db/graph/views/layer_graph.rs +++ b/raphtory/src/db/graph/views/layer_graph.rs @@ -6,7 +6,7 @@ use crate::{ core::{entities::LayerIds, utils::errors::GraphError}, db::api::{ properties::internal::InheritPropertiesOps, - storage::edges::{edge_ref::EdgeStorageRef, edge_storage_ops::EdgeStorageOps}, + storage::graph::edges::{edge_ref::EdgeStorageRef, edge_storage_ops::EdgeStorageOps}, view::{ internal::{ Base, EdgeFilterOps, Immutable, InheritCoreOps, InheritListOps, InheritMaterialize, diff --git a/raphtory/src/db/graph/views/node_subgraph.rs b/raphtory/src/db/graph/views/node_subgraph.rs index 19c5c07788..e0adb05f70 100644 --- a/raphtory/src/db/graph/views/node_subgraph.rs +++ b/raphtory/src/db/graph/views/node_subgraph.rs @@ -2,7 +2,7 @@ use crate::{ core::entities::{LayerIds, VID}, db::api::{ properties::internal::InheritPropertiesOps, - storage::{ + storage::graph::{ edges::{edge_ref::EdgeStorageRef, edge_storage_ops::EdgeStorageOps}, nodes::{node_ref::NodeStorageRef, node_storage_ops::NodeStorageOps}, }, diff --git a/raphtory/src/db/graph/views/node_type_filtered_subgraph.rs b/raphtory/src/db/graph/views/node_type_filtered_subgraph.rs index 950ea627b4..1e2fe73447 100644 --- a/raphtory/src/db/graph/views/node_type_filtered_subgraph.rs +++ b/raphtory/src/db/graph/views/node_type_filtered_subgraph.rs @@ -2,7 +2,7 @@ use crate::{ core::entities::LayerIds, db::api::{ properties::internal::InheritPropertiesOps, - storage::nodes::{node_ref::NodeStorageRef, node_storage_ops::NodeStorageOps}, + storage::graph::nodes::{node_ref::NodeStorageRef, node_storage_ops::NodeStorageOps}, view::internal::{ Base, Immutable, InheritCoreOps, InheritEdgeFilterOps, InheritLayerOps, InheritListOps, InheritMaterialize, InheritTimeSemantics, NodeFilterOps, Static, diff --git a/raphtory/src/db/graph/views/window_graph.rs b/raphtory/src/db/graph/views/window_graph.rs index 2b863cbf45..2f7f44ea65 100644 --- a/raphtory/src/db/graph/views/window_graph.rs +++ b/raphtory/src/db/graph/views/window_graph.rs @@ -48,7 +48,7 @@ use crate::{ properties::internal::{ InheritStaticPropertiesOps, TemporalPropertiesOps, TemporalPropertyViewOps, }, - storage::{edges::edge_ref::EdgeStorageRef, nodes::node_ref::NodeStorageRef}, + storage::graph::{edges::edge_ref::EdgeStorageRef, nodes::node_ref::NodeStorageRef}, view::{ internal::{ Base, EdgeFilterOps, Immutable, InheritCoreOps, InheritLayerOps, diff --git a/raphtory/src/db/mod.rs b/raphtory/src/db/mod.rs index 22537f5b97..63e711afda 100644 --- a/raphtory/src/db/mod.rs +++ b/raphtory/src/db/mod.rs @@ -1,4 +1,3 @@ pub mod api; pub mod graph; pub mod task; -pub mod utils; diff --git a/raphtory/src/db/task/edge/eval_edge.rs b/raphtory/src/db/task/edge/eval_edge.rs index 29b022aadf..d16a5e9b40 100644 --- a/raphtory/src/db/task/edge/eval_edge.rs +++ b/raphtory/src/db/task/edge/eval_edge.rs @@ -18,7 +18,7 @@ use crate::{ use crate::db::task::edge::eval_edges::EvalEdges; -use crate::db::{api::storage::storage_ops::GraphStorage, task::eval_graph::EvalGraph}; +use crate::db::{api::storage::graph::storage_ops::GraphStorage, task::eval_graph::EvalGraph}; use std::{cell::RefCell, rc::Rc}; pub struct EvalEdgeView<'graph, 'a, G, GH, CS: Clone, S> { diff --git a/raphtory/src/db/task/edge/eval_edges.rs b/raphtory/src/db/task/edge/eval_edges.rs index f2d725303b..abbfe2b868 100644 --- a/raphtory/src/db/task/edge/eval_edges.rs +++ b/raphtory/src/db/task/edge/eval_edges.rs @@ -6,7 +6,7 @@ use crate::{ db::{ api::{ properties::Properties, - storage::storage_ops::GraphStorage, + storage::graph::storage_ops::GraphStorage, view::{internal::OneHopFilter, BaseEdgeViewOps, BoxedLIter}, }, graph::edges::Edges, diff --git a/raphtory/src/db/task/eval_graph.rs b/raphtory/src/db/task/eval_graph.rs index 421d3c9379..18b35de24b 100644 --- a/raphtory/src/db/task/eval_graph.rs +++ b/raphtory/src/db/task/eval_graph.rs @@ -4,7 +4,7 @@ use crate::{ state::compute_state::{ComputeState, ComputeStateVec}, }, db::{ - api::storage::storage_ops::GraphStorage, + api::storage::graph::storage_ops::GraphStorage, task::{ edge::eval_edge::EvalEdgeView, node::{eval_node::EvalNodeView, eval_node_state::EVState}, diff --git a/raphtory/src/db/task/node/eval_node.rs b/raphtory/src/db/task/node/eval_node.rs index c62d5f8da1..5e94799308 100644 --- a/raphtory/src/db/task/node/eval_node.rs +++ b/raphtory/src/db/task/node/eval_node.rs @@ -11,7 +11,7 @@ use crate::{ db::{ api::{ properties::Properties, - storage::storage_ops::GraphStorage, + storage::graph::storage_ops::GraphStorage, view::{internal::OneHopFilter, BaseNodeViewOps, BoxedLIter, IntoDynBoxed}, }, graph::{create_node_type_filter, edges::Edges, node::NodeView, path::PathFromNode}, diff --git a/raphtory/src/db/task/task_runner.rs b/raphtory/src/db/task/task_runner.rs index 38e205475a..ee7db33f0e 100644 --- a/raphtory/src/db/task/task_runner.rs +++ b/raphtory/src/db/task/task_runner.rs @@ -14,7 +14,7 @@ use crate::{ }, }, db::{ - api::{storage::storage_ops::GraphStorage, view::StaticGraphViewOps}, + api::{storage::graph::storage_ops::GraphStorage, view::StaticGraphViewOps}, task::{ eval_graph::EvalGraph, node::{eval_node::EvalNodeView, eval_node_state::EVState}, diff --git a/raphtory/src/db/utils/doc_strings.rs b/raphtory/src/db/utils/doc_strings.rs deleted file mode 100644 index 6d0cbf3210..0000000000 --- a/raphtory/src/db/utils/doc_strings.rs +++ /dev/null @@ -1,49 +0,0 @@ -#[macro_export] -macro_rules! default_layer_doc_string { - () => { - " -Create a view including all the edges in the default layer - -Returns: - a view including all the edges in the default layer" - }; -} - -#[macro_export] -macro_rules! layers_doc_string { - () => { - " -Create a view including all the edges in the layers `names` - -Arguments: - name (str) : the name of the layers to include - -Returns: - a view including all the edges in the layers `names`" - }; -} - -#[macro_export] -macro_rules! layers_name_doc_string { - () => { - " -Create a view including all the edges in the layers `name` - -Arguments: - name (str) : the name of the layer to include - -Returns: - a view including all the edges in the layer `name`" - }; -} - -#[macro_export] -macro_rules! window_size_doc_string { - () => { - " -Returns the size of the window covered by this view - -Returns: - int: the size of the window" - }; -} diff --git a/raphtory/src/db/utils/mod.rs b/raphtory/src/db/utils/mod.rs deleted file mode 100644 index aefeca739a..0000000000 --- a/raphtory/src/db/utils/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod doc_strings; diff --git a/raphtory/src/disk_graph/graph_impl/edge_storage_ops.rs b/raphtory/src/disk_graph/graph_impl/edge_storage_ops.rs index cde9636dc2..88201f3fc1 100644 --- a/raphtory/src/disk_graph/graph_impl/edge_storage_ops.rs +++ b/raphtory/src/disk_graph/graph_impl/edge_storage_ops.rs @@ -4,7 +4,7 @@ use crate::{ storage::timeindex::{TimeIndex, TimeIndexOps}, Prop, }, - db::api::storage::{ + db::api::storage::graph::{ edges::edge_storage_ops::{EdgeStorageOps, TimeIndexRef}, tprop_storage_ops::TPropOps, }, diff --git a/raphtory/src/disk_graph/graph_impl/interop.rs b/raphtory/src/disk_graph/graph_impl/interop.rs index 86be7390a1..19b2761ea5 100644 --- a/raphtory/src/disk_graph/graph_impl/interop.rs +++ b/raphtory/src/disk_graph/graph_impl/interop.rs @@ -5,7 +5,7 @@ use crate::{ Direction, }, db::api::{ - storage::{ + storage::graph::{ edges::edge_storage_ops::EdgeStorageOps, nodes::node_storage_ops::NodeStorageOps, tprop_storage_ops::TPropOps, }, diff --git a/raphtory/src/disk_graph/graph_impl/mod.rs b/raphtory/src/disk_graph/graph_impl/mod.rs index 4db0c7a36c..eb9f80b7ca 100644 --- a/raphtory/src/disk_graph/graph_impl/mod.rs +++ b/raphtory/src/disk_graph/graph_impl/mod.rs @@ -1,9 +1,6 @@ use std::path::Path; -use crate::{ - disk_graph::{DiskGraphError, DiskGraphStorage}, - prelude::Graph, -}; +use crate::{core::utils::errors::GraphError, disk_graph::DiskGraphStorage, prelude::Graph}; mod edge_storage_ops; mod interop; @@ -24,7 +21,7 @@ impl Graph { pub fn persist_as_disk_graph( &self, graph_dir: impl AsRef, - ) -> Result { + ) -> Result { DiskGraphStorage::from_graph(self, graph_dir) } } @@ -44,7 +41,7 @@ mod test { use pometry_storage::{graph::TemporalGraph, properties::Properties}; use crate::{ - db::api::{storage::storage_ops::GraphStorage, view::StaticGraphViewOps}, + db::api::{storage::graph::storage_ops::GraphStorage, view::StaticGraphViewOps}, disk_graph::Time, prelude::*, }; diff --git a/raphtory/src/disk_graph/graph_impl/prop_conversion.rs b/raphtory/src/disk_graph/graph_impl/prop_conversion.rs index 4a19bb9243..b09dc6a8ee 100644 --- a/raphtory/src/disk_graph/graph_impl/prop_conversion.rs +++ b/raphtory/src/disk_graph/graph_impl/prop_conversion.rs @@ -10,7 +10,7 @@ use crate::{ PropType, }, db::api::{ - storage::{nodes::node_storage_ops::NodeStorageOps, tprop_storage_ops::TPropOps}, + storage::graph::{nodes::node_storage_ops::NodeStorageOps, tprop_storage_ops::TPropOps}, view::internal::CoreGraphOps, }, prelude::{Graph, Prop, PropUnwrap}, diff --git a/raphtory/src/disk_graph/graph_impl/tprops.rs b/raphtory/src/disk_graph/graph_impl/tprops.rs index ab9d0dcb4f..e6de5a58e6 100644 --- a/raphtory/src/disk_graph/graph_impl/tprops.rs +++ b/raphtory/src/disk_graph/graph_impl/tprops.rs @@ -4,7 +4,7 @@ use crate::{ types::{NativeType, Offset}, }, core::storage::timeindex::TimeIndexIntoOps, - db::api::{storage::tprop_storage_ops::TPropOps, view::IntoDynBoxed}, + db::api::{storage::graph::tprop_storage_ops::TPropOps, view::IntoDynBoxed}, prelude::Prop, }; use pometry_storage::{ diff --git a/raphtory/src/disk_graph/mod.rs b/raphtory/src/disk_graph/mod.rs index c82698dceb..2bed4f3050 100644 --- a/raphtory/src/disk_graph/mod.rs +++ b/raphtory/src/disk_graph/mod.rs @@ -12,7 +12,7 @@ use crate::{ }, utils::errors::GraphError, }, - db::{api::storage::storage_ops, graph::views::deletion_graph::PersistentGraph}, + db::{api::storage::graph::storage_ops, graph::views::deletion_graph::PersistentGraph}, disk_graph::graph_impl::{prop_conversion::make_node_properties_from_graph, ParquetLayerCols}, prelude::{Graph, Layer}, }; @@ -37,12 +37,6 @@ pub mod prelude { pub use pometry_storage::chunked_array::array_ops::*; } -#[derive(thiserror::Error, Debug)] -pub enum DiskGraphError { - #[error("Raphtory Arrow Error: {0}")] - RAError(#[from] RAError), -} - #[derive(Clone, Debug)] pub struct DiskGraphStorage { pub(crate) inner: Arc, @@ -227,7 +221,7 @@ impl DiskGraphStorage { &self, other: &DiskGraphStorage, new_graph_dir: impl AsRef, - ) -> Result { + ) -> Result { let graph_dir = new_graph_dir.as_ref(); let inner = merge_graphs(graph_dir, &self.inner, &other.inner)?; Ok(DiskGraphStorage::new(inner)) @@ -256,7 +250,7 @@ impl DiskGraphStorage { let resolved_id = edge_meta .resolve_prop_id(prop_name, data_type.into(), false) .expect("Arrow data types should without failing"); - if id != resolved_id { + if resolved_id != id { println!("Warning: Layers with different edge properties are not supported by the high-level apis on top of the disk_graph graph yet, edge properties will not be available to high-level apis"); edge_meta = Meta::new(); break; @@ -294,7 +288,7 @@ impl DiskGraphStorage { } } - pub fn from_graph(graph: &Graph, graph_dir: impl AsRef) -> Result { + pub fn from_graph(graph: &Graph, graph_dir: impl AsRef) -> Result { let inner_graph = TemporalGraph::from_graph(graph, graph_dir.as_ref(), || { make_node_properties_from_graph(graph, graph_dir.as_ref()) })?; diff --git a/raphtory/src/disk_graph/storage_interface/edge.rs b/raphtory/src/disk_graph/storage_interface/edge.rs index 7616864d25..2f99189cf8 100644 --- a/raphtory/src/disk_graph/storage_interface/edge.rs +++ b/raphtory/src/disk_graph/storage_interface/edge.rs @@ -3,7 +3,7 @@ use crate::{ entities::{edges::edge_ref::EdgeRef, LayerIds, EID, ELID}, storage::timeindex::TimeIndexOps, }, - db::api::storage::edges::edge_storage_ops::EdgeStorageIntoOps, + db::api::storage::graph::edges::edge_storage_ops::EdgeStorageIntoOps, }; use pometry_storage::{edge::Edge, edges::Edges, graph::TemporalGraph, timestamps::TimeStamps}; use raphtory_api::core::storage::timeindex::TimeIndexEntry; diff --git a/raphtory/src/disk_graph/storage_interface/edges.rs b/raphtory/src/disk_graph/storage_interface/edges.rs index 443e199bab..771639fc2a 100644 --- a/raphtory/src/disk_graph/storage_interface/edges.rs +++ b/raphtory/src/disk_graph/storage_interface/edges.rs @@ -1,6 +1,6 @@ use crate::{ core::entities::{LayerIds, EID}, - db::api::storage::variants::layer_variants::LayerVariants, + db::api::storage::graph::variants::layer_variants::LayerVariants, disk_graph::storage_interface::{edge::DiskEdge, edges_ref::DiskEdgesRef}, }; use pometry_storage::{graph::TemporalGraph, graph_fragment::TempColGraphFragment}; diff --git a/raphtory/src/disk_graph/storage_interface/edges_ref.rs b/raphtory/src/disk_graph/storage_interface/edges_ref.rs index 631baaaf51..d3fcf4f4ed 100644 --- a/raphtory/src/disk_graph/storage_interface/edges_ref.rs +++ b/raphtory/src/disk_graph/storage_interface/edges_ref.rs @@ -1,6 +1,6 @@ use crate::{ core::entities::{LayerIds, EID}, - db::api::storage::variants::layer_variants::LayerVariants, + db::api::storage::graph::variants::layer_variants::LayerVariants, disk_graph::storage_interface::edge::DiskEdge, }; use pometry_storage::{graph::TemporalGraph, graph_fragment::TempColGraphFragment}; diff --git a/raphtory/src/disk_graph/storage_interface/node.rs b/raphtory/src/disk_graph/storage_interface/node.rs index e739ff8a02..034b9782c6 100644 --- a/raphtory/src/disk_graph/storage_interface/node.rs +++ b/raphtory/src/disk_graph/storage_interface/node.rs @@ -4,7 +4,7 @@ use crate::{ Direction, }, db::api::{ - storage::{ + storage::graph::{ nodes::node_storage_ops::{NodeStorageIntoOps, NodeStorageOps}, tprop_storage_ops::TPropOps, variants::{direction_variants::DirectionVariants, layer_variants::LayerVariants}, diff --git a/raphtory/src/graph_loader/company_house.rs b/raphtory/src/graph_loader/company_house.rs index 6167a5d76c..9231fbae59 100644 --- a/raphtory/src/graph_loader/company_house.rs +++ b/raphtory/src/graph_loader/company_house.rs @@ -28,7 +28,7 @@ pub fn company_house_graph(path: Option) -> Graph { fn restore_from_bincode(encoded_data_dir: &PathBuf) -> Option { if encoded_data_dir.exists() { let now = Instant::now(); - let g = Graph::load_from_file(encoded_data_dir.as_path(), false) + let g = Graph::decode(encoded_data_dir.as_path()) .map_err(|err| { println!( "Restoring from bincode failed with error: {}! Reloading file!", @@ -155,8 +155,7 @@ pub fn company_house_graph(path: Option) -> Graph { now.elapsed().as_secs() ); - g.save_to_file(encoded_data_dir) - .expect("Failed to save graph"); + g.encode(encoded_data_dir).expect("Failed to save graph"); g }); diff --git a/raphtory/src/graph_loader/neo4j_examples.rs b/raphtory/src/graph_loader/neo4j_examples.rs index 112aaef70c..61e08a9de0 100644 --- a/raphtory/src/graph_loader/neo4j_examples.rs +++ b/raphtory/src/graph_loader/neo4j_examples.rs @@ -37,7 +37,7 @@ fn load_movies(row: Row, graph: &rap::Graph) { actor_name, film_title, NO_PROPS, - Some(relation_type.as_str()), + Some(relation_type), ) .unwrap(); } diff --git a/raphtory/src/graph_loader/stable_coins.rs b/raphtory/src/graph_loader/stable_coins.rs index ccb15899bc..1f31ff82c2 100644 --- a/raphtory/src/graph_loader/stable_coins.rs +++ b/raphtory/src/graph_loader/stable_coins.rs @@ -35,10 +35,10 @@ pub fn stable_coin_graph(path: Option, subset: bool) -> Graph { unzip_file(zip_str,dir_str).expect("Failed to unzip stable coin data from https://snap.stanford.edu/data/ERC20-stablecoins.zip"); } - fn restore_from_bincode(encoded_data_dir: &PathBuf) -> Option { - if encoded_data_dir.exists() { + fn restore_from_file(encoded_data_file: &PathBuf) -> Option { + if encoded_data_file.exists() { let now = Instant::now(); - let g = Graph::load_from_file(encoded_data_dir.as_path(), false) + let g = Graph::decode(encoded_data_file.as_path()) .map_err(|err| { println!( "Restoring from bincode failed with error: {}! Reloading file!", @@ -49,7 +49,7 @@ pub fn stable_coin_graph(path: Option, subset: bool) -> Graph { println!( "Loaded graph from encoded data files {} with {} nodes, {} edges which took {} seconds", - encoded_data_dir.to_str().unwrap(), + encoded_data_file.to_str().unwrap(), g.count_nodes(), g.count_edges(), now.elapsed().as_secs() @@ -62,7 +62,7 @@ pub fn stable_coin_graph(path: Option, subset: bool) -> Graph { } let encoded_data_dir = data_dir.join("graphdb.bincode"); - let g = restore_from_bincode(&encoded_data_dir).unwrap_or_else(|| { + let g = restore_from_file(&encoded_data_dir).unwrap_or_else(|| { let g = Graph::new(); let now = Instant::now(); @@ -106,8 +106,7 @@ pub fn stable_coin_graph(path: Option, subset: bool) -> Graph { now.elapsed().as_secs() ); - g.save_to_file(encoded_data_dir) - .expect("Failed to save graph"); + g.encode(encoded_data_dir).expect("Failed to save graph"); g }); diff --git a/raphtory/src/io/arrow/dataframe.rs b/raphtory/src/io/arrow/dataframe.rs index 2090d727c4..76b5a6ed0b 100644 --- a/raphtory/src/io/arrow/dataframe.rs +++ b/raphtory/src/io/arrow/dataframe.rs @@ -20,14 +20,6 @@ impl DFView where I: Iterator>, { - pub(crate) fn new(names: Vec, chunks: I, num_rows: usize) -> Self { - Self { - names, - chunks, - num_rows, - } - } - pub fn check_cols_exist(&self, cols: &[&str]) -> Result<(), GraphError> { let non_cols: Vec<&&str> = cols .iter() diff --git a/raphtory/src/io/neo4j_loader.rs b/raphtory/src/io/neo4j_loader.rs index 882d771afd..3013c76841 100644 --- a/raphtory/src/io/neo4j_loader.rs +++ b/raphtory/src/io/neo4j_loader.rs @@ -1,5 +1,6 @@ use crate::db::graph::graph as rap; use neo4rs::*; + /// A struct that defines the Neo4J loader with configurable options. pub struct Neo4JConnection { // The created graph object given the arguments @@ -23,21 +24,13 @@ impl Neo4JConnection { Ok(Self { neo_graph: graph }) } - pub async fn run(&self, query: Query) -> Result<()> { - self.neo_graph.run(query).await - } - - pub async fn execute(&self, query: Query) -> Result { - self.neo_graph.execute(query).await - } - pub async fn load_query_into_graph( &self, g: &rap::Graph, query: Query, loader: fn(Row, &rap::Graph), ) -> Result<()> { - let mut result = self.neo_graph.execute(query).await.unwrap(); + let mut result = self.neo_graph.execute(query).await?; while let Ok(Some(row)) = result.next().await { loader(row, g); @@ -87,7 +80,7 @@ mod neo_loader_test { actor_name, film_title, NO_PROPS, - Some(relation_type.as_str()), + Some(relation_type), ) .unwrap(); } @@ -128,11 +121,13 @@ mod neo_loader_test { .await .unwrap(); - neo.run(query("CREATE (p:Person {id: $id})").param("id", 3)) + neo.neo_graph + .run(query("CREATE (p:Person {id: $id})").param("id", 3)) .await .unwrap(); let mut result = neo + .neo_graph .execute(query("MATCH (p:Person {id: $id}) RETURN p").param("id", 3)) .await .unwrap(); diff --git a/raphtory/src/lib.rs b/raphtory/src/lib.rs index 0e48df2be3..3acb134bf7 100644 --- a/raphtory/src/lib.rs +++ b/raphtory/src/lib.rs @@ -82,7 +82,7 @@ //! raphtory is created by [Pometry](https://pometry.com). //! We are always looking for contributors to help us improve the library. //! If you are interested in contributing, please see -//! our [Github repository](https://github.com/Raphtory/raphtory) +//! our [GitHub repository](https://github.com/Raphtory/raphtory) pub mod algorithms; pub mod core; pub mod db; @@ -107,6 +107,9 @@ pub mod vectors; #[cfg(feature = "io")] pub mod io; +#[cfg(feature = "proto")] +pub mod serialise; + pub mod prelude { pub const NO_PROPS: [(&str, Prop); 0] = []; pub use crate::{ @@ -123,18 +126,14 @@ pub mod prelude { }, }; pub use raphtory_api::core::{entities::GID, input::input_node::InputNode}; + + #[cfg(feature = "proto")] + pub use crate::serialise::{CacheOps, StableDecode, StableEncode}; } -// Upgrade this version number every time you make a breaking change to Graph structure. -pub const BINCODE_VERSION: u32 = 3u32; #[cfg(feature = "storage")] pub use polars_arrow as arrow2; -#[cfg(feature = "proto")] -mod serialise { - include!(concat!(env!("OUT_DIR"), "/serialise.rs")); -} - #[cfg(test)] mod test_utils { use crate::prelude::Graph; diff --git a/raphtory/src/python/graph/disk_graph.rs b/raphtory/src/python/graph/disk_graph.rs index 9d66f6ebaf..b67fd91b14 100644 --- a/raphtory/src/python/graph/disk_graph.rs +++ b/raphtory/src/python/graph/disk_graph.rs @@ -6,12 +6,10 @@ use crate::{ }, core::utils::errors::GraphError, db::graph::views::deletion_graph::PersistentGraph, - disk_graph::{graph_impl::ParquetLayerCols, DiskGraphError, DiskGraphStorage}, + disk_graph::{graph_impl::ParquetLayerCols, DiskGraphStorage}, io::arrow::dataframe::{DFChunk, DFView}, prelude::Graph, - python::{ - graph::graph::PyGraph, types::repr::StructReprBuilder, utils::errors::adapt_err_value, - }, + python::{graph::graph::PyGraph, types::repr::StructReprBuilder}, }; use itertools::Itertools; /// A columnar temporal graph. @@ -21,12 +19,6 @@ use pyo3::{ }; use std::path::Path; -impl From for PyErr { - fn from(value: DiskGraphError) -> Self { - adapt_err_value(&value) - } -} - #[derive(Clone)] #[pyclass(name = "DiskGraphStorage")] pub struct PyDiskGraph { @@ -117,10 +109,7 @@ impl<'a> FromPyObject<'a> for ParquetLayerColsList<'a> { #[pymethods] impl PyGraph { /// save graph in disk_graph format and memory map the result - pub fn persist_as_disk_graph( - &self, - graph_dir: &str, - ) -> Result { + pub fn persist_as_disk_graph(&self, graph_dir: &str) -> Result { self.graph.persist_as_disk_graph(graph_dir) } } @@ -212,7 +201,7 @@ impl PyDiskGraph { &self, other: &Self, graph_dir: &str, - ) -> Result { + ) -> Result { self.graph.merge_by_sorted_gids(&other.graph, graph_dir) } diff --git a/raphtory/src/python/graph/graph.rs b/raphtory/src/python/graph/graph.rs index 6828eacfa0..a5643f624f 100644 --- a/raphtory/src/python/graph/graph.rs +++ b/raphtory/src/python/graph/graph.rs @@ -7,10 +7,7 @@ use crate::{ algorithms::components::LargestConnectedComponent, core::{entities::nodes::node_ref::NodeRef, utils::errors::GraphError}, db::{ - api::view::{ - internal::{CoreGraphOps, DynamicGraph, IntoDynamic, MaterializedGraph}, - serialise::{StableDecode, StableEncoder}, - }, + api::view::internal::{CoreGraphOps, DynamicGraph, IntoDynamic, MaterializedGraph}, graph::{edge::EdgeView, node::NodeView, views::node_subgraph::NodeSubgraph}, }, io::parquet_loaders::*, @@ -22,13 +19,14 @@ use crate::{ }, utils::PyTime, }, + serialise::{StableDecode, StableEncode}, }; -use pyo3::{prelude::*, types::PyBytes}; +use pyo3::prelude::*; use raphtory_api::core::{entities::GID, storage::arc_str::ArcStr}; use std::{ collections::HashMap, fmt::{Debug, Formatter}, - path::{Path, PathBuf}, + path::PathBuf, }; /// A temporal graph. @@ -38,6 +36,8 @@ pub struct PyGraph { pub graph: Graph, } +impl_serialise!(PyGraph, graph: Graph, "Graph"); + impl Debug for PyGraph { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.graph) @@ -151,26 +151,14 @@ impl PyGraph { } #[cfg(feature = "storage")] - pub fn to_disk_graph(&self, graph_dir: String) -> PyResult> { + pub fn to_disk_graph(&self, graph_dir: String) -> Result { + use crate::db::api::storage::graph::storage_ops::GraphStorage; use std::sync::Arc; - use crate::db::api::storage::storage_ops::GraphStorage; - let disk_graph = Graph::persist_as_disk_graph(&self.graph, graph_dir)?; let storage = GraphStorage::Disk(Arc::new(disk_graph)); let graph = Graph::from_internal_graph(storage); - - Python::with_gil(|py| { - Ok(Py::new( - py, - ( - Self { - graph: graph.clone(), - }, - PyGraphView::from(graph.clone()), - ), - )?) - }) + Ok(graph) } /// Adds a new node with the given id and properties to the graph. @@ -365,30 +353,6 @@ impl PyGraph { // Alternative constructors are tricky, see: https://gist.github.com/redshiftzero/648e4feeff3843ffd9924f13625f839c - /// Loads a graph from the given path. - /// - /// Arguments: - /// path (str): The path to the graph. - /// - /// Returns: - /// Graph: The loaded graph. - #[staticmethod] - #[pyo3(signature = (path, force = false))] - pub fn load_from_file(path: &str, force: bool) -> Result { - Graph::load_from_file(path, force) - } - - /// Saves the graph to the given path. - /// - /// Arguments: - /// path (str): The path to the graph. - /// - /// Returns: - /// None - pub fn save_to_file(&self, path: &str) -> Result<(), GraphError> { - self.graph.save_to_file(Path::new(path)) - } - /// Returns all the node types in the graph. /// /// Returns: @@ -397,19 +361,6 @@ impl PyGraph { self.graph.get_all_node_types() } - /// Get bincode encoded graph - pub fn bincode<'py>(&'py self, py: Python<'py>) -> Result<&'py PyBytes, GraphError> { - let bytes = MaterializedGraph::from(self.graph.clone()).bincode()?; - Ok(PyBytes::new(py, &bytes)) - } - - /// Creates a graph from a bincode encoded graph - #[staticmethod] - fn from_bincode(bytes: &[u8]) -> Result, GraphError> { - let graph = MaterializedGraph::from_bincode(bytes)?; - Ok(graph.into_events()) - } - /// Gives the large connected component of a graph. /// /// # Example Usage: diff --git a/raphtory/src/python/graph/graph_with_deletions.rs b/raphtory/src/python/graph/graph_with_deletions.rs index 59ddd40949..b0518110fc 100644 --- a/raphtory/src/python/graph/graph_with_deletions.rs +++ b/raphtory/src/python/graph/graph_with_deletions.rs @@ -10,10 +10,7 @@ use crate::{ db::{ api::{ mutation::{AdditionOps, PropertyAdditionOps}, - view::{ - internal::{CoreGraphOps, MaterializedGraph}, - serialise::StableEncoder, - }, + view::internal::CoreGraphOps, }, graph::{edge::EdgeView, node::NodeView, views::deletion_graph::PersistentGraph}, }, @@ -23,19 +20,19 @@ use crate::{ utils::PyTime, }, }; -use pyo3::{prelude::*, types::PyBytes}; +use pyo3::prelude::*; use raphtory_api::core::{entities::GID, storage::arc_str::ArcStr}; use std::{ collections::HashMap, fmt::{Debug, Formatter}, - path::{Path, PathBuf}, + path::PathBuf, }; use super::{ graph::{PyGraph, PyGraphEncoder}, io::pandas_loaders::*, }; -use crate::io::parquet_loaders::*; +use crate::{io::parquet_loaders::*, serialise::StableEncode}; /// A temporal graph that allows edges and nodes to be deleted. #[derive(Clone)] @@ -44,6 +41,8 @@ pub struct PyPersistentGraph { pub(crate) graph: PersistentGraph, } +impl_serialise!(PyPersistentGraph, graph: PersistentGraph, "PersistentGraph"); + impl Debug for PyPersistentGraph { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.graph) @@ -327,31 +326,6 @@ impl PyPersistentGraph { // Alternative constructors are tricky, see: https://gist.github.com/redshiftzero/648e4feeff3843ffd9924f13625f839c - /// Loads a graph from the given path. - /// - /// Arguments: - /// path (str): The path to the graph. - /// - /// Returns: - /// Graph: The loaded graph. - #[staticmethod] - #[pyo3(signature = (path, force = false))] - pub fn load_from_file(path: &str, force: bool) -> Result { - let file_path: PathBuf = [env!("CARGO_MANIFEST_DIR"), path].iter().collect(); - PersistentGraph::load_from_file(file_path, force) - } - - /// Saves the graph to the given path. - /// - /// Arguments: - /// path (str): The path to the graph. - /// - /// Returns: - /// None - pub fn save_to_file(&self, path: &str) -> Result<(), GraphError> { - self.graph.save_to_file(Path::new(path)) - } - /// Returns all the node types in the graph. /// /// Returns: @@ -360,19 +334,6 @@ impl PyPersistentGraph { self.graph.get_all_node_types() } - /// Get bincode encoded graph - pub fn bincode<'py>(&'py self, py: Python<'py>) -> Result<&'py PyBytes, GraphError> { - let bytes = MaterializedGraph::from(self.graph.clone()).bincode()?; - Ok(PyBytes::new(py, &bytes)) - } - - /// Creates a graph from a bincode encoded graph - #[staticmethod] - fn from_bincode(bytes: &[u8]) -> Result, GraphError> { - let graph = MaterializedGraph::from_bincode(bytes)?; - Ok(graph.into_persistent()) - } - /// Get event graph pub fn event_graph<'py>(&'py self) -> PyResult> { PyGraph::py_from_db_graph(self.graph.event_graph()) @@ -556,7 +517,7 @@ impl PyPersistentGraph { shared_const_properties: Option>, ) -> Result<(), GraphError> { load_nodes_from_pandas( - &self.graph.0, + self.graph.core_graph(), df, id, time, @@ -635,7 +596,7 @@ impl PyPersistentGraph { layer_in_df: Option, ) -> Result<(), GraphError> { load_edges_from_pandas( - &self.graph.0, + self.graph.core_graph(), df, src, dst, @@ -712,7 +673,15 @@ impl PyPersistentGraph { layer: Option<&str>, layer_in_df: Option, ) -> Result<(), GraphError> { - load_edges_deletions_from_pandas(&self.graph.0, df, src, dst, time, layer, layer_in_df) + load_edges_deletions_from_pandas( + self.graph.core_graph(), + df, + src, + dst, + time, + layer, + layer_in_df, + ) } /// Load edges deletions from a Parquet file into the graph. @@ -767,7 +736,7 @@ impl PyPersistentGraph { shared_const_properties: Option>, ) -> Result<(), GraphError> { load_node_props_from_pandas( - &self.graph.0, + self.graph.core_graph(), df, id, const_properties.as_ref().map(|props| props.as_ref()), @@ -827,7 +796,7 @@ impl PyPersistentGraph { layer_in_df: Option, ) -> Result<(), GraphError> { load_edge_props_from_pandas( - &self.graph.0, + self.graph.core_graph(), df, src, dst, diff --git a/raphtory/src/python/graph/io/pandas_loaders.rs b/raphtory/src/python/graph/io/pandas_loaders.rs index be0317abc6..e165391c70 100644 --- a/raphtory/src/python/graph/io/pandas_loaders.rs +++ b/raphtory/src/python/graph/io/pandas_loaders.rs @@ -1,6 +1,6 @@ use crate::{ core::{utils::errors::GraphError, Prop}, - db::api::{storage::storage_ops::GraphStorage, view::internal::CoreGraphOps}, + db::api::{storage::graph::storage_ops::GraphStorage, view::internal::CoreGraphOps}, io::arrow::{dataframe::*, df_loaders::*}, python::graph::io::*, }; diff --git a/raphtory/src/python/graph/views/graph_view.rs b/raphtory/src/python/graph/views/graph_view.rs index 724895cf16..ef293a636f 100644 --- a/raphtory/src/python/graph/views/graph_view.rs +++ b/raphtory/src/python/graph/views/graph_view.rs @@ -33,7 +33,7 @@ use crate::{ }, }; use chrono::prelude::*; -use pyo3::{prelude::*, types::PyBytes}; +use pyo3::prelude::*; use raphtory_api::core::storage::arc_str::ArcStr; impl IntoPy for MaterializedGraph { @@ -53,15 +53,7 @@ impl IntoPy for DynamicGraph { impl<'source> FromPyObject<'source> for DynamicGraph { fn extract(ob: &'source PyAny) -> PyResult { - ob.extract::>() - .map(|g| g.graph.clone()) - .or_else(|err| { - let res = ob.call_method0("bincode").map_err(|_| err)?; // return original error as probably more helpful - // assume we have a graph at this point, the res probably should not fail - let b = res.extract::<&[u8]>()?; - let g = MaterializedGraph::from_bincode(b)?; - Ok(g.into_dynamic()) - }) + ob.extract::>().map(|g| g.graph.clone()) } } /// Graph view is a read-only version of a graph at a certain point in time. @@ -350,12 +342,6 @@ impl PyGraphView { self.graph.materialize() } - /// Get bincode encoded graph - pub fn bincode<'py>(&'py self, py: Python<'py>) -> Result<&'py PyBytes, GraphError> { - let bytes = self.graph.materialize()?.bincode()?; - Ok(PyBytes::new(py, &bytes)) - } - /// Displays the graph pub fn __repr__(&self) -> String { self.repr() diff --git a/raphtory/src/python/types/macros/trait_impl/mod.rs b/raphtory/src/python/types/macros/trait_impl/mod.rs index fcd619dcae..eb786ae389 100644 --- a/raphtory/src/python/types/macros/trait_impl/mod.rs +++ b/raphtory/src/python/types/macros/trait_impl/mod.rs @@ -16,3 +16,6 @@ mod repr; mod iterable_mixin; mod node_state; + +#[macro_use] +mod serialise; diff --git a/raphtory/src/python/types/macros/trait_impl/serialise.rs b/raphtory/src/python/types/macros/trait_impl/serialise.rs new file mode 100644 index 0000000000..9f4ac453c6 --- /dev/null +++ b/raphtory/src/python/types/macros/trait_impl/serialise.rs @@ -0,0 +1,85 @@ +/// Macro for implementing all the Cache methods on a python wrapper +/// +/// # Arguments +/// * obj: The struct the methods should be implemented for +/// * field: The name of the struct field holding the rust struct implementing `Cache` +/// * base_type: The rust type of `field` +/// * name: The name of the object that appears in the docstring +macro_rules! impl_serialise { + ($obj:ty, $field:ident: $base_type:ty, $name:literal) => { + #[pyo3::pymethods] + impl $obj { + #[doc = concat!(" Write ", $name, " to cache file and initialise the cache.")] + /// + /// Future updates are tracked. Use `write_updates` to persist them to the + /// cache file. If the file already exists its contents are overwritten. + /// + /// Arguments: + /// path (str): The path to the cache file + fn cache(&self, path: &str) -> Result<(), GraphError> { + $crate::serialise::CacheOps::cache(&self.$field, path) + } + + /// Persist the new updates by appending them to the cache file. + fn write_updates(&self) -> Result<(), GraphError> { + $crate::serialise::CacheOps::write_updates(&self.$field) + } + + #[doc = concat!(" Load ", $name, " from a file and initialise it as a cache file.")] + /// + /// Future updates are tracked. Use `write_updates` to persist them to the + /// cache file. + /// + /// Arguments: + /// path (str): The path to the cache file + /// + /// Returns: + #[doc = concat!(" ", $name)] + #[staticmethod] + fn load_cached(path: &str) -> Result<$base_type, GraphError> { + <$base_type as $crate::serialise::CacheOps>::load_cached(path) + } + + #[doc = concat!(" Load ", $name, " from a file.")] + /// + /// Arguments: + /// path (str): The path to the file. + /// + /// Returns: + #[doc = concat!(" ", $name)] + #[staticmethod] + fn load_from_file(path: &str) -> Result<$base_type, GraphError> { + <$base_type as $crate::serialise::StableDecode>::decode(path) + } + + #[doc = concat!(" Saves the ", $name, " to the given path.")] + /// + /// Arguments: + /// path (str): The path to the file. + fn save_to_file(&self, path: &str) -> Result<(), GraphError> { + $crate::serialise::StableEncode::encode(&self.$field, path) + } + + #[doc = concat!(" Load ", $name, " from serialised bytes.")] + /// + /// Arguments: + /// bytes (Bytes): The serialised bytes to decode + /// + /// Returns: + #[doc = concat!(" ", $name)] + #[staticmethod] + fn deserialise(bytes: &[u8]) -> Result<$base_type, GraphError> { + <$base_type as $crate::serialise::StableDecode>::decode_from_bytes(bytes) + } + + #[doc = concat!(" Serialise ", $name, " to bytes.")] + /// + /// Returns: + /// Bytes + fn serialise<'py>(&self, py: Python<'py>) -> &'py pyo3::types::PyBytes { + let bytes = $crate::serialise::StableEncode::encode_to_vec(&self.$field); + pyo3::types::PyBytes::new(py, &bytes) + } + } + }; +} diff --git a/raphtory/src/search/mod.rs b/raphtory/src/search/mod.rs index c022fcd35e..cde2682fc7 100644 --- a/raphtory/src/search/mod.rs +++ b/raphtory/src/search/mod.rs @@ -1,16 +1,8 @@ // search goes here pub mod into_indexed; - -use std::{collections::HashSet, ops::Deref, path::Path, sync::Arc}; - -use raphtory_api::core::storage::arc_str::ArcStr; -use rayon::{prelude::ParallelIterator, slice::ParallelSlice}; -use tantivy::{ - collector::TopDocs, - schema::{Field, Schema, SchemaBuilder, Value, FAST, INDEXED, STORED, TEXT}, - Index, IndexReader, IndexSettings, IndexWriter, TantivyDocument, TantivyError, -}; +#[cfg(feature = "proto")] +mod serialise; use crate::{ core::{ @@ -25,16 +17,24 @@ use crate::{ db::{ api::{ mutation::internal::{InheritPropertyAdditionOps, InternalAdditionOps}, - storage::edges::edge_storage_ops::EdgeStorageOps, + storage::graph::edges::edge_storage_ops::EdgeStorageOps, view::{ internal::{DynamicGraph, InheritViewOps, IntoDynamic, Static}, - Base, MaterializedGraph, StaticGraphViewOps, + Base, StaticGraphViewOps, }, }, graph::{edge::EdgeView, node::NodeView}, }, prelude::*, }; +use raphtory_api::core::storage::{arc_str::ArcStr, dict_mapper::MaybeNew}; +use rayon::{prelude::ParallelIterator, slice::ParallelSlice}; +use std::{collections::HashSet, ops::Deref, sync::Arc}; +use tantivy::{ + collector::TopDocs, + schema::{Field, Schema, SchemaBuilder, Value, FAST, INDEXED, STORED, TEXT}, + Index, IndexReader, IndexSettings, IndexWriter, TantivyDocument, TantivyError, +}; #[derive(Clone)] pub struct IndexedGraph { @@ -93,12 +93,6 @@ impl + IntoDynamic> IndexedGraph { } } -impl IndexedGraph { - pub fn save_to_file(&self, path: impl AsRef) -> Result<(), GraphError> { - self.graph.save_to_file(path) - } -} - impl<'graph, G: GraphViewOps<'graph>> IndexedGraph { pub fn graph(&self) -> &G { &self.graph @@ -765,27 +759,31 @@ impl InternalAdditionOps for Indexe self.graph.next_event_id() } #[inline] - fn resolve_layer(&self, layer: Option<&str>) -> Result { + fn resolve_layer(&self, layer: Option<&str>) -> Result, GraphError> { self.graph.resolve_layer(layer) } #[inline] - fn set_node_type(&self, v_id: VID, node_type: &str) -> Result<(), GraphError> { - self.graph.set_node_type(v_id, node_type)?; - let node_type_field = self.node_index.schema().get_field(fields::NODE_TYPE)?; + fn resolve_node(&self, n: V) -> Result, GraphError> { + self.graph.resolve_node(n) + } + + fn resolve_node_and_type( + &self, + id: V, + node_type: &str, + ) -> Result, MaybeNew)>, GraphError> { + let res = self.graph.resolve_node_and_type(id, node_type)?; + let (vid, _) = res.inner(); let mut document = TantivyDocument::new(); + let node_type_field = self.node_index.schema().get_field(fields::NODE_TYPE)?; document.add_text(node_type_field, node_type); let node_id_field = self.node_index.schema().get_field(fields::VERTEX_ID)?; - document.add_u64(node_id_field, v_id.as_u64()); + document.add_u64(node_id_field, vid.inner().as_u64()); let mut writer = self.node_index.writer(50_000_000)?; writer.add_document(document)?; writer.commit()?; - Ok(()) - } - - #[inline] - fn resolve_node(&self, n: V) -> Result { - self.graph.resolve_node(n) + Ok(res) } #[inline] @@ -794,7 +792,7 @@ impl InternalAdditionOps for Indexe prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.graph.resolve_graph_property(prop, dtype, is_static) } @@ -804,7 +802,7 @@ impl InternalAdditionOps for Indexe prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.graph.resolve_node_property(prop, dtype, is_static) } @@ -814,7 +812,7 @@ impl InternalAdditionOps for Indexe prop: &str, dtype: PropType, is_static: bool, - ) -> Result { + ) -> Result, GraphError> { self.graph.resolve_edge_property(prop, dtype, is_static) } @@ -822,7 +820,7 @@ impl InternalAdditionOps for Indexe &self, t: TimeIndexEntry, v: VID, - props: Vec<(usize, Prop)>, + props: &[(usize, Prop)], ) -> Result<(), GraphError> { let mut document = TantivyDocument::new(); // add time to the document @@ -862,9 +860,9 @@ impl InternalAdditionOps for Indexe _t: TimeIndexEntry, _src: VID, _dst: VID, - _props: Vec<(usize, Prop)>, + _props: &[(usize, Prop)], _layer: usize, - ) -> Result { + ) -> Result, GraphError> { todo!() } @@ -872,7 +870,7 @@ impl InternalAdditionOps for Indexe &self, _t: TimeIndexEntry, _edge: EID, - _props: Vec<(usize, Prop)>, + _props: &[(usize, Prop)], _layer: usize, ) -> Result<(), GraphError> { todo!() @@ -915,9 +913,10 @@ mod test { } #[test] + #[cfg(feature = "proto")] #[ignore = "this test is for experiments with the jira graph"] fn load_jira_graph() -> Result<(), GraphError> { - let graph = Graph::load_from_file("/tmp/graphs/jira", false).expect("failed to load graph"); + let graph = Graph::decode("/tmp/graphs/jira").expect("failed to load graph"); assert!(graph.count_nodes() > 0); let now = SystemTime::now(); diff --git a/raphtory/src/search/serialise.rs b/raphtory/src/search/serialise.rs new file mode 100644 index 0000000000..7d53768eae --- /dev/null +++ b/raphtory/src/search/serialise.rs @@ -0,0 +1,34 @@ +use crate::{ + core::utils::errors::GraphError, + prelude::{GraphViewOps, StableDecode, StableEncode}, + search::IndexedGraph, + serialise::{ + incremental::{GraphWriter, InternalCache}, + ProtoGraph, + }, +}; +use std::path::Path; + +impl StableEncode for IndexedGraph { + fn encode_to_proto(&self) -> ProtoGraph { + self.graph.encode_to_proto() + } +} + +impl<'graph, G: StableDecode + GraphViewOps<'graph>> StableDecode for IndexedGraph { + fn decode_from_proto(graph: &ProtoGraph) -> Result { + let inner = G::decode_from_proto(graph)?; + let indexed = Self::from_graph(&inner)?; + Ok(indexed) + } +} + +impl InternalCache for IndexedGraph { + fn init_cache(&self, path: impl AsRef) -> Result<(), GraphError> { + self.graph.init_cache(path) + } + + fn get_cache(&self) -> Option<&GraphWriter> { + self.graph.get_cache() + } +} diff --git a/raphtory/src/graph.proto b/raphtory/src/serialise/graph.proto similarity index 97% rename from raphtory/src/graph.proto rename to raphtory/src/serialise/graph.proto index fe1c899c45..4c247923d0 100644 --- a/raphtory/src/graph.proto +++ b/raphtory/src/serialise/graph.proto @@ -100,6 +100,8 @@ message GraphUpdate { DelEdge del_edge = 5; UpdateEdgeCProps update_edge_cprops = 6; UpdateEdgeTProps update_edge_tprops = 7; + + UpdateNodeType update_node_type = 8; } message UpdateNodeTProps { @@ -114,6 +116,11 @@ message GraphUpdate { repeated PropPair properties = 2; } + message UpdateNodeType { + uint64 id = 1; + uint64 type_id = 2; + } + message UpdateEdgeTProps { uint64 eid = 1; int64 time = 2; diff --git a/raphtory/src/serialise/incremental.rs b/raphtory/src/serialise/incremental.rs new file mode 100644 index 0000000000..e02ea5d37b --- /dev/null +++ b/raphtory/src/serialise/incremental.rs @@ -0,0 +1,263 @@ +use crate::{ + core::{utils::errors::GraphError, Prop, PropType}, + db::{ + api::{storage::storage::Storage, view::MaterializedGraph}, + graph::views::deletion_graph::PersistentGraph, + }, + prelude::Graph, + serialise::{ + serialise::{CacheOps, StableDecode, StableEncode}, + ProtoGraph, + }, +}; +use parking_lot::Mutex; +use prost::Message; +use raphtory_api::core::{ + entities::{GidRef, EID, VID}, + storage::{dict_mapper::MaybeNew, timeindex::TimeIndexEntry}, +}; +use std::{ + fmt::Debug, + fs::{File, OpenOptions}, + io::Write, + mem, + ops::DerefMut, + path::Path, +}; + +#[derive(Debug)] +pub struct GraphWriter { + writer: Mutex, + proto_delta: Mutex, +} + +impl GraphWriter { + pub fn new(file: File) -> Self { + Self { + writer: Mutex::new(file), + proto_delta: Default::default(), + } + } + pub fn write(&self) -> Result<(), GraphError> { + let proto = mem::take(self.proto_delta.lock().deref_mut()); + let bytes = proto.encode_to_vec(); + if !bytes.is_empty() { + self.writer.lock().write_all(&bytes)?; + } + Ok(()) + } + + #[inline] + pub fn resolve_layer(&self, layer: Option<&str>, layer_id: MaybeNew) { + layer_id.if_new(|id| { + if let Some(layer) = layer { + self.proto_delta.lock().new_layer(layer, id) + } + }); + } + + pub fn resolve_node(&self, vid: MaybeNew, gid: GidRef) { + vid.if_new(|vid| self.proto_delta.lock().new_node(gid, vid, 0)); + } + + pub fn resolve_node_and_type( + &self, + node_and_type: MaybeNew<(MaybeNew, MaybeNew)>, + node_type: &str, + gid: GidRef, + ) { + if let MaybeNew::New((MaybeNew::Existing(node_id), type_id)) = node_and_type { + // type assignment changed but node already exists + self.proto_delta + .lock() + .update_node_type(node_id, type_id.inner()); + } + if let (MaybeNew::New(node_id), type_id) = node_and_type.inner() { + self.proto_delta + .lock() + .new_node(gid, node_id, type_id.inner()); + } + if let (_, MaybeNew::New(type_id)) = node_and_type.inner() { + self.proto_delta.lock().new_node_type(node_type, type_id); + } + } + + pub fn resolve_graph_property( + &self, + prop: &str, + prop_id: MaybeNew, + dtype: PropType, + is_static: bool, + ) { + prop_id.if_new(|id| { + if is_static { + self.proto_delta.lock().new_graph_cprop(prop, id); + } else { + self.proto_delta.lock().new_graph_tprop(prop, id, &dtype); + } + }); + } + + pub fn resolve_node_property( + &self, + prop: &str, + prop_id: MaybeNew, + dtype: PropType, + is_static: bool, + ) { + prop_id.if_new(|id| { + if is_static { + self.proto_delta.lock().new_node_cprop(prop, id, &dtype); + } else { + self.proto_delta.lock().new_node_tprop(prop, id, &dtype); + } + }); + } + + pub fn resolve_edge_property( + &self, + prop: &str, + prop_id: MaybeNew, + dtype: PropType, + is_static: bool, + ) { + prop_id.if_new(|id| { + if is_static { + self.proto_delta.lock().new_edge_cprop(prop, id, &dtype); + } else { + self.proto_delta.lock().new_edge_tprop(prop, id, &dtype); + } + }); + } + + pub fn add_node_update(&self, t: TimeIndexEntry, v: VID, props: &[(usize, Prop)]) { + self.proto_delta + .lock() + .update_node_tprops(v, t, props.iter().map(|(id, prop)| (*id, prop))) + } + + pub fn resolve_edge(&self, eid: MaybeNew, src: VID, dst: VID) { + eid.if_new(|eid| self.proto_delta.lock().new_edge(src, dst, eid)); + } + + pub fn add_edge_update( + &self, + t: TimeIndexEntry, + edge: EID, + props: &[(usize, Prop)], + layer: usize, + ) { + self.proto_delta.lock().update_edge_tprops( + edge, + t, + layer, + props.iter().map(|(id, prop)| (*id, prop)), + ) + } + pub fn add_graph_tprops(&self, t: TimeIndexEntry, props: &[(usize, Prop)]) { + self.proto_delta + .lock() + .update_graph_tprops(t, props.iter().map(|(id, prop)| (*id, prop))) + } + + pub fn add_graph_cprops(&self, props: &[(usize, Prop)]) { + self.proto_delta + .lock() + .update_graph_cprops(props.iter().map(|(id, prop)| (*id, prop))) + } + + pub fn add_node_cprops(&self, node: VID, props: &[(usize, Prop)]) { + self.proto_delta + .lock() + .update_node_cprops(node, props.iter().map(|(id, prop)| (*id, prop))) + } + + pub fn add_edge_cprops(&self, edge: EID, layer: usize, props: &[(usize, Prop)]) { + self.proto_delta.lock().update_edge_cprops( + edge, + layer, + props.iter().map(|(id, prop)| (*id, prop)), + ) + } + + pub fn delete_edge(&self, edge: EID, t: TimeIndexEntry, layer: usize) { + self.proto_delta.lock().del_edge(edge, layer, t) + } +} + +pub(crate) trait InternalCache { + /// Initialise the cache by pointing it at a proto file. + /// Future updates will be appended to the cache. + fn init_cache(&self, path: impl AsRef) -> Result<(), GraphError>; + + /// Get the cache writer if it is initialised. + fn get_cache(&self) -> Option<&GraphWriter>; +} + +impl InternalCache for Storage { + fn init_cache(&self, path: impl AsRef) -> Result<(), GraphError> { + self.cache.get_or_try_init(|| { + let file = OpenOptions::new().append(true).open(path)?; + Ok::<_, GraphError>(GraphWriter::new(file)) + })?; + Ok(()) + } + + fn get_cache(&self) -> Option<&GraphWriter> { + self.cache.get() + } +} + +impl InternalCache for Graph { + fn init_cache(&self, path: impl AsRef) -> Result<(), GraphError> { + self.inner.init_cache(path) + } + + fn get_cache(&self) -> Option<&GraphWriter> { + self.inner.get_cache() + } +} + +impl InternalCache for PersistentGraph { + fn init_cache(&self, path: impl AsRef) -> Result<(), GraphError> { + self.0.init_cache(path) + } + + fn get_cache(&self) -> Option<&GraphWriter> { + self.0.get_cache() + } +} + +impl InternalCache for MaterializedGraph { + fn init_cache(&self, path: impl AsRef) -> Result<(), GraphError> { + match self { + MaterializedGraph::EventGraph(g) => g.init_cache(path), + MaterializedGraph::PersistentGraph(g) => g.init_cache(path), + } + } + + fn get_cache(&self) -> Option<&GraphWriter> { + match self { + MaterializedGraph::EventGraph(g) => g.get_cache(), + MaterializedGraph::PersistentGraph(g) => g.get_cache(), + } + } +} + +impl CacheOps for G { + fn cache(&self, path: impl AsRef) -> Result<(), GraphError> { + self.encode(path.as_ref())?; + self.init_cache(path) + } + + fn write_updates(&self) -> Result<(), GraphError> { + let cache = self.get_cache().ok_or(GraphError::CacheNotInnitialised)?; + cache.write() + } + + fn load_cached(path: impl AsRef) -> Result { + let graph = Self::decode(path.as_ref())?; + graph.init_cache(path)?; + Ok(graph) + } +} diff --git a/raphtory/src/serialise/mod.rs b/raphtory/src/serialise/mod.rs new file mode 100644 index 0000000000..f418fc8417 --- /dev/null +++ b/raphtory/src/serialise/mod.rs @@ -0,0 +1,9 @@ +pub(crate) mod incremental; +mod serialise; + +mod proto { + include!(concat!(env!("OUT_DIR"), "/serialise.rs")); +} + +pub use proto::Graph as ProtoGraph; +pub use serialise::{CacheOps, StableDecode, StableEncode}; diff --git a/raphtory/src/db/api/view/serialise.rs b/raphtory/src/serialise/serialise.rs similarity index 82% rename from raphtory/src/db/api/view/serialise.rs rename to raphtory/src/serialise/serialise.rs index c7e32d10a5..7a36156fef 100644 --- a/raphtory/src/db/api/view/serialise.rs +++ b/raphtory/src/serialise/serialise.rs @@ -1,4 +1,3 @@ -use super::MaterializedGraph; use crate::{ core::{ entities::{ @@ -14,19 +13,21 @@ use crate::{ mutation::internal::{ InternalAdditionOps, InternalDeletionOps, InternalPropertyAdditionOps, }, - storage::{ + storage::graph::{ edges::edge_storage_ops::EdgeStorageOps, nodes::node_storage_ops::NodeStorageOps, storage_ops::GraphStorage, tprop_storage_ops::TPropOps, }, - view::internal::CoreGraphOps, + view::{internal::CoreGraphOps, MaterializedGraph}, }, graph::views::deletion_graph::PersistentGraph, }, prelude::Graph, - serialise, serialise::{ - graph_update::*, new_meta::*, new_node, new_node::Gid, prop, - prop_type::PropType as SPropType, GraphUpdate, NewEdge, NewMeta, NewNode, + proto, + proto::{ + graph_update::*, new_meta::*, new_node, new_node::Gid, prop, + prop_type::PropType as SPropType, GraphUpdate, NewEdge, NewMeta, NewNode, + }, }, }; use chrono::{DateTime, Datelike, NaiveDate, NaiveDateTime, NaiveTime, Timelike}; @@ -47,17 +48,17 @@ macro_rules! zip_tprop_updates { &$iter .map(|(key, values)| values.iter().map(move |(t, v)| (t, (key, v)))) .kmerge_by(|(left_t, _), (right_t, _)| left_t <= right_t) - .group_by(|(t, _)| *t) + .chunk_by(|(t, _)| *t) }; } -pub trait StableEncoder { - fn encode_to_proto(&self) -> serialise::Graph; +pub trait StableEncode { + fn encode_to_proto(&self) -> proto::Graph; fn encode_to_vec(&self) -> Vec { self.encode_to_proto().encode_to_vec() } - fn stable_serialise(&self, path: impl AsRef) -> Result<(), GraphError> { + fn encode(&self, path: impl AsRef) -> Result<(), GraphError> { let mut file = File::create(path)?; let bytes = self.encode_to_vec(); file.write_all(&bytes)?; @@ -66,9 +67,9 @@ pub trait StableEncoder { } pub trait StableDecode: Sized { - fn decode_from_proto(graph: &serialise::Graph) -> Result; + fn decode_from_proto(graph: &proto::Graph) -> Result; fn decode_from_bytes(bytes: &[u8]) -> Result { - let graph = serialise::Graph::decode(bytes)?; + let graph = proto::Graph::decode(bytes)?; Self::decode_from_proto(&graph) } fn decode(path: impl AsRef) -> Result { @@ -79,6 +80,19 @@ pub trait StableDecode: Sized { } } +pub trait CacheOps: Sized { + /// Write graph to file and append future updates to the same file. + /// + /// If the file already exists, it's contents are overwritten + fn cache(&self, path: impl AsRef) -> Result<(), GraphError>; + + /// Persist the new updates by appending them to the cache file. + fn write_updates(&self) -> Result<(), GraphError>; + + /// Load graph from file and append future updates to the same file + fn load_cached(path: impl AsRef) -> Result; +} + fn as_proto_prop_type(p_type: &PropType) -> SPropType { match p_type { PropType::Str => SPropType::Str, @@ -214,6 +228,14 @@ impl GraphUpdate { Self::new(Update::UpdateGraphTprops(inner)) } + fn update_node_type(node_id: VID, type_id: usize) -> Self { + let inner = UpdateNodeType { + id: node_id.as_u64(), + type_id: type_id as u64, + }; + Self::new(Update::UpdateNodeType(inner)) + } + fn update_node_cprops( node_id: VID, properties: impl Iterator)>, @@ -313,8 +335,8 @@ impl PropPair { } } -impl serialise::Graph { - fn new_edge(&mut self, src: VID, dst: VID, eid: EID) { +impl proto::Graph { + pub fn new_edge(&mut self, src: VID, dst: VID, eid: EID) { let edge = NewEdge { src: src.as_u64(), dst: dst.as_u64(), @@ -323,7 +345,7 @@ impl serialise::Graph { self.edges.push(edge); } - fn new_node(&mut self, gid: GidRef, vid: VID, type_id: usize) { + pub fn new_node(&mut self, gid: GidRef, vid: VID, type_id: usize) { let type_id = type_id as u64; let gid = match gid { GidRef::U64(id) => new_node::Gid::GidU64(id), @@ -337,43 +359,46 @@ impl serialise::Graph { self.nodes.push(node); } - fn new_graph_cprop(&mut self, key: &str, id: usize) { + pub fn new_graph_cprop(&mut self, key: &str, id: usize) { self.metas.push(NewMeta::new_graph_cprop(key, id)); } - fn new_graph_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { + pub fn new_graph_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { self.metas.push(NewMeta::new_graph_tprop(key, id, dtype)); } - fn new_node_cprop(&mut self, key: &str, id: usize, dtype: &PropType) { + pub fn new_node_cprop(&mut self, key: &str, id: usize, dtype: &PropType) { self.metas.push(NewMeta::new_node_cprop(key, id, dtype)); } - fn new_node_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { + pub fn new_node_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { self.metas.push(NewMeta::new_node_tprop(key, id, dtype)); } - fn new_edge_cprop(&mut self, key: &str, id: usize, dtype: &PropType) { + pub fn new_edge_cprop(&mut self, key: &str, id: usize, dtype: &PropType) { self.metas.push(NewMeta::new_edge_cprop(key, id, dtype)); } - fn new_edge_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { + pub fn new_edge_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { self.metas.push(NewMeta::new_edge_tprop(key, id, dtype)) } - fn new_layer(&mut self, layer: &str, id: usize) { + pub fn new_layer(&mut self, layer: &str, id: usize) { self.metas.push(NewMeta::new_layer(layer, id)); } - fn new_node_type(&mut self, node_type: &str, id: usize) { + pub fn new_node_type(&mut self, node_type: &str, id: usize) { self.metas.push(NewMeta::new_node_type(node_type, id)); } - fn update_graph_cprops(&mut self, values: impl Iterator)>) { + pub fn update_graph_cprops( + &mut self, + values: impl Iterator)>, + ) { self.updates.push(GraphUpdate::update_graph_cprops(values)); } - fn update_graph_tprops( + pub fn update_graph_tprops( &mut self, time: TimeIndexEntry, values: impl IntoIterator)>, @@ -382,7 +407,11 @@ impl serialise::Graph { .push(GraphUpdate::update_graph_tprops(time, values)); } - fn update_node_cprops( + pub fn update_node_type(&mut self, node_id: VID, type_id: usize) { + self.updates + .push(GraphUpdate::update_node_type(node_id, type_id)) + } + pub fn update_node_cprops( &mut self, node_id: VID, properties: impl Iterator)>, @@ -391,7 +420,7 @@ impl serialise::Graph { .push(GraphUpdate::update_node_cprops(node_id, properties)); } - fn update_node_tprops( + pub fn update_node_tprops( &mut self, node_id: VID, time: TimeIndexEntry, @@ -401,7 +430,7 @@ impl serialise::Graph { .push(GraphUpdate::update_node_tprops(node_id, time, properties)); } - fn update_edge_tprops( + pub fn update_edge_tprops( &mut self, eid: EID, time: TimeIndexEntry, @@ -413,7 +442,7 @@ impl serialise::Graph { )); } - fn update_edge_cprops( + pub fn update_edge_cprops( &mut self, eid: EID, layer_id: usize, @@ -423,14 +452,14 @@ impl serialise::Graph { .push(GraphUpdate::update_edge_cprops(eid, layer_id, properties)); } - fn del_edge(&mut self, eid: EID, layer_id: usize, time: TimeIndexEntry) { + pub fn del_edge(&mut self, eid: EID, layer_id: usize, time: TimeIndexEntry) { self.updates .push(GraphUpdate::del_edge(eid, layer_id, time)) } } -impl StableEncoder for GraphStorage { - fn encode_to_proto(&self) -> serialise::Graph { +impl StableEncode for GraphStorage { + fn encode_to_proto(&self) -> proto::Graph { #[cfg(feature = "storage")] if let GraphStorage::Disk(storage) = self { assert!( @@ -440,7 +469,7 @@ impl StableEncoder for GraphStorage { } let storage = self.lock(); - let mut graph = serialise::Graph::default(); + let mut graph = proto::Graph::default(); // Graph Properties let graph_meta = storage.graph_meta(); @@ -467,7 +496,7 @@ impl StableEncoder for GraphStorage { .collect::>() }) .kmerge_by(|(left_t, _), (right_t, _)| left_t <= right_t) - .group_by(|(t, _)| *t) + .chunk_by(|(t, _)| *t) { graph.update_graph_tprops(t, group.map(|(_, v)| v)); } @@ -590,24 +619,33 @@ impl StableEncoder for GraphStorage { } } -impl StableEncoder for Graph { - fn encode_to_proto(&self) -> serialise::Graph { +impl StableEncode for Graph { + fn encode_to_proto(&self) -> proto::Graph { let mut graph = self.core_graph().encode_to_proto(); - graph.set_graph_type(serialise::GraphType::Event); + graph.set_graph_type(proto::GraphType::Event); graph } } -impl StableEncoder for PersistentGraph { - fn encode_to_proto(&self) -> serialise::Graph { +impl StableEncode for PersistentGraph { + fn encode_to_proto(&self) -> proto::Graph { let mut graph = self.core_graph().encode_to_proto(); - graph.set_graph_type(serialise::GraphType::Persistent); + graph.set_graph_type(proto::GraphType::Persistent); graph } } +impl StableEncode for MaterializedGraph { + fn encode_to_proto(&self) -> proto::Graph { + match self { + MaterializedGraph::EventGraph(graph) => graph.encode_to_proto(), + MaterializedGraph::PersistentGraph(graph) => graph.encode_to_proto(), + } + } +} + impl StableDecode for TemporalGraph { - fn decode_from_proto(graph: &serialise::Graph) -> Result { + fn decode_from_proto(graph: &proto::Graph) -> Result { let storage = Self::default(); graph.metas.par_iter().for_each(|meta| { if let Some(meta) = meta.meta.as_ref() { @@ -691,23 +729,24 @@ impl StableDecode for TemporalGraph { Update::UpdateNodeCprops(props) => { storage.internal_update_constant_node_properties( VID(props.id as usize), - collect_props(&props.properties)?, + &collect_props(&props.properties)?, )?; } Update::UpdateNodeTprops(props) => { let time = TimeIndexEntry(props.time, props.secondary as usize); let node = VID(props.id as usize); let props = collect_props(&props.properties)?; - storage.internal_add_node(time, node, props)?; + storage.internal_add_node(time, node, &props)?; } Update::UpdateGraphCprops(props) => { - storage.internal_update_constant_properties(collect_props( + storage.internal_update_constant_properties(&collect_props( &props.properties, )?)?; } Update::UpdateGraphTprops(props) => { let time = TimeIndexEntry(props.time, props.secondary as usize); - storage.internal_add_properties(time, collect_props(&props.properties)?)?; + storage + .internal_add_properties(time, &collect_props(&props.properties)?)?; } Update::DelEdge(del_edge) => { let time = TimeIndexEntry(del_edge.time, del_edge.secondary as usize); @@ -721,7 +760,7 @@ impl StableDecode for TemporalGraph { storage.internal_update_constant_edge_properties( EID(props.eid as usize), props.layer_id as usize, - collect_props(&props.properties)?, + &collect_props(&props.properties)?, )?; } Update::UpdateEdgeTprops(props) => { @@ -730,10 +769,15 @@ impl StableDecode for TemporalGraph { storage.internal_add_edge_update( time, eid, - collect_props(&props.properties)?, + &collect_props(&props.properties)?, props.layer_id as usize, )?; } + Update::UpdateNodeType(update) => { + let id = VID(update.id as usize); + let type_id = update.type_id as usize; + storage.storage.get_node_mut(id).node_type = type_id; + } } } Ok::<_, GraphError>(()) @@ -743,7 +787,7 @@ impl StableDecode for TemporalGraph { } impl StableDecode for GraphStorage { - fn decode_from_proto(graph: &serialise::Graph) -> Result { + fn decode_from_proto(graph: &proto::Graph) -> Result { Ok(GraphStorage::Unlocked(Arc::new( TemporalGraph::decode_from_proto(graph)?, ))) @@ -751,11 +795,11 @@ impl StableDecode for GraphStorage { } impl StableDecode for MaterializedGraph { - fn decode_from_proto(graph: &serialise::Graph) -> Result { + fn decode_from_proto(graph: &proto::Graph) -> Result { let storage = GraphStorage::decode_from_proto(graph)?; let graph = match graph.graph_type() { - serialise::GraphType::Event => Self::EventGraph(Graph::from_internal_graph(storage)), - serialise::GraphType::Persistent => { + proto::GraphType::Event => Self::EventGraph(Graph::from_internal_graph(storage)), + proto::GraphType::Persistent => { Self::PersistentGraph(PersistentGraph::from_internal_graph(storage)) } }; @@ -764,22 +808,22 @@ impl StableDecode for MaterializedGraph { } impl StableDecode for Graph { - fn decode_from_proto(graph: &serialise::Graph) -> Result { + fn decode_from_proto(graph: &proto::Graph) -> Result { match graph.graph_type() { - serialise::GraphType::Event => { + proto::GraphType::Event => { let storage = GraphStorage::decode_from_proto(graph)?; Ok(Graph::from_internal_graph(storage)) } - serialise::GraphType::Persistent => Err(GraphError::GraphLoadError), + proto::GraphType::Persistent => Err(GraphError::GraphLoadError), } } } impl StableDecode for PersistentGraph { - fn decode_from_proto(graph: &serialise::Graph) -> Result { + fn decode_from_proto(graph: &proto::Graph) -> Result { match graph.graph_type() { - serialise::GraphType::Event => Err(GraphError::GraphLoadError), - serialise::GraphType::Persistent => { + proto::GraphType::Event => Err(GraphError::GraphLoadError), + proto::GraphType::Persistent => { let storage = GraphStorage::decode_from_proto(graph)?; Ok(PersistentGraph::from_internal_graph(storage)) } @@ -883,7 +927,7 @@ fn collect_props<'a>( iter.into_iter().map(as_prop).collect() } -fn as_proto_prop(prop: &Prop) -> serialise::Prop { +fn as_proto_prop(prop: &Prop) -> proto::Prop { let value: prop::Value = match prop { Prop::Bool(b) => prop::Value::BoolValue(*b), Prop::U8(u) => prop::Value::U8((*u).into()), @@ -951,7 +995,7 @@ fn as_proto_prop(prop: &Prop) -> serialise::Prop { } }; - serialise::Prop { value: Some(value) } + proto::Prop { value: Some(value) } } #[cfg(test)] @@ -966,7 +1010,7 @@ mod proto_test { graph::graph::assert_graph_equal, }, prelude::*, - serialise::GraphType, + serialise::{proto::GraphType, ProtoGraph}, }; #[test] @@ -974,7 +1018,7 @@ mod proto_test { let temp_file = tempfile::NamedTempFile::new().unwrap(); let g1 = Graph::new(); g1.add_node(1, "Alice", NO_PROPS, None).unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); } @@ -986,7 +1030,7 @@ mod proto_test { g1.add_node(1, "Alice", NO_PROPS, None).unwrap(); g1.add_node(2, "Bob", [("age", Prop::U32(47))], None) .unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); } @@ -1003,7 +1047,7 @@ mod proto_test { n1.update_constant_properties([("name", Prop::Str("Bob".into()))]) .expect("Failed to update constant properties"); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); } @@ -1015,7 +1059,7 @@ mod proto_test { g1.add_node(1, "Alice", NO_PROPS, None).unwrap(); g1.add_node(2, "Bob", NO_PROPS, None).unwrap(); g1.add_edge(3, "Alice", "Bob", NO_PROPS, None).unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); } @@ -1026,7 +1070,7 @@ mod proto_test { let g1 = Graph::new().persistent_graph(); g1.add_edge(3, "Alice", "Bob", NO_PROPS, None).unwrap(); g1.delete_edge(19, "Alice", "Bob", None).unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = PersistentGraph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); @@ -1043,7 +1087,7 @@ mod proto_test { g1.add_node(2, "Bob", NO_PROPS, None).unwrap(); g1.add_edge(3, "Alice", "Bob", [("kind", "friends")], None) .unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); } @@ -1055,7 +1099,7 @@ mod proto_test { let e1 = g1.add_edge(3, "Alice", "Bob", NO_PROPS, None).unwrap(); e1.update_constant_properties([("friends", true)], None) .expect("Failed to update constant properties"); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); } @@ -1068,7 +1112,7 @@ mod proto_test { .unwrap(); g1.add_edge(7, "Bob", "Charlie", [("friends", false)], Some("two")) .unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); } @@ -1081,7 +1125,7 @@ mod proto_test { let temp_file = tempfile::NamedTempFile::new().unwrap(); let g1 = Graph::new(); g1.add_node(1, "Alice", props.clone(), None).unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); @@ -1107,7 +1151,7 @@ mod proto_test { let temp_file = tempfile::NamedTempFile::new().unwrap(); let g1 = Graph::new(); g1.add_edge(1, "Alice", "Bob", props.clone(), None).unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); @@ -1135,7 +1179,7 @@ mod proto_test { let e = g1.add_edge(1, "Alice", "Bob", NO_PROPS, Some("a")).unwrap(); e.update_constant_properties(props.clone(), Some("a")) .expect("Failed to update constant properties"); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); @@ -1161,7 +1205,7 @@ mod proto_test { let n = g1.add_node(1, "Alice", NO_PROPS, None).unwrap(); n.update_constant_properties(props.clone()) .expect("Failed to update constant properties"); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); @@ -1186,7 +1230,7 @@ mod proto_test { .expect("Failed to add constant properties"); let temp_file = tempfile::NamedTempFile::new().unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); @@ -1208,7 +1252,7 @@ mod proto_test { } let temp_file = tempfile::NamedTempFile::new().unwrap(); - g1.stable_serialise(&temp_file).unwrap(); + g1.encode(&temp_file).unwrap(); let g2 = Graph::decode(&temp_file).unwrap(); assert_graph_equal(&g1, &g2); @@ -1230,7 +1274,7 @@ mod proto_test { #[test] fn manually_test_append() { - let mut graph1 = serialise::Graph::default(); + let mut graph1 = proto::Graph::default(); graph1.set_graph_type(GraphType::Event); graph1.new_node(GidRef::Str("1"), VID(0), 0); graph1.new_node(GidRef::Str("2"), VID(1), 0); @@ -1243,7 +1287,7 @@ mod proto_test { ); let mut bytes1 = graph1.encode_to_vec(); - let mut graph2 = serialise::Graph::default(); + let mut graph2 = proto::Graph::default(); graph2.new_node(GidRef::Str("3"), VID(2), 0); graph2.new_edge(VID(0), VID(2), EID(1)); graph2.update_edge_tprops( @@ -1306,6 +1350,106 @@ mod proto_test { } } + #[test] + fn test_incremental_writing_on_graph() { + let g = Graph::new(); + let mut props = vec![]; + write_props_to_vec(&mut props); + let temp_cache_file = tempfile::NamedTempFile::new().unwrap(); + + g.cache(temp_cache_file.path()).unwrap(); + + for t in 0..props.len() { + g.add_properties(t as i64, (&props[t..t + 1]).to_vec()) + .expect("Failed to add constant properties"); + } + g.write_updates().unwrap(); + + g.add_constant_properties(props.clone()) + .expect("Failed to add constant properties"); + g.write_updates().unwrap(); + + let n = g.add_node(1, "Alice", NO_PROPS, None).unwrap(); + n.update_constant_properties(props.clone()) + .expect("Failed to update constant properties"); + g.write_updates().unwrap(); + + let e = g.add_edge(1, "Alice", "Bob", NO_PROPS, Some("a")).unwrap(); + e.update_constant_properties(props.clone(), Some("a")) + .expect("Failed to update constant properties"); + g.write_updates().unwrap(); + + g.add_edge(2, "Alice", "Bob", props.clone(), None).unwrap(); + g.add_node(1, "Charlie", props.clone(), None).unwrap(); + g.write_updates().unwrap(); + + g.add_edge(7, "Alice", "Bob", NO_PROPS, Some("one")) + .unwrap(); + g.add_edge(7, "Bob", "Charlie", [("friends", false)], Some("two")) + .unwrap(); + g.write_updates().unwrap(); + println!("{g:?}"); + + let g2 = Graph::decode(temp_cache_file.path()).unwrap(); + println!("{g2:?}"); + + assert_graph_equal(&g, &g2); + } + + #[test] + fn test_incremental_writing_on_persistent_graph() { + let g = PersistentGraph::new(); + let mut props = vec![]; + write_props_to_vec(&mut props); + let temp_cache_file = tempfile::NamedTempFile::new().unwrap(); + + g.cache(temp_cache_file.path()).unwrap(); + + for t in 0..props.len() { + g.add_properties(t as i64, (&props[t..t + 1]).to_vec()) + .expect("Failed to add constant properties"); + } + g.write_updates().unwrap(); + + g.add_constant_properties(props.clone()) + .expect("Failed to add constant properties"); + g.write_updates().unwrap(); + + let n = g.add_node(1, "Alice", NO_PROPS, None).unwrap(); + n.update_constant_properties(props.clone()) + .expect("Failed to update constant properties"); + g.write_updates().unwrap(); + + let e = g.add_edge(1, "Alice", "Bob", NO_PROPS, Some("a")).unwrap(); + e.update_constant_properties(props.clone(), Some("a")) + .expect("Failed to update constant properties"); + g.write_updates().unwrap(); + + g.add_edge(2, "Alice", "Bob", props.clone(), None).unwrap(); + g.add_node(1, "Charlie", props.clone(), None).unwrap(); + g.write_updates().unwrap(); + + g.add_edge(7, "Alice", "Bob", NO_PROPS, Some("one")) + .unwrap(); + g.add_edge(7, "Bob", "Charlie", [("friends", false)], Some("two")) + .unwrap(); + g.write_updates().unwrap(); + println!("{g:?}"); + + let g2 = PersistentGraph::decode(temp_cache_file.path()).unwrap(); + println!("{g2:?}"); + + assert_graph_equal(&g, &g2); + } + + // we rely on this to make sure writing no updates does not actually write anything to file + #[test] + fn empty_proto_is_empty_bytes() { + let proto = ProtoGraph::default(); + let bytes = proto.encode_to_vec(); + assert!(bytes.is_empty()) + } + fn write_props_to_vec(props: &mut Vec<(&str, Prop)>) { props.push(("name", Prop::Str("Alice".into()))); props.push(("age", Prop::U32(47))); diff --git a/raphtory/src/vectors/embeddings.rs b/raphtory/src/vectors/embeddings.rs index 18e949c339..337d95a98a 100644 --- a/raphtory/src/vectors/embeddings.rs +++ b/raphtory/src/vectors/embeddings.rs @@ -13,6 +13,7 @@ pub async fn openai_embedding(texts: Vec) -> Vec { input: EmbeddingInput::StringArray(texts), user: None, encoding_format: None, + dimensions: None, }; let response = client.embeddings().create(request).await.unwrap(); println!("Generated embeddings successfully");