diff --git a/DEPS b/DEPS index 21daf8203b5e..855d798f27d6 100644 --- a/DEPS +++ b/DEPS @@ -44,7 +44,7 @@ deps = { "url": "https://github.com/ronaldoussoren/macholib.git@36a6777ccd0891c5d1b44ba885573d7c90740015", "condition": "checkout_mac", }, - "components/brave_wallet/browser/zcash/rust/librustzcash/src": "https://github.com/brave/librustzcash.git@4d44f5dc3429dce7df37359b8b3c4716807770ea", + "components/brave_wallet/browser/zcash/rust/librustzcash/src": "https://github.com/brave/librustzcash.git@5b496c6ead39a0e0f32337f9b6bbf63e7ce92830", } recursedeps = [ diff --git a/components/brave_wallet/browser/zcash/rust/BUILD.gn b/components/brave_wallet/browser/zcash/rust/BUILD.gn index b3a61b43a311..673e5d05dfd6 100644 --- a/components/brave_wallet/browser/zcash/rust/BUILD.gn +++ b/components/brave_wallet/browser/zcash/rust/BUILD.gn @@ -67,6 +67,7 @@ rust_static_library("rust_lib") { cxx_bindings = [ "lib.rs" ] deps = [ + "librustzcash:zcash_client_backend", "librustzcash:zcash_primitives", "//brave/components/brave_wallet/rust:rust_lib", "//brave/third_party/rust/incrementalmerkletree/v0_5:lib", @@ -74,6 +75,7 @@ rust_static_library("rust_lib") { "//brave/third_party/rust/nonempty/v0_7:lib", "//brave/third_party/rust/orchard/v0_8:lib", "//brave/third_party/rust/rand/v0_8:lib", + "//brave/third_party/rust/shardtree/v0_3:lib", "//brave/third_party/rust/zcash_note_encryption/v0_4:lib", "//third_party/rust/byteorder/v1:lib", ] diff --git a/components/brave_wallet/browser/zcash/rust/Cargo.toml b/components/brave_wallet/browser/zcash/rust/Cargo.toml index 2e5090defb7c..cb3ad11d2952 100644 --- a/components/brave_wallet/browser/zcash/rust/Cargo.toml +++ b/components/brave_wallet/browser/zcash/rust/Cargo.toml @@ -10,7 +10,9 @@ cxx = { version = "1" } orchard = { version = "0.8.0", default-features = false } rand = "0.8" zcash_primitives = { version = "0.15.1", default-features = false } -zcash_note_encryption = "0.4" +zcash_note_encryption = "0.4" +zcash_client_backend = { version = "0.12.1", default-features = false } +shardtree = { version="0.3", features=["legacy-api"] } [lib] name = "zcash" diff --git a/components/brave_wallet/browser/zcash/rust/librustzcash/BUILD.gn b/components/brave_wallet/browser/zcash/rust/librustzcash/BUILD.gn index a1b9a61cfc74..9fba4ac46629 100644 --- a/components/brave_wallet/browser/zcash/rust/librustzcash/BUILD.gn +++ b/components/brave_wallet/browser/zcash/rust/librustzcash/BUILD.gn @@ -23,7 +23,10 @@ rust_static_library("zcash_protocol") { } rust_static_library("zcash_encoding") { - visibility = [ ":zcash_primitives" ] + visibility = [ + ":zcash_client_backend", + ":zcash_primitives", + ] crate_name = "zcash_encoding" crate_root = "src/components/zcash_encoding/src/lib.rs" sources = [ "src/components/zcash_encoding/src/lib.rs" ] @@ -36,6 +39,7 @@ rust_static_library("zcash_encoding") { rust_static_library("zcash_primitives") { visibility = [ + ":zcash_client_backend", "//brave/components/brave_wallet/browser/zcash/rust:rust_lib", "//brave/components/brave_wallet/browser/zcash/rust:rust_lib_cxx_generated", ] @@ -58,3 +62,23 @@ rust_static_library("zcash_primitives") { "//third_party/rust/byteorder/v1:lib", ] } + +rust_static_library("zcash_client_backend") { + visibility = [ + "//brave/components/brave_wallet/browser/zcash/rust:rust_lib", + "//brave/components/brave_wallet/browser/zcash/rust:rust_lib_cxx_generated", + ] + crate_name = "zcash_client_backend" + crate_root = "src/zcash_client_backend/src/lib.rs" + sources = [ + "src/zcash_client_backend/src/lib.rs", + "src/zcash_client_backend/src/serialization.rs", + "src/zcash_client_backend/src/serialization/shardtree.rs", + ] + deps = [ + ":zcash_encoding", + ":zcash_primitives", + "//brave/third_party/rust/byteorder/v1:lib", + "//brave/third_party/rust/shardtree/v0_3:lib", + ] +} diff --git a/script/brave_license_helper.py b/script/brave_license_helper.py index 0de1a8f16aa2..7842448eeba6 100644 --- a/script/brave_license_helper.py +++ b/script/brave_license_helper.py @@ -1,7 +1,7 @@ # Copyright (c) 2019 The Brave Authors. All rights reserved. # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at https://mozilla.org/MPL/2.0/. */ +# You can obtain one at https://mozilla.org/MPL/2.0/. import os import re @@ -60,6 +60,7 @@ def AddBraveCredits(root, prune_paths, special_cases, prune_dirs, os.path.join('brave', 'third_party', 'rust', 'anyhow'), os.path.join('brave', 'third_party', 'rust', 'base64'), os.path.join('brave', 'third_party', 'rust', 'bitflags'), + os.path.join('brave', 'third_party', 'rust', 'bitflags', 'v2'), os.path.join('brave', 'third_party', 'rust', 'byteorder', 'v1'), os.path.join('brave', 'third_party', 'rust', 'cfg_if'), os.path.join('brave', 'third_party', 'rust', 'cxx'), diff --git a/third_party/rust/chromium_crates_io/Cargo.lock b/third_party/rust/chromium_crates_io/Cargo.lock index db0f4efe4ba5..c9077f1540c2 100644 --- a/third_party/rust/chromium_crates_io/Cargo.lock +++ b/third_party/rust/chromium_crates_io/Cargo.lock @@ -2238,6 +2238,17 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shardtree" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 2.6.0", + "either", + "incrementalmerkletree", + "tracing", +] + [[package]] name = "signature" version = "2.2.0" @@ -2868,10 +2879,22 @@ dependencies = [ "cxx", "orchard", "rand 0.8.5", + "shardtree", + "zcash_client_backend", "zcash_note_encryption", "zcash_primitives", ] +[[package]] +name = "zcash_client_backend" +version = "0.12.1" +dependencies = [ + "byteorder", + "shardtree", + "zcash_encoding", + "zcash_primitives", +] + [[package]] name = "zcash_encoding" version = "0.2.0" diff --git a/third_party/rust/chromium_crates_io/Cargo.toml b/third_party/rust/chromium_crates_io/Cargo.toml index 14678e8b9d75..0f4ce84f7812 100644 --- a/third_party/rust/chromium_crates_io/Cargo.toml +++ b/third_party/rust/chromium_crates_io/Cargo.toml @@ -242,3 +242,8 @@ package = "zerocopy" [patch.crates-io.zerocopy_derive_v0_7] path = "../../../../third_party/rust/chromium_crates_io/vendor/zerocopy-derive-0.7.35" package = "zerocopy-derive" + +[patch.crates-io.zcash_client_backend_v0_12] +path = "../../../components/brave_wallet/browser/zcash/rust/librustzcash/src/zcash_client_backend" +package = "zcash_client_backend" + diff --git a/third_party/rust/chromium_crates_io/gnrt_config.toml b/third_party/rust/chromium_crates_io/gnrt_config.toml index ad316991e441..422f5a042a89 100644 --- a/third_party/rust/chromium_crates_io/gnrt_config.toml +++ b/third_party/rust/chromium_crates_io/gnrt_config.toml @@ -453,3 +453,6 @@ license_files = ['../../../../../common/licenses/Apache-2.0'] [crate.zcash_protocol] license_files = ['../../../../../common/licenses/Apache-2.0'] + +[crate.zcash_client_backend] +license_files = ['../../../../../common/licenses/Apache-2.0'] diff --git a/third_party/rust/chromium_crates_io/vendor/bitflags-2.6.0/Cargo.lock b/third_party/rust/chromium_crates_io/vendor/bitflags-2.6.0/Cargo.lock new file mode 100644 index 000000000000..5f98ce021923 --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/bitflags-2.6.0/Cargo.lock @@ -0,0 +1,383 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "bitflags" +version = "2.6.0" +dependencies = [ + "arbitrary", + "bytemuck", + "compiler_builtins", + "rustc-std-workspace-core", + "rustversion", + "serde", + "serde_derive", + "serde_json", + "serde_test", + "trybuild", + "zerocopy", +] + +[[package]] +name = "bytemuck" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee891b04274a59bd38b412188e24b849617b2e45a0fd8d057deb63e7403761b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "compiler_builtins" +version = "0.1.112" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15acab2bb4fe4dad1f1e31f3d9e714f50ef561a0f87dd8a9da004f14d455e1a" + +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustc-std-workspace-core" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1956f5517128a2b6f23ab2dadf1a976f4f5b27962e7724c2bf3d45e539ec098c" + +[[package]] +name = "rustversion" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "serde" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_spanned" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_test" +version = "1.0.176" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a2f49ace1498612d14f7e0b8245519584db8299541dfe31a06374a828d620ab" +dependencies = [ + "serde", +] + +[[package]] +name = "syn" +version = "2.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "toml" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + +[[package]] +name = "trybuild" +version = "1.0.96" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33a5f13f11071020bb12de7a16b925d2d58636175c20c11dc5f96cb64bb6c9b3" +dependencies = [ + "glob", + "serde", + "serde_derive", + "serde_json", + "termcolor", + "toml", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "winapi-util" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] +name = "winnow" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +dependencies = [ + "memchr", +] + +[[package]] +name = "zerocopy" +version = "0.7.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/.cargo-checksum.json b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/.cargo-checksum.json new file mode 100644 index 000000000000..697c9ce2fbb4 --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{}} diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/.cargo_vcs_info.json b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/.cargo_vcs_info.json new file mode 100644 index 000000000000..3696c4bd55ee --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "7c862c16b31aa3883fcbb61f30af9930eb01cc0b" + }, + "path_in_vcs": "shardtree" +} \ No newline at end of file diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/CHANGELOG.md b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/CHANGELOG.md new file mode 100644 index 000000000000..ec396f469eb3 --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/CHANGELOG.md @@ -0,0 +1,40 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to Rust's notion of +[Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## Unreleased + +## [0.3.1] - 2024-04-03 + +### Fixed +- Fixes a missing transitive dependency when using the `test-dependencies` feature flag. + +## [0.3.0] - 2024-03-25 + +### Added +- `ShardTree::{store, store_mut}` +- `ShardTree::insert_frontier` + +### Changed +- `shardtree::error::InsertionError` has new variant `MarkedRetentionInvalid` + +## [0.2.0] - 2023-11-07 + +### Added +- `ShardTree::{root_at_checkpoint_id, root_at_checkpoint_id_caching}` +- `ShardTree::{witness_at_checkpoint_id, witness_at_checkpoint_id_caching}` + +### Changed +- `ShardTree::root_at_checkpoint` and `ShardTree::root_at_checkpoint_caching` have + been renamed to `root_at_checkpoint_depth` and `root_at_checkpoint_depth_caching`, + respectively. +- `ShardTree::witness` and `ShardTree::witness_caching` have + been renamed to `witness_at_checkpoint_depth` and `witness_at_checkpoint_depth_caching`, + respectively. + +## [0.1.0] - 2023-09-08 + +Initial release! diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/Cargo.toml b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/Cargo.toml new file mode 100644 index 000000000000..f9b913eaf58d --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/Cargo.toml @@ -0,0 +1,73 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.60" +name = "shardtree" +version = "0.3.1" +authors = ["Kris Nuttycombe "] +description = "A space-efficient Merkle tree with witnessing of marked leaves, checkpointing & state restoration." +homepage = "https://github.com/zcash/incrementalmerkletree" +categories = [ + "algorithms", + "data-structures", +] +license = "MIT OR Apache-2.0" +repository = "https://github.com/zcash/incrementalmerkletree" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = [ + "--cfg", + "docsrs", +] + +[dependencies.assert_matches] +version = "1.5" +optional = true + +[dependencies.bitflags] +version = "2" + +[dependencies.either] +version = "1.8" + +[dependencies.incrementalmerkletree] +version = "0.5" + +[dependencies.proptest] +version = "1.0.0" +optional = true + +[dependencies.tracing] +version = "0.1" + +[dev-dependencies.assert_matches] +version = "1.5" + +[dev-dependencies.incrementalmerkletree] +version = "0.5" +features = ["test-dependencies"] + +[dev-dependencies.proptest] +version = "1.0.0" + +[features] +legacy-api = ["incrementalmerkletree/legacy-api"] +test-dependencies = [ + "proptest", + "assert_matches", + "incrementalmerkletree/test-dependencies", +] + +[target."cfg(unix)".dev-dependencies.tempfile] +version = ">=3, <3.7.0" diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/Cargo.toml.orig b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/Cargo.toml.orig new file mode 100644 index 000000000000..6bae130f4380 --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/Cargo.toml.orig @@ -0,0 +1,42 @@ +[package] +name = "shardtree" +version = "0.3.1" +authors = [ + "Kris Nuttycombe ", +] +edition = "2021" +rust-version = "1.60" +license = "MIT OR Apache-2.0" +description = "A space-efficient Merkle tree with witnessing of marked leaves, checkpointing & state restoration." +homepage = "https://github.com/zcash/incrementalmerkletree" +repository = "https://github.com/zcash/incrementalmerkletree" +categories = ["algorithms", "data-structures"] + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[dependencies] +assert_matches = { version = "1.5", optional = true } +bitflags = "2" +either = "1.8" +incrementalmerkletree = { version = "0.5", path = "../incrementalmerkletree" } +proptest = { version = "1.0.0", optional = true } +tracing = "0.1" + +[dev-dependencies] +assert_matches = "1.5" +incrementalmerkletree = { version = "0.5", path = "../incrementalmerkletree", features = ["test-dependencies"] } +proptest = "1.0.0" + +[features] +# The legacy-api feature guards types and functions that are useful for +# migrating data previously managed using `incrementalmerkletree/legacy-api` +# types into the `ShardTree` data structure. +legacy-api = ["incrementalmerkletree/legacy-api"] +# The test-depenencies feature can be enabled to expose types and functions +# that are useful for testing `shardtree` functionality. +test-dependencies = ["proptest", "assert_matches", "incrementalmerkletree/test-dependencies"] + +[target.'cfg(unix)'.dev-dependencies] +tempfile = ">=3, <3.7.0" # 3.7 has MSRV 1.63 diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/LICENSE-APACHE b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/LICENSE-APACHE new file mode 100644 index 000000000000..1e5006dc141e --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/LICENSE-APACHE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/LICENSE-MIT b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/LICENSE-MIT new file mode 100644 index 000000000000..94ac1a7cecaf --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/LICENSE-MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015-2021 The Electric Coin Company + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/README.md b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/README.md new file mode 100644 index 000000000000..8b1146702819 --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/README.md @@ -0,0 +1,57 @@ +# `shardtree` + +This is a Rust crate that provides an implementation of a fixed-depth Merkle +tree structure that is densely filled from the left. It supports: + +- *Out-of-order insertion*: leaves and nodes may be inserted into the tree in + arbitrary order. The structure will keep track of the right-most filled + position as the frontier of the tree; any unfilled leaves to the left of this + position are considered "missing", while any unfilled leaves to the right of + this position are considered "empty". +- *Witnessing*: Individual leaves of the Merkle tree may be marked such that + witnesses will be maintained for the marked leaves as additional nodes are + inserted into the tree, but leaf and node data not specifically required to + maintain these witnesses is not retained, for space efficiency. +- *Checkpointing*: the tree may be reset to a previously checkpointed state, up + to a fixed number of checkpoints. + +The tree is represented as an ordered collection of fixed-depth subtrees, or +"shards". The roots of the shards form the leaves in the "cap". + +``` +Level + 3 root \ + / \ | + / \ | + 2 / \ } cap + / \ / \ | + / \ / \ | + 1 A B C D / \ + / \ / \ / \ / \ } shards + 0 /\ /\ /\ /\ /\ /\ /\ /\ / +``` + +This structure enables witnesses for marked leaves to be advanced up to recent +checkpoints or the latest state of the tree, without having to insert each +intermediate leaf individually. Instead, only the roots of all complete shards +between the one containing the marked leaf and the tree frontier need to be +inserted, along with the necessary nodes to build a path from the marked leaf to +the root of the shard containing it. + +## [`Documentation`](https://docs.rs/shardtree) + +## License + +Licensed under either of + + * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or + http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall +be dual licensed as above, without any additional terms or conditions. diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/proptest-regressions/lib.txt b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/proptest-regressions/lib.txt new file mode 100644 index 000000000000..ae687508a666 --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/proptest-regressions/lib.txt @@ -0,0 +1,22 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 38b4ca3c029291dfe2a6b5907c33a2e8ae7900f19c759c5db74b616ab19f6c5c # shrinks to ops = [Append(SipHashable(0), MC), Rewind, Rewind] +cc f1a96f73b9f3ba2a2e4b5271037322150450c573b6c3a5ef34f71a540ff0fad2 # shrinks to ops = [Append("a", C), Rewind, Rewind] +cc ad5f5d4276adea6e928376c6dc8c013745e60a8b507e6fa4e716f6c35477fe65 # shrinks to ops = [Append("a", E), Append("a", E), Append("a", E), Append("a", E), Append("a", E), Append("a", E), Append("a", E), Append("a", E), Append("a", E)] +cc ab2690ff3cf593d2e7a78b2f56d76699e525391b87852bbf15315a1f36742f48 # shrinks to ops = [Append(SipHashable(0), C), Append(SipHashable(0), E), Append(SipHashable(0), E), Unmark(Position(0))] +cc 1603359084b0c614a2ff9008036a5e10db9f32fb31b3333e59aaa517686e174d # shrinks to ops = [Append(SipHashable(0), Checkpoint { id: (), is_marked: false })] +cc faaf929be4b27e652712b705e297bfe15c53767102516e56882d177ac6fc58d9 # shrinks to ops = [CurrentPosition, Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Checkpoint { id: (), is_marked: true }), Append("a", Marked), Checkpoint(()), Append("a", Checkpoint { id: (), is_marked: false }), Append("a", Checkpoint { id: (), is_marked: false }), Append("a", Checkpoint { id: (), is_marked: false }), Witness(Position(3), 5)] +cc 8836c27ead7afb8d10092bdaeed33eb31007eaa47e3c3fca248cc00fbce772a3 # shrinks to ops = [Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral)] +cc 2303f82f255c60d7bdd45e2d13ff526bdc1d7f5ce846a7c07b38ab7f255c0300 # shrinks to ops = [Append("a", Marked), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral)] +cc cf1a33ef6df58bbf7cc199b8b1879c3a078e7784aa3edc0aba9ca03772bea5f2 # shrinks to ops = [Append(SipHashable(0), Checkpoint { id: (), is_marked: false }), Append(SipHashable(0), Checkpoint { id: (), is_marked: false }), Append(SipHashable(0), Checkpoint { id: (), is_marked: false }), Checkpoint(()), Append(SipHashable(0), Checkpoint { id: (), is_marked: false }), Checkpoint(()), Rewind, Rewind, Rewind, Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Checkpoint { id: (), is_marked: false }), Append(SipHashable(0), Checkpoint { id: (), is_marked: false }), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Checkpoint { id: (), is_marked: true }), Witness(Position(8), 2)] +cc 544e027d994eaf7f97b1c8d9ee7b35522a64a610b1430d56d74ec947018b759d # shrinks to ops = [Append("a", Marked), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Append("a", Ephemeral), Checkpoint(()), Append("a", Marked), Checkpoint(()), Checkpoint(()), Append("a", Checkpoint { id: (), is_marked: false }), Rewind, Rewind, Witness(Position(7), 2)] +cc 55d00b68a0f0a02f83ab53f18a29d16d0233153b69a01414a1622104e0eead31 # shrinks to ops = [Append("a", Marked), Append("a", Checkpoint { id: (), is_marked: false }), Append("a", Marked), Checkpoint(()), Checkpoint(()), Checkpoint(()), Append("a", Checkpoint { id: (), is_marked: false }), Append("a", Checkpoint { id: (), is_marked: false }), Witness(Position(0), 7)] +cc 9dd966ff1ab66965c5b84153ae13f684258560cdd5e84c7deb24f724cb12aba7 # shrinks to ops = [Append("a", Marked), Append("a", Ephemeral), Append("a", Checkpoint { id: (), is_marked: true }), Checkpoint(()), Append("a", Checkpoint { id: (), is_marked: false }), Rewind, Rewind, Append("a", Checkpoint { id: (), is_marked: false }), Append("a", Checkpoint { id: (), is_marked: false }), Checkpoint(()), Witness(Position(2), 4)] +cc d53a73021238de143764ee1d48b944abb93bd4bc54f35d16e514261220d3eb78 # shrinks to ops = [Append(SipHashable(0), Marked), Unmark(Position(0))] +cc d9460b8acbc5b4d112cae5d9e2296fcd793999b2b2e1d5405722f2bd8d176c31 # shrinks to ops = [Append("a", Checkpoint { id: (), is_marked: true }), Rewind, Append("a", Ephemeral), Rewind, Unmark(Position(0))] +cc 644c7763bc7bdc65bd9e6eb156b3b1a9b0632571a571c462bd44f3e04a389ca0 # shrinks to ops = [Append("a", Ephemeral), Append("a", Checkpoint { id: (), is_marked: true }), Append("a", Ephemeral), Append("a", Ephemeral), Unmark(Position(1)), Witness(Position(1), 0)] +cc 12790169d3df4280dd155d9cdfa76719318b8ec97a80bd562b7cb182d4f9bc79 # shrinks to ops = [CurrentPosition, CurrentPosition, Append(SipHashable(0), Ephemeral), Append(SipHashable(0), Marked), Append(SipHashable(0), Ephemeral), Checkpoint(()), Checkpoint(()), Checkpoint(()), Unmark(Position(1)), Checkpoint(()), Witness(Position(1), 0)] diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/batch.rs b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/batch.rs new file mode 100644 index 000000000000..632cb5fa9256 --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/batch.rs @@ -0,0 +1,560 @@ +//! Helpers for inserting many leaves into a tree at once. + +use std::{collections::BTreeMap, fmt, ops::Range, sync::Arc}; + +use incrementalmerkletree::{Address, Hashable, Level, Position, Retention}; +use tracing::trace; + +use crate::{ + error::{InsertionError, ShardTreeError}, + store::{Checkpoint, ShardStore}, + IncompleteAt, LocatedPrunableTree, LocatedTree, Node, RetentionFlags, ShardTree, Tree, +}; + +impl< + H: Hashable + Clone + PartialEq, + C: Clone + fmt::Debug + Ord, + S: ShardStore, + const DEPTH: u8, + const SHARD_HEIGHT: u8, + > ShardTree +{ + /// Put a range of values into the subtree to fill leaves starting from the given position. + /// + /// This operation will pad the tree until it contains enough subtrees to reach the starting + /// position. It will fully consume the provided iterator, constructing successive subtrees + /// until no more values are available. It aggressively prunes the tree as it goes, retaining + /// only nodes that either have [`Retention::Marked`] retention, are required to construct a + /// witness for such marked nodes, or that must be retained in order to make it possible to + /// truncate the tree to any position with [`Retention::Checkpoint`] retention. + /// + /// This operation returns the final position at which a leaf was inserted, and the vector of + /// [`IncompleteAt`] values that identify addresses at which [`Node::Nil`] nodes were + /// introduced to the tree, as well as whether or not those newly introduced nodes will need to + /// be filled with values in order to produce witnesses for inserted leaves with + /// [`Retention::Marked`] retention. + /// + /// This method operates on a single thread. If you have parallelism available, consider using + /// [`LocatedPrunableTree::from_iter`] and [`Self::insert_tree`] instead. + #[allow(clippy::type_complexity)] + pub fn batch_insert)>>( + &mut self, + mut start: Position, + values: I, + ) -> Result)>, ShardTreeError> { + trace!("Batch inserting from {:?}", start); + let mut values = values.peekable(); + let mut subtree_root_addr = Self::subtree_addr(start); + let mut max_insert_position = None; + let mut all_incomplete = vec![]; + loop { + if values.peek().is_some() { + let mut res = self + .store + .get_shard(subtree_root_addr) + .map_err(ShardTreeError::Storage)? + .unwrap_or_else(|| LocatedTree::empty(subtree_root_addr)) + .batch_insert(start, values)? + .expect( + "Iterator containing leaf values to insert was verified to be nonempty.", + ); + self.store + .put_shard(res.subtree) + .map_err(ShardTreeError::Storage)?; + for (id, position) in res.checkpoints.into_iter() { + self.store + .add_checkpoint(id, Checkpoint::at_position(position)) + .map_err(ShardTreeError::Storage)?; + } + + values = res.remainder; + subtree_root_addr = subtree_root_addr.next_at_level(); + max_insert_position = res.max_insert_position; + start = max_insert_position.unwrap() + 1; + all_incomplete.append(&mut res.incomplete); + } else { + break; + } + } + + self.prune_excess_checkpoints()?; + Ok(max_insert_position.map(|p| (p, all_incomplete))) + } +} + +/// A type for the result of a batch insertion operation. +/// +/// This result type contains the newly constructed tree, the addresses any new incomplete internal +/// nodes within that tree that were introduced as a consequence of that insertion, and the +/// remainder of the iterator that provided the inserted values. +#[derive(Debug)] +pub struct BatchInsertionResult)>> { + /// The updated tree after all insertions have been performed. + pub subtree: LocatedPrunableTree, + /// A flag identifying whether the constructed subtree contains a marked node. + pub contains_marked: bool, + /// The vector of addresses of [`Node::Nil`] nodes that were inserted into the tree as part of + /// the insertion operation, for nodes that are required in order to construct a witness for + /// each inserted leaf with [`Retention::Marked`] retention. + pub incomplete: Vec, + /// The maximum position at which a leaf was inserted. + pub max_insert_position: Option, + /// The positions of all leaves with [`Retention::Checkpoint`] retention that were inserted. + pub checkpoints: BTreeMap, + /// The unconsumed remainder of the iterator from which leaves were inserted, if the tree + /// was completely filled before the iterator was fully consumed. + pub remainder: I, +} + +impl LocatedPrunableTree { + /// Append a values from an iterator, beginning at the first available position in the tree. + /// + /// Returns an error if the tree is full. If the position at the end of the iterator is outside + /// of the subtree's range, the unconsumed part of the iterator will be returned as part of + /// the result. + pub fn batch_append)>>( + &self, + values: I, + ) -> Result>, InsertionError> { + let append_position = self + .max_position() + .map(|p| p + 1) + .unwrap_or_else(|| self.root_addr.position_range_start()); + self.batch_insert(append_position, values) + } + + /// Put a range of values into the subtree by consuming the given iterator, starting at the + /// specified position. + /// + /// The start position must exist within the position range of this subtree. If the position at + /// the end of the iterator is outside of the subtree's range, the unconsumed part of the + /// iterator will be returned as part of the result. + /// + /// Returns `Ok(None)` if the provided iterator is empty, `Ok(Some(BatchInsertionResult))` if + /// values were successfully inserted, or an error if the start position provided is outside + /// of this tree's position range or if a conflict with an existing subtree root is detected. + pub fn batch_insert)>>( + &self, + start: Position, + values: I, + ) -> Result>, InsertionError> { + trace!("Batch inserting into {:?} from {:?}", self.root_addr, start); + let subtree_range = self.root_addr.position_range(); + let contains_start = subtree_range.contains(&start); + if contains_start { + let position_range = Range { + start, + end: subtree_range.end, + }; + Self::from_iter(position_range, self.root_addr.level(), values) + .map(|mut res| { + let (subtree, mut incomplete) = self + .clone() + .insert_subtree(res.subtree, res.contains_marked)?; + res.subtree = subtree; + res.incomplete.append(&mut incomplete); + Ok(res) + }) + .transpose() + } else { + Err(InsertionError::OutOfRange(start, subtree_range)) + } + } + + /// Builds a [`LocatedPrunableTree`] from an iterator of level-0 leaves. + /// + /// This may be used in conjunction with [`ShardTree::insert_tree`] to support + /// partially-parallelizable tree construction. Multiple subtrees may be constructed in + /// parallel from iterators over (preferably, though not necessarily) disjoint leaf ranges, and + /// [`ShardTree::insert_tree`] may be used to insert those subtrees into the [`ShardTree`] in + /// arbitrary order. + /// + /// # Parameters: + /// * `position_range` - The range of leaf positions at which values will be inserted. This + /// range is also used to place an upper bound on the number of items that will be consumed + /// from the `values` iterator. + /// * `prune_below` - Nodes with [`Retention::Ephemeral`] retention that are not required to be retained + /// in order to construct a witness for a marked node or to make it possible to rewind to a + /// checkpointed node may be pruned so long as their address is at less than the specified + /// level. + /// * `values` - The iterator of `(H, Retention)` pairs from which to construct the tree. + pub fn from_iter)>>( + position_range: Range, + prune_below: Level, + mut values: I, + ) -> Option> { + trace!( + position_range = ?position_range, + prune_below = ?prune_below, + "Creating minimal tree for insertion" + ); + + // A stack of complete subtrees to be inserted as descendants into the subtree labeled + // with the addresses at which they will be inserted, along with their root hashes. + let mut fragments: Vec<(Self, bool)> = vec![]; + let mut position = position_range.start; + let mut checkpoints: BTreeMap = BTreeMap::new(); + while position < position_range.end { + if let Some((value, retention)) = values.next() { + if let Retention::Checkpoint { id, .. } = &retention { + checkpoints.insert(id.clone(), position); + } + + let rflags = RetentionFlags::from(retention); + let mut subtree = LocatedTree { + root_addr: Address::from(position), + root: Tree(Node::Leaf { + value: (value.clone(), rflags), + }), + }; + + if position.is_right_child() { + // At right-hand positions, we are completing a subtree and so we unite + // fragments up the stack until we get the largest possible subtree + while let Some((potential_sibling, marked)) = fragments.pop() { + if potential_sibling.root_addr.parent() == subtree.root_addr.parent() { + subtree = unite(potential_sibling, subtree, prune_below); + } else { + // this is not a sibling node, so we push it back on to the stack + // and are done + fragments.push((potential_sibling, marked)); + break; + } + } + } + + fragments.push((subtree, rflags.is_marked())); + position += 1; + } else { + break; + } + } + trace!("Initial fragments: {:?}", fragments); + + if position > position_range.start { + let last_position = position - 1; + let minimal_tree_addr = + Address::from(position_range.start).common_ancestor(&last_position.into()); + trace!("Building minimal tree at {:?}", minimal_tree_addr); + build_minimal_tree(fragments, minimal_tree_addr, prune_below).map( + |(to_insert, contains_marked, incomplete)| BatchInsertionResult { + subtree: to_insert, + contains_marked, + incomplete, + max_insert_position: Some(last_position), + checkpoints, + remainder: values, + }, + ) + } else { + None + } + } +} + +// Unite two subtrees by either adding a parent node, or a leaf containing the Merkle root +// of such a parent if both nodes are ephemeral leaves. +// +// `unite` is only called when both root addrs have the same parent. `batch_insert` never +// constructs Nil nodes, so we don't create any incomplete root information here. +fn unite( + lroot: LocatedPrunableTree, + rroot: LocatedPrunableTree, + prune_below: Level, +) -> LocatedPrunableTree { + assert_eq!(lroot.root_addr.parent(), rroot.root_addr.parent()); + LocatedTree { + root_addr: lroot.root_addr.parent(), + root: if lroot.root_addr.level() < prune_below { + Tree::unite(lroot.root_addr.level(), None, lroot.root, rroot.root) + } else { + Tree(Node::Parent { + ann: None, + left: Arc::new(lroot.root), + right: Arc::new(rroot.root), + }) + }, + } +} + +/// Combines the given subtree with an empty sibling node to obtain the next level +/// subtree. +/// +/// `expect_left_child` is set to a constant at each callsite, to ensure that this +/// function is only called on either the left-most or right-most subtree. +fn combine_with_empty( + root: LocatedPrunableTree, + expect_left_child: bool, + incomplete: &mut Vec, + contains_marked: bool, + prune_below: Level, +) -> LocatedPrunableTree { + assert_eq!(expect_left_child, root.root_addr.is_left_child()); + let sibling_addr = root.root_addr.sibling(); + incomplete.push(IncompleteAt { + address: sibling_addr, + required_for_witness: contains_marked, + }); + let sibling = LocatedTree { + root_addr: sibling_addr, + root: Tree(Node::Nil), + }; + let (lroot, rroot) = if root.root_addr.is_left_child() { + (root, sibling) + } else { + (sibling, root) + }; + unite(lroot, rroot, prune_below) +} + +// Builds a single tree from the provided stack of subtrees, which must be non-overlapping +// and in position order. Returns the resulting tree, a flag indicating whether the +// resulting tree contains a `MARKED` node, and the vector of [`IncompleteAt`] values for +// [`Node::Nil`] nodes that were introduced in the process of constructing the tree. +fn build_minimal_tree( + mut xs: Vec<(LocatedPrunableTree, bool)>, + root_addr: Address, + prune_below: Level, +) -> Option<(LocatedPrunableTree, bool, Vec)> { + // First, consume the stack from the right, building up a single tree + // until we can't combine any more. + if let Some((mut cur, mut contains_marked)) = xs.pop() { + let mut incomplete = vec![]; + while let Some((top, top_marked)) = xs.pop() { + while cur.root_addr.level() < top.root_addr.level() { + cur = combine_with_empty(cur, true, &mut incomplete, top_marked, prune_below); + } + + if cur.root_addr.level() == top.root_addr.level() { + contains_marked = contains_marked || top_marked; + if cur.root_addr.is_right_child() { + // We have a left child and a right child, so unite them. + cur = unite(top, cur, prune_below); + } else { + // This is a left child, so we build it up one more level and then + // we've merged as much as we can from the right and need to work from + // the left + xs.push((top, top_marked)); + cur = combine_with_empty(cur, true, &mut incomplete, top_marked, prune_below); + break; + } + } else { + // top.root_addr.level < cur.root_addr.level, so we've merged as much as we + // can from the right and now need to work from the left. + xs.push((top, top_marked)); + break; + } + } + + // Ensure we can work from the left in a single pass by making this right-most subtree + while cur.root_addr.level() + 1 < root_addr.level() { + cur = combine_with_empty(cur, true, &mut incomplete, contains_marked, prune_below); + } + + // push our accumulated max-height right hand node back on to the stack. + xs.push((cur, contains_marked)); + + // From the stack of subtrees, construct a single sparse tree that can be + // inserted/merged into the existing tree + let res_tree = xs.into_iter().fold( + None, + |acc: Option>, (next_tree, next_marked)| { + if let Some(mut prev_tree) = acc { + // add nil branches to build up the left tree until we can merge it + // with the right + while prev_tree.root_addr.level() < next_tree.root_addr.level() { + contains_marked = contains_marked || next_marked; + prev_tree = combine_with_empty( + prev_tree, + false, + &mut incomplete, + next_marked, + prune_below, + ); + } + + Some(unite(prev_tree, next_tree, prune_below)) + } else { + Some(next_tree) + } + }, + ); + + res_tree.map(|t| (t, contains_marked, incomplete)) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use std::iter; + + use incrementalmerkletree::{Address, Level, Position, Retention}; + + use super::{LocatedPrunableTree, RetentionFlags}; + use crate::{ + store::memory::MemoryShardStore, + tree::tests::{leaf, nil, parent}, + BatchInsertionResult, ShardTree, + }; + + #[test] + fn located_from_iter_non_sibling_adjacent() { + let res = LocatedPrunableTree::from_iter::<(), _>( + Position::from(3)..Position::from(5), + Level::new(0), + vec![ + ("d".to_string(), Retention::Ephemeral), + ("e".to_string(), Retention::Ephemeral), + ] + .into_iter(), + ) + .unwrap(); + assert_eq!( + res.subtree, + LocatedPrunableTree { + root_addr: Address::from_parts(3.into(), 0), + root: parent( + parent( + nil(), + parent(nil(), leaf(("d".to_string(), RetentionFlags::EPHEMERAL))) + ), + parent( + parent(leaf(("e".to_string(), RetentionFlags::EPHEMERAL)), nil()), + nil() + ) + ) + }, + ); + } + + #[test] + fn located_insert() { + let tree = LocatedPrunableTree::empty(Address::from_parts(Level::from(2), 0)); + let (base, _, _) = tree + .append::<()>("a".to_string(), Retention::Ephemeral) + .unwrap(); + assert_eq!(base.right_filled_root(), Ok("a___".to_string())); + + // Perform an in-order insertion. + let (in_order, pos, _) = base + .append::<()>("b".to_string(), Retention::Ephemeral) + .unwrap(); + assert_eq!(pos, 1.into()); + assert_eq!(in_order.right_filled_root(), Ok("ab__".to_string())); + + // On the same tree, perform an out-of-order insertion. + let out_of_order = base + .batch_insert::<(), _>( + Position::from(3), + vec![("d".to_string(), Retention::Ephemeral)].into_iter(), + ) + .unwrap() + .unwrap(); + assert_eq!( + out_of_order.subtree, + LocatedPrunableTree { + root_addr: Address::from_parts(2.into(), 0), + root: parent( + parent(leaf(("a".to_string(), RetentionFlags::EPHEMERAL)), nil()), + parent(nil(), leaf(("d".to_string(), RetentionFlags::EPHEMERAL))) + ) + } + ); + + let complete = out_of_order + .subtree + .batch_insert::<(), _>( + Position::from(1), + vec![ + ("b".to_string(), Retention::Ephemeral), + ("c".to_string(), Retention::Ephemeral), + ] + .into_iter(), + ) + .unwrap() + .unwrap(); + assert_eq!(complete.subtree.right_filled_root(), Ok("abcd".to_string())); + } + + #[allow(clippy::type_complexity)] + pub(super) fn build_insert_tree_and_batch_insert( + leaves: Vec<(String, Retention)>, + ) -> ( + ShardTree, 6, 3>, + ShardTree, 6, 3>, + ) { + let max_checkpoints = 10; + let start = Position::from(0); + let end = start + leaves.len() as u64; + + // Construct a tree using `ShardTree::insert_tree`. + let mut left = ShardTree::new(MemoryShardStore::empty(), max_checkpoints); + if let Some(BatchInsertionResult { + subtree, + checkpoints, + mut remainder, + .. + }) = LocatedPrunableTree::from_iter(start..end, 0.into(), leaves.clone().into_iter()) + { + assert_eq!(remainder.next(), None); + left.insert_tree(subtree, checkpoints).unwrap(); + } + + // Construct a tree using `ShardTree::batch_insert`. + let mut right = ShardTree::new(MemoryShardStore::empty(), max_checkpoints); + right.batch_insert(start, leaves.into_iter()).unwrap(); + + (left, right) + } + + #[test] + fn batch_insert_matches_insert_tree() { + { + let (lhs, rhs) = build_insert_tree_and_batch_insert(vec![]); + assert_eq!(lhs.max_leaf_position(0), Ok(None)); + assert_eq!(rhs.max_leaf_position(0), Ok(None)); + } + + for i in 0..64 { + let num_leaves = i + 1; + let leaves = iter::repeat(("a".into(), Retention::Ephemeral)) + .take(num_leaves) + .collect(); + let expected_root = (0..64) + .map(|c| if c < num_leaves { 'a' } else { '_' }) + .fold(String::with_capacity(64), |mut acc, c| { + acc.push(c); + acc + }); + + let (lhs, rhs) = build_insert_tree_and_batch_insert(leaves); + assert_eq!(lhs.max_leaf_position(0), Ok(Some(Position::from(i as u64)))); + assert_eq!(rhs.max_leaf_position(0), Ok(Some(Position::from(i as u64)))); + + assert_eq!(lhs.root_at_checkpoint_depth(0).unwrap(), expected_root); + assert_eq!(rhs.root_at_checkpoint_depth(0).unwrap(), expected_root); + } + } +} + +#[cfg(test)] +mod proptests { + use proptest::prelude::*; + + use super::tests::build_insert_tree_and_batch_insert; + use crate::testing::{arb_char_str, arb_leaves}; + + proptest! { + #[test] + fn batch_insert_matches_insert_tree( + leaves in arb_leaves(arb_char_str()) + ) { + let (left, right) = build_insert_tree_and_batch_insert(leaves); + + // Check that the resulting trees are equal. + assert_eq!(left.root_at_checkpoint_depth(0), right.root_at_checkpoint_depth(0)); + } + } +} diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/error.rs b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/error.rs new file mode 100644 index 000000000000..2debfb718080 --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/error.rs @@ -0,0 +1,161 @@ +//! Error types for this crate. + +use std::fmt; +use std::ops::Range; + +use incrementalmerkletree::{Address, Position}; + +/// The error type for operations on a [`ShardTree`]. +/// +/// The parameter `S` is set to [`ShardStore::Error`]. +/// +/// [`ShardTree`]: crate::ShardTree +/// [`ShardStore::Error`]: crate::ShardStore::Error +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ShardTreeError { + Query(QueryError), + Insert(InsertionError), + Storage(S), +} + +impl From for ShardTreeError { + fn from(err: QueryError) -> Self { + ShardTreeError::Query(err) + } +} + +impl From for ShardTreeError { + fn from(err: InsertionError) -> Self { + ShardTreeError::Insert(err) + } +} + +impl fmt::Display for ShardTreeError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self { + ShardTreeError::Query(q) => q.fmt(f), + ShardTreeError::Insert(i) => i.fmt(f), + ShardTreeError::Storage(s) => { + write!( + f, + "An error occurred persisting or retrieving tree data: {}", + s + ) + } + } + } +} + +impl std::error::Error for ShardTreeError +where + SE: std::error::Error + 'static, +{ + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match &self { + ShardTreeError::Storage(e) => Some(e), + _ => None, + } + } +} + +/// Errors which can occur when inserting values into a subtree. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum InsertionError { + /// The caller attempted to insert a subtree into a tree that does not contain + /// the subtree's root address. + NotContained(Address), + /// The start of the range of positions provided for insertion is not included + /// in the range of positions within this subtree. + OutOfRange(Position, Range), + /// An existing root hash conflicts with the root hash of a node being inserted. + Conflict(Address), + /// An out-of-order checkpoint was detected + /// + /// Checkpoint identifiers must be in nondecreasing order relative to tree positions. + CheckpointOutOfOrder, + /// An append operation has exceeded the capacity of the tree. + TreeFull, + /// An input data structure had malformed data when attempting to insert a value + /// at the given address + InputMalformed(Address), + // The caller attempted to mark the empty tree state as corresponding to the state + // for a spendable note. + MarkedRetentionInvalid, +} + +impl fmt::Display for InsertionError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self { + InsertionError::NotContained(addr) => { + write!(f, "Tree does not contain a root at address {:?}", addr) + } + InsertionError::OutOfRange(p, r) => { + write!( + f, + "Attempted insertion point {:?} is not in range {:?}", + p, r + ) + } + InsertionError::Conflict(addr) => write!( + f, + "Inserted root conflicts with existing root at address {:?}", + addr + ), + InsertionError::CheckpointOutOfOrder => { + write!(f, "Cannot append out-of-order checkpoint identifier.") + } + InsertionError::TreeFull => write!(f, "Note commitment tree is full."), + InsertionError::InputMalformed(addr) => { + write!(f, "Input malformed for insertion at address {:?}", addr) + } + InsertionError::MarkedRetentionInvalid => { + write!(f, "Cannot use `Marked` retention for the empty tree.") + } + } + } +} + +impl std::error::Error for InsertionError {} + +/// Errors which can occur when querying a [`ShardTree`]. +/// +/// [`ShardTree`]: crate::ShardTree +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum QueryError { + /// The caller attempted to query the value at an address within a tree that does not contain + /// that address. + NotContained(Address), + /// A leaf required by a given checkpoint has been pruned, or is otherwise not accessible in + /// the tree. + CheckpointPruned, + /// It is not possible to compute a root for one or more subtrees because they contain + /// [`Node::Nil`] values at positions that cannot be replaced with default hashes. + /// + /// [`Node::Nil`]: crate::Node::Nil + TreeIncomplete(Vec
), +} + +impl fmt::Display for QueryError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match &self { + QueryError::NotContained(addr) => { + write!(f, "Tree does not contain a root at address {:?}", addr) + } + QueryError::CheckpointPruned => { + write!( + f, + "The leaf corresponding to the requested checkpoint is not present in the tree." + ) + } + QueryError::TreeIncomplete(addrs) => { + write!( + f, + "Unable to compute root; missing values for nodes {:?}", + addrs + ) + } + } + } +} + +impl std::error::Error for QueryError {} diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/legacy.rs b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/legacy.rs new file mode 100644 index 000000000000..39570619c4fd --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/legacy.rs @@ -0,0 +1,318 @@ +use std::fmt; + +use incrementalmerkletree::{witness::IncrementalWitness, Address, Hashable, Level, Retention}; + +use crate::{ + store::ShardStore, InsertionError, LocatedPrunableTree, LocatedTree, PrunableTree, + RetentionFlags, ShardTree, ShardTreeError, Tree, +}; + +impl< + H: Hashable + Clone + PartialEq, + C: Clone + fmt::Debug + Ord, + S: ShardStore, + const DEPTH: u8, + const SHARD_HEIGHT: u8, + > ShardTree +{ + /// Add the leaf and ommers of the provided witness as nodes within the subtree corresponding + /// to the frontier's position, and update the cap to include the nodes of the witness at + /// levels greater than or equal to the shard height. Also, if the witness spans multiple + /// subtrees, update the subtree corresponding to the current witness "tip" accordingly. + pub fn insert_witness_nodes( + &mut self, + witness: IncrementalWitness, + checkpoint_id: S::CheckpointId, + ) -> Result<(), ShardTreeError> { + let leaf_position = witness.witnessed_position(); + let subtree_root_addr = Address::above_position(Self::subtree_level(), leaf_position); + + let shard = self + .store + .get_shard(subtree_root_addr) + .map_err(ShardTreeError::Storage)? + .unwrap_or_else(|| LocatedTree::empty(subtree_root_addr)); + + let (updated_subtree, supertree, tip_subtree) = + shard.insert_witness_nodes(witness, checkpoint_id)?; + + self.store + .put_shard(updated_subtree) + .map_err(ShardTreeError::Storage)?; + + if let Some(supertree) = supertree { + let new_cap = LocatedTree { + root_addr: Self::root_addr(), + root: self.store.get_cap().map_err(ShardTreeError::Storage)?, + } + .insert_subtree(supertree, true)?; + + self.store + .put_cap(new_cap.0.root) + .map_err(ShardTreeError::Storage)?; + } + + if let Some(tip_subtree) = tip_subtree { + let tip_subtree_addr = Address::above_position( + Self::subtree_level(), + tip_subtree.root_addr().position_range_start(), + ); + + let tip_shard = self + .store + .get_shard(tip_subtree_addr) + .map_err(ShardTreeError::Storage)? + .unwrap_or_else(|| LocatedTree::empty(tip_subtree_addr)); + + self.store + .put_shard(tip_shard.insert_subtree(tip_subtree, false)?.0) + .map_err(ShardTreeError::Storage)?; + } + + Ok(()) + } +} + +/// Operations on [`LocatedTree`]s that are annotated with Merkle hashes. +impl LocatedPrunableTree { + fn combine_optional( + opt_t0: Option, + opt_t1: Option, + contains_marked: bool, + ) -> Result, InsertionError> { + match (opt_t0, opt_t1) { + (Some(t0), Some(t1)) => { + let into = LocatedTree { + root_addr: t0.root_addr().common_ancestor(&t1.root_addr()), + root: Tree::empty(), + }; + + into.insert_subtree(t0, contains_marked) + .and_then(|(into, _)| into.insert_subtree(t1, contains_marked)) + .map(|(t, _)| Some(t)) + } + (t0, t1) => Ok(t0.or(t1)), + } + } + + fn from_witness_filled_nodes( + leaf_addr: Address, + mut filled: impl Iterator, + split_at: Level, + ) -> (Self, Option) { + // add filled nodes to the subtree; here, we do not need to worry about + // whether or not these nodes can be invalidated by a rewind + let mut addr = leaf_addr; + let mut subtree = Tree::empty(); + while addr.level() < split_at { + if addr.is_left_child() { + // the current root is a left child, so take the right sibling from the + // filled iterator + if let Some(right) = filled.next() { + // once we have a right-hand node, add a parent with the current tree + // as the left-hand sibling + subtree = Tree::parent( + None, + subtree, + Tree::leaf((right.clone(), RetentionFlags::EPHEMERAL)), + ); + } else { + break; + } + } else { + // the current address is for a right child, so add an empty left sibling + subtree = Tree::parent(None, Tree::empty(), subtree); + } + + addr = addr.parent(); + } + + let subtree = LocatedTree { + root_addr: addr, + root: subtree, + }; + + // add filled nodes to the supertree + let supertree = if addr.level() == split_at { + let mut supertree = None; + for right in filled { + // build up the right-biased tree until we get a left-hand node + while addr.is_right_child() { + supertree = supertree.map(|t| Tree::parent(None, Tree::empty(), t)); + addr = addr.parent(); + } + + // once we have a left-hand root, add a parent with the current ommer as the right-hand sibling + supertree = Some(Tree::parent( + None, + supertree.unwrap_or_else(PrunableTree::empty), + Tree::leaf((right.clone(), RetentionFlags::EPHEMERAL)), + )); + addr = addr.parent(); + } + + supertree.map(|t| LocatedTree { + root_addr: addr, + root: t, + }) + } else { + None + }; + + (subtree, supertree) + } + + /// Insert the nodes belonging to the given incremental witness to this tree, truncating the + /// witness to the given position. + /// + /// Returns a copy of this tree updated to include the witness nodes, any partial supertree that is + /// produced from nodes "higher" in the witness tree + pub fn insert_witness_nodes( + &self, + witness: IncrementalWitness, + checkpoint_id: C, + ) -> Result<(Self, Option, Option), InsertionError> { + let subtree_range = self.root_addr.position_range(); + if subtree_range.contains(&witness.witnessed_position()) { + // construct the subtree and cap based on the frontier containing the + // witnessed position + let (past_subtree, past_supertree) = self.insert_frontier_nodes::( + witness.tree().to_frontier().take().unwrap(), + &Retention::Marked, + )?; + + // construct subtrees from the `filled` nodes of the witness + let (future_subtree, future_supertree) = Self::from_witness_filled_nodes( + Address::from(witness.witnessed_position()), + witness.filled().iter().cloned(), + self.root_addr.level(), + ); + + // construct subtrees from the `cursor` part of the witness + let cursor_trees = witness.cursor().as_ref().filter(|c| c.size() > 0).map(|c| { + Self::from_frontier_parts( + witness.tip_position(), + c.leaf() + .cloned() + .expect("Cannot have an empty leaf for a non-empty tree"), + c.ommers_iter().cloned(), + &Retention::Checkpoint { + id: checkpoint_id, + is_marked: false, + }, + self.root_addr.level(), + ) + }); + + let (subtree, _) = past_subtree.insert_subtree(future_subtree, true)?; + + let supertree = + LocatedPrunableTree::combine_optional(past_supertree, future_supertree, true)?; + + Ok(if let Some((cursor_sub, cursor_super)) = cursor_trees { + let (complete_subtree, fragment) = + if subtree.root_addr().contains(&cursor_sub.root_addr()) { + // the cursor subtree can be absorbed into the current subtree + (subtree.insert_subtree(cursor_sub, false)?.0, None) + } else { + // the cursor subtree must be maintained separately + (subtree, Some(cursor_sub)) + }; + + let complete_supertree = + LocatedPrunableTree::combine_optional(supertree, cursor_super, false)?; + + (complete_subtree, complete_supertree, fragment) + } else { + (subtree, supertree, None) + }) + } else { + Err(InsertionError::OutOfRange( + witness.witnessed_position(), + subtree_range, + )) + } + } +} + +#[cfg(test)] +mod tests { + use assert_matches::assert_matches; + + use incrementalmerkletree::{ + frontier::CommitmentTree, witness::IncrementalWitness, Address, Level, Position, + }; + + use crate::{LocatedPrunableTree, RetentionFlags, Tree}; + + #[test] + fn insert_witness_nodes() { + let mut base_tree = CommitmentTree::::empty(); + for c in 'a'..'h' { + base_tree.append(c.to_string()).unwrap(); + } + let mut witness = IncrementalWitness::from_tree(base_tree); + for c in 'h'..'z' { + witness.append(c.to_string()).unwrap(); + } + + let root_addr = Address::from_parts(Level::from(3), 0); + let tree = LocatedPrunableTree::empty(root_addr); + let result = tree.insert_witness_nodes(witness, 3usize); + assert_matches!(result, Ok((ref _t, Some(ref _c), Some(ref _r)))); + + if let Ok((t, Some(c), Some(r))) = result { + // verify that we can find the "marked" leaf + assert_eq!( + t.root.root_hash(root_addr, Position::from(7)), + Ok("abcdefg_".to_string()) + ); + + assert_eq!( + c.root, + Tree::parent( + None, + Tree::parent( + None, + Tree::empty(), + Tree::leaf(("ijklmnop".to_string(), RetentionFlags::EPHEMERAL)), + ), + Tree::parent( + None, + Tree::leaf(("qrstuvwx".to_string(), RetentionFlags::EPHEMERAL)), + Tree::empty() + ) + ) + ); + + assert_eq!( + r.root + .root_hash(Address::from_parts(Level::from(3), 3), Position::from(25)), + Ok("y_______".to_string()) + ); + } + } + + #[test] + fn insert_witness_nodes_sub_shard_height() { + let mut base_tree = CommitmentTree::::empty(); + for c in 'a'..='c' { + base_tree.append(c.to_string()).unwrap(); + } + let mut witness = IncrementalWitness::from_tree(base_tree); + witness.append("d".to_string()).unwrap(); + + let root_addr = Address::from_parts(Level::from(3), 0); + let tree = LocatedPrunableTree::empty(root_addr); + let result = tree.insert_witness_nodes(witness, 3usize); + assert_matches!(result, Ok((ref _t, None, None))); + + if let Ok((t, None, None)) = result { + // verify that we can find the "marked" leaf + assert_eq!( + t.root.root_hash(root_addr, Position::from(3)), + Ok("abc_____".to_string()) + ); + } + } +} diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/lib.rs b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/lib.rs new file mode 100644 index 000000000000..1f4510af26a6 --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/lib.rs @@ -0,0 +1,1547 @@ +//! `shardtree` is a space-efficient fixed-depth Merkle tree structure that is densely +//! filled from the left. It supports: +//! +//! - *Out-of-order insertion*: leaves and nodes may be inserted into the tree in +//! arbitrary order. The structure will keep track of the right-most filled position as +//! the frontier of the tree; any unfilled leaves to the left of this position are +//! considered "missing", while any unfilled leaves to the right of this position are +//! considered "empty". +//! - *Witnessing*: Individual leaves of the Merkle tree may be marked such that witnesses +//! will be maintained for the marked leaves as additional nodes are inserted into the +//! tree, but leaf and node data not specifically required to maintain these witnesses +//! is not retained, for space efficiency. +//! - *Checkpointing*: the tree may be reset to a previously checkpointed state, up to a +//! fixed number of checkpoints. +//! +//! Due to its structure (described in the [`store`] module), witnesses for marked leaves +//! can be advanced up to recent checkpoints or the latest state of the tree, without +//! having to insert each intermediate leaf individually. Instead, only the roots of all +//! complete shards between the one containing the marked leaf and the tree frontier need +//! to be inserted, along with the necessary nodes to build a path from the marked leaf to +//! the root of the shard containing it. + +#![cfg_attr(docsrs, feature(doc_cfg))] + +use core::fmt::Debug; +use either::Either; +use incrementalmerkletree::frontier::Frontier; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; +use tracing::trace; + +use incrementalmerkletree::{ + frontier::NonEmptyFrontier, Address, Hashable, Level, MerklePath, Position, Retention, +}; + +use self::{ + error::{InsertionError, QueryError, ShardTreeError}, + store::{Checkpoint, ShardStore, TreeState}, +}; + +mod batch; +pub use self::batch::BatchInsertionResult; + +mod tree; +pub use self::tree::{LocatedTree, Node, Tree}; + +mod prunable; +pub use self::prunable::{IncompleteAt, LocatedPrunableTree, PrunableTree, RetentionFlags}; + +pub mod error; +pub mod store; + +#[cfg(any(bench, test, feature = "test-dependencies"))] +pub mod testing; + +#[cfg(feature = "legacy-api")] +#[cfg_attr(docsrs, doc(cfg(feature = "legacy-api")))] +mod legacy; + +/// A sparse binary Merkle tree of the specified depth, represented as an ordered collection of +/// subtrees (shards) of a given maximum height. +/// +/// This tree maintains a collection of "checkpoints" which represent positions, usually near the +/// front of the tree, that are maintained such that it's possible to truncate nodes to the right +/// of the specified position. +#[derive(Debug)] +pub struct ShardTree { + /// The vector of tree shards. + store: S, + /// The maximum number of checkpoints to retain before pruning. + max_checkpoints: usize, +} + +impl< + H: Hashable + Clone + PartialEq, + C: Clone + Debug + Ord, + S: ShardStore, + const DEPTH: u8, + const SHARD_HEIGHT: u8, + > ShardTree +{ + /// Creates a new empty tree. + pub fn new(store: S, max_checkpoints: usize) -> Self { + Self { + store, + max_checkpoints, + } + } + + /// Consumes this tree and returns its underlying [`ShardStore`]. + pub fn into_store(self) -> S { + self.store + } + + /// Returns a reference to the underlying [`ShardStore`]. + pub fn store(&self) -> &S { + &self.store + } + + /// Returns a mutable reference to the underlying [`ShardStore`]. + pub fn store_mut(&mut self) -> &mut S { + &mut self.store + } + + /// Returns the root address of the tree. + pub fn root_addr() -> Address { + Address::from_parts(Level::from(DEPTH), 0) + } + + /// Returns the fixed level of subtree roots within the vector of subtrees used as this tree's + /// representation. + pub fn subtree_level() -> Level { + Level::from(SHARD_HEIGHT) + } + + /// Returns the root address of the subtree that contains the specified position. + pub fn subtree_addr(pos: Position) -> Address { + Address::above_position(Self::subtree_level(), pos) + } + + fn max_subtree_index() -> u64 { + (0x1 << (DEPTH - SHARD_HEIGHT)) - 1 + } + + /// Returns the leaf value at the specified position, if it is a marked leaf. + pub fn get_marked_leaf( + &self, + position: Position, + ) -> Result, ShardTreeError> { + Ok(self + .store + .get_shard(Self::subtree_addr(position)) + .map_err(ShardTreeError::Storage)? + .and_then(|t| t.value_at_position(position).cloned()) + .and_then(|(v, r)| if r.is_marked() { Some(v) } else { None })) + } + + /// Returns the positions of marked leaves in the tree. + pub fn marked_positions(&self) -> Result, ShardTreeError> { + let mut result = BTreeSet::new(); + for subtree_addr in &self + .store + .get_shard_roots() + .map_err(ShardTreeError::Storage)? + { + if let Some(subtree) = self + .store + .get_shard(*subtree_addr) + .map_err(ShardTreeError::Storage)? + { + result.append(&mut subtree.marked_positions()); + } + } + Ok(result) + } + + /// Inserts a new root into the tree at the given address. + /// + /// The level associated with the given address may not exceed `DEPTH`. + /// This will return an error if the specified hash conflicts with any existing annotation. + pub fn insert(&mut self, root_addr: Address, value: H) -> Result<(), ShardTreeError> { + if root_addr.level() > Self::root_addr().level() { + return Err(ShardTreeError::Insert(InsertionError::NotContained( + root_addr, + ))); + } + + let to_insert = LocatedTree { + root_addr, + root: Tree::leaf((value, RetentionFlags::EPHEMERAL)), + }; + + // The cap will retain nodes at the level of the shard roots or higher. + if root_addr.level() >= Self::subtree_level() { + let cap = LocatedTree { + root: self.store.get_cap().map_err(ShardTreeError::Storage)?, + root_addr: Self::root_addr(), + }; + + cap.insert_subtree(to_insert.clone(), false) + .map_err(ShardTreeError::Insert) + .and_then(|(updated_cap, _)| { + self.store + .put_cap(updated_cap.root) + .map_err(ShardTreeError::Storage) + })?; + } + + if let Either::Left(shard_root_addr) = root_addr.context(Self::subtree_level()) { + let shard = self + .store + .get_shard(shard_root_addr) + .map_err(ShardTreeError::Storage)? + .unwrap_or_else(|| LocatedTree { + root_addr: shard_root_addr, + root: Tree::empty(), + }); + + let updated_shard = shard + .insert_subtree(to_insert, false) + .map_err(ShardTreeError::Insert) + .map(|(t, _)| t)?; + + self.store + .put_shard(updated_shard) + .map_err(ShardTreeError::Storage)?; + } + + Ok(()) + } + + /// Append a single value at the first available position in the tree. + /// + /// Prefer to use [`Self::batch_insert`] when appending multiple values, as these operations + /// require fewer traversals of the tree than are necessary when performing multiple sequential + /// calls to [`Self::append`]. + pub fn append( + &mut self, + value: H, + retention: Retention, + ) -> Result<(), ShardTreeError> { + if let Retention::Checkpoint { id, .. } = &retention { + if self + .store + .max_checkpoint_id() + .map_err(ShardTreeError::Storage)? + .as_ref() + >= Some(id) + { + return Err(InsertionError::CheckpointOutOfOrder.into()); + } + } + + let (append_result, position, checkpoint_id) = + if let Some(subtree) = self.store.last_shard().map_err(ShardTreeError::Storage)? { + if subtree.root.is_complete() { + let addr = subtree.root_addr; + + if addr.index() < Self::max_subtree_index() { + LocatedTree::empty(addr.next_at_level()).append(value, retention)? + } else { + return Err(InsertionError::TreeFull.into()); + } + } else { + subtree.append(value, retention)? + } + } else { + let root_addr = Address::from_parts(Self::subtree_level(), 0); + LocatedTree::empty(root_addr).append(value, retention)? + }; + + self.store + .put_shard(append_result) + .map_err(ShardTreeError::Storage)?; + if let Some(c) = checkpoint_id { + self.store + .add_checkpoint(c, Checkpoint::at_position(position)) + .map_err(ShardTreeError::Storage)?; + } + + self.prune_excess_checkpoints()?; + + Ok(()) + } + + /// Add the leaf and ommers of the provided frontier to the tree. + /// + /// The leaf and ommers will be added as nodes within the subtree corresponding to the + /// frontier's position, and the tree state corresponding to that frontier will be marked as + /// specified by the leaf retention. + /// + /// This method may be used to add a checkpoint for the empty tree; note that + /// [`Retention::Marked`] is invalid for the empty tree. + pub fn insert_frontier( + &mut self, + frontier: Frontier, + leaf_retention: Retention, + ) -> Result<(), ShardTreeError> { + if let Some(nonempty_frontier) = frontier.take() { + self.insert_frontier_nodes(nonempty_frontier, leaf_retention) + } else { + match leaf_retention { + Retention::Ephemeral => Ok(()), + Retention::Checkpoint { + id, + is_marked: false, + } => self + .store + .add_checkpoint(id, Checkpoint::tree_empty()) + .map_err(ShardTreeError::Storage), + Retention::Checkpoint { + is_marked: true, .. + } + | Retention::Marked => Err(ShardTreeError::Insert( + InsertionError::MarkedRetentionInvalid, + )), + } + } + } + + /// Add the leaf and ommers of the provided frontier as nodes within the subtree corresponding + /// to the frontier's position, and update the cap to include the ommer nodes at levels greater + /// than or equal to the shard height. + pub fn insert_frontier_nodes( + &mut self, + frontier: NonEmptyFrontier, + leaf_retention: Retention, + ) -> Result<(), ShardTreeError> { + let leaf_position = frontier.position(); + let subtree_root_addr = Address::above_position(Self::subtree_level(), leaf_position); + trace!("Subtree containing nodes: {:?}", subtree_root_addr); + + let (updated_subtree, supertree) = self + .store + .get_shard(subtree_root_addr) + .map_err(ShardTreeError::Storage)? + .unwrap_or_else(|| LocatedTree::empty(subtree_root_addr)) + .insert_frontier_nodes(frontier, &leaf_retention)?; + + self.store + .put_shard(updated_subtree) + .map_err(ShardTreeError::Storage)?; + + if let Some(supertree) = supertree { + let new_cap = LocatedTree { + root_addr: Self::root_addr(), + root: self.store.get_cap().map_err(ShardTreeError::Storage)?, + } + .insert_subtree(supertree, leaf_retention.is_marked())?; + + self.store + .put_cap(new_cap.0.root) + .map_err(ShardTreeError::Storage)?; + } + + if let Retention::Checkpoint { id, is_marked: _ } = leaf_retention { + trace!("Adding checkpoint {:?} at {:?}", id, leaf_position); + self.store + .add_checkpoint(id, Checkpoint::at_position(leaf_position)) + .map_err(ShardTreeError::Storage)?; + } + + self.prune_excess_checkpoints()?; + Ok(()) + } + + /// Insert a tree by decomposing it into its `SHARD_HEIGHT` or smaller parts (if necessary) + /// and inserting those at their appropriate locations. + pub fn insert_tree( + &mut self, + tree: LocatedPrunableTree, + checkpoints: BTreeMap, + ) -> Result, ShardTreeError> { + let mut all_incomplete = vec![]; + for subtree in tree.decompose_to_level(Self::subtree_level()).into_iter() { + // `ShardTree::max_leaf_position` relies on the invariant that the last shard + // in the subtrees vector is never created without a leaf then being added to + // it. `LocatedTree::decompose_to_level` can return a trailing empty subtree + // for some inputs, and given that it is always correct to not insert an empty + // subtree into `self`, we maintain the invariant by skipping empty subtrees. + if subtree.root().is_empty() { + continue; + } + + // `LocatedTree::decompose_to_level` will return the tree as-is if it is + // smaller than a shard, so we can't assume that the address of `subtree` is a + // valid shard address. + let root_addr = Self::subtree_addr(subtree.root_addr.position_range_start()); + + let contains_marked = subtree.root.contains_marked(); + let (new_subtree, mut incomplete) = self + .store + .get_shard(root_addr) + .map_err(ShardTreeError::Storage)? + .unwrap_or_else(|| LocatedTree::empty(root_addr)) + .insert_subtree(subtree, contains_marked)?; + self.store + .put_shard(new_subtree) + .map_err(ShardTreeError::Storage)?; + all_incomplete.append(&mut incomplete); + } + + for (id, position) in checkpoints.into_iter() { + self.store + .add_checkpoint(id, Checkpoint::at_position(position)) + .map_err(ShardTreeError::Storage)?; + } + self.prune_excess_checkpoints()?; + + Ok(all_incomplete) + } + + /// Adds a checkpoint at the rightmost leaf state of the tree. + pub fn checkpoint(&mut self, checkpoint_id: C) -> Result> { + fn go( + root_addr: Address, + root: &PrunableTree, + ) -> Option<(PrunableTree, Position)> { + match root { + Tree(Node::Parent { ann, left, right }) => { + let (l_addr, r_addr) = root_addr.children().unwrap(); + go(r_addr, right).map_or_else( + || { + go(l_addr, left).map(|(new_left, pos)| { + ( + Tree::unite( + l_addr.level(), + ann.clone(), + new_left, + Tree(Node::Nil), + ), + pos, + ) + }) + }, + |(new_right, pos)| { + Some(( + Tree::unite( + l_addr.level(), + ann.clone(), + left.as_ref().clone(), + new_right, + ), + pos, + )) + }, + ) + } + Tree(Node::Leaf { value: (h, r) }) => Some(( + Tree(Node::Leaf { + value: (h.clone(), *r | RetentionFlags::CHECKPOINT), + }), + root_addr.max_position(), + )), + Tree(Node::Nil) => None, + } + } + + // checkpoint identifiers at the tip must be in increasing order + if self + .store + .max_checkpoint_id() + .map_err(ShardTreeError::Storage)? + .as_ref() + >= Some(&checkpoint_id) + { + return Ok(false); + } + + // Update the rightmost subtree to add the `CHECKPOINT` flag to the right-most leaf (which + // need not be a level-0 leaf; it's fine to rewind to a pruned state). + if let Some(subtree) = self.store.last_shard().map_err(ShardTreeError::Storage)? { + if let Some((replacement, pos)) = go(subtree.root_addr, &subtree.root) { + self.store + .put_shard(LocatedTree { + root_addr: subtree.root_addr, + root: replacement, + }) + .map_err(ShardTreeError::Storage)?; + self.store + .add_checkpoint(checkpoint_id, Checkpoint::at_position(pos)) + .map_err(ShardTreeError::Storage)?; + + // early return once we've updated the tree state + self.prune_excess_checkpoints()?; + return Ok(true); + } + } + + self.store + .add_checkpoint(checkpoint_id, Checkpoint::tree_empty()) + .map_err(ShardTreeError::Storage)?; + + // TODO: it should not be necessary to do this on every checkpoint, + // but currently that's how the reference tree behaves so we're maintaining + // those semantics for test compatibility. + self.prune_excess_checkpoints()?; + Ok(true) + } + + fn prune_excess_checkpoints(&mut self) -> Result<(), ShardTreeError> { + let checkpoint_count = self + .store + .checkpoint_count() + .map_err(ShardTreeError::Storage)?; + trace!( + "Tree has {} checkpoints, max is {}", + checkpoint_count, + self.max_checkpoints, + ); + if checkpoint_count > self.max_checkpoints { + // Batch removals by subtree & create a list of the checkpoint identifiers that + // will be removed from the checkpoints map. + let remove_count = checkpoint_count - self.max_checkpoints; + let mut checkpoints_to_delete = vec![]; + let mut clear_positions: BTreeMap> = + BTreeMap::new(); + self.store + .with_checkpoints(checkpoint_count, |cid, checkpoint| { + // When removing is true, we are iterating through the range of + // checkpoints being removed. When remove is false, we are + // iterating through the range of checkpoints that are being + // retained. + let removing = checkpoints_to_delete.len() < remove_count; + + if removing { + checkpoints_to_delete.push(cid.clone()); + } + + let mut clear_at = |pos, flags_to_clear| { + let subtree_addr = Self::subtree_addr(pos); + if removing { + // Mark flags to be cleared from the given position. + clear_positions + .entry(subtree_addr) + .and_modify(|to_clear| { + to_clear + .entry(pos) + .and_modify(|flags| *flags |= flags_to_clear) + .or_insert(flags_to_clear); + }) + .or_insert_with(|| BTreeMap::from([(pos, flags_to_clear)])); + } else { + // Unmark flags that might have been marked for clearing above + // but which we now know we need to preserve. + if let Some(to_clear) = clear_positions.get_mut(&subtree_addr) { + if let Some(flags) = to_clear.get_mut(&pos) { + *flags &= !flags_to_clear; + } + } + } + }; + + // Clear or preserve the checkpoint leaf. + if let TreeState::AtPosition(pos) = checkpoint.tree_state() { + clear_at(pos, RetentionFlags::CHECKPOINT) + } + + // Clear or preserve the leaves that have been marked for removal. + for unmark_pos in checkpoint.marks_removed().iter() { + clear_at(*unmark_pos, RetentionFlags::MARKED) + } + + Ok(()) + }) + .map_err(ShardTreeError::Storage)?; + + // Remove any nodes that are fully preserved by later checkpoints. + clear_positions.retain(|_, to_clear| { + to_clear.retain(|_, flags| !flags.is_empty()); + !to_clear.is_empty() + }); + + trace!( + "Removing checkpoints {:?}, pruning subtrees {:?}", + checkpoints_to_delete, + clear_positions, + ); + + // Prune each affected subtree + for (subtree_addr, positions) in clear_positions.into_iter() { + let cleared = self + .store + .get_shard(subtree_addr) + .map_err(ShardTreeError::Storage)? + .map(|subtree| subtree.clear_flags(positions)); + if let Some(cleared) = cleared { + self.store + .put_shard(cleared) + .map_err(ShardTreeError::Storage)?; + } + } + + // Now that the leaves have been pruned, actually remove the checkpoints + for c in checkpoints_to_delete { + self.store + .remove_checkpoint(&c) + .map_err(ShardTreeError::Storage)?; + } + } + + Ok(()) + } + + /// Truncates the tree, discarding all information after the checkpoint at the specified depth. + /// + /// This will also discard all checkpoints with depth less than or equal to the specified depth. + /// Returns `true` if the truncation succeeds or has no effect, or `false` if no checkpoint + /// exists at the specified depth. + pub fn truncate_to_depth( + &mut self, + checkpoint_depth: usize, + ) -> Result> { + if checkpoint_depth == 0 { + Ok(true) + } else if let Some((checkpoint_id, c)) = self + .store + .get_checkpoint_at_depth(checkpoint_depth) + .map_err(ShardTreeError::Storage)? + { + self.truncate_removing_checkpoint_internal(&checkpoint_id, &c)?; + Ok(true) + } else { + Ok(false) + } + } + + /// Truncates the tree, discarding all information after the specified checkpoint. + /// + /// This will also discard all checkpoints with depth less than or equal to the specified depth. + /// Returns `true` if the truncation succeeds or has no effect, or `false` if no checkpoint + /// exists for the specified checkpoint identifier. + pub fn truncate_removing_checkpoint( + &mut self, + checkpoint_id: &C, + ) -> Result> { + if let Some(c) = self + .store + .get_checkpoint(checkpoint_id) + .map_err(ShardTreeError::Storage)? + { + self.truncate_removing_checkpoint_internal(checkpoint_id, &c)?; + Ok(true) + } else { + Ok(false) + } + } + + fn truncate_removing_checkpoint_internal( + &mut self, + checkpoint_id: &C, + checkpoint: &Checkpoint, + ) -> Result<(), ShardTreeError> { + match checkpoint.tree_state() { + TreeState::Empty => { + self.store + .truncate(Address::from_parts(Self::subtree_level(), 0)) + .map_err(ShardTreeError::Storage)?; + self.store + .truncate_checkpoints(checkpoint_id) + .map_err(ShardTreeError::Storage)?; + self.store + .put_cap(Tree::empty()) + .map_err(ShardTreeError::Storage)?; + } + TreeState::AtPosition(position) => { + let subtree_addr = Self::subtree_addr(position); + let replacement = self + .store + .get_shard(subtree_addr) + .map_err(ShardTreeError::Storage)? + .and_then(|s| s.truncate_to_position(position)); + + let cap_tree = LocatedTree { + root_addr: Self::root_addr(), + root: self.store.get_cap().map_err(ShardTreeError::Storage)?, + }; + + if let Some(truncated) = cap_tree.truncate_to_position(position) { + self.store + .put_cap(truncated.root) + .map_err(ShardTreeError::Storage)?; + }; + + if let Some(truncated) = replacement { + self.store + .truncate(subtree_addr) + .map_err(ShardTreeError::Storage)?; + self.store + .put_shard(truncated) + .map_err(ShardTreeError::Storage)?; + self.store + .truncate_checkpoints(checkpoint_id) + .map_err(ShardTreeError::Storage)?; + } + } + } + + Ok(()) + } + + /// Computes the root of any subtree of this tree rooted at the given address, with the overall + /// tree truncated to the specified position. + /// + /// The specified address is not required to be at any particular level, though it cannot + /// exceed the level corresponding to the maximum depth of the tree. Nodes to the right of the + /// given position, and parents of such nodes, will be replaced by the empty root for the + /// associated level. + /// + /// Use [`Self::root_at_checkpoint_depth`] to obtain the root of the overall tree. + pub fn root( + &self, + address: Address, + truncate_at: Position, + ) -> Result> { + assert!(Self::root_addr().contains(&address)); + + // traverse the cap from root to leaf depth-first, either returning an existing + // cached value for the node or inserting the computed value into the cache + let (root, _) = self.root_internal( + &LocatedPrunableTree { + root_addr: Self::root_addr(), + root: self.store.get_cap().map_err(ShardTreeError::Storage)?, + }, + address, + truncate_at, + )?; + Ok(root) + } + + pub fn root_caching( + &mut self, + address: Address, + truncate_at: Position, + ) -> Result> { + let (root, updated_cap) = self.root_internal( + &LocatedPrunableTree { + root_addr: Self::root_addr(), + root: self.store.get_cap().map_err(ShardTreeError::Storage)?, + }, + address, + truncate_at, + )?; + if let Some(updated_cap) = updated_cap { + self.store + .put_cap(updated_cap) + .map_err(ShardTreeError::Storage)?; + } + Ok(root) + } + + // compute the root, along with an optional update to the cap + #[allow(clippy::type_complexity)] + fn root_internal( + &self, + cap: &LocatedPrunableTree, + // The address at which we want to compute the root hash + target_addr: Address, + // An inclusive lower bound for positions whose leaf values will be replaced by empty + // roots. + truncate_at: Position, + ) -> Result<(H, Option>), ShardTreeError> { + match &cap.root { + Tree(Node::Parent { ann, left, right }) => { + match ann { + Some(cached_root) if target_addr.contains(&cap.root_addr) => { + Ok((cached_root.as_ref().clone(), None)) + } + _ => { + // Compute the roots of the left and right children and hash them together. + // We skip computation in any subtrees that will not have data included in + // the final result. + let (l_addr, r_addr) = cap.root_addr.children().unwrap(); + let l_result = if r_addr.contains(&target_addr) { + None + } else { + Some(self.root_internal( + &LocatedPrunableTree { + root_addr: l_addr, + root: left.as_ref().clone(), + }, + if l_addr.contains(&target_addr) { + target_addr + } else { + l_addr + }, + truncate_at, + )?) + }; + let r_result = if l_addr.contains(&target_addr) { + None + } else { + Some(self.root_internal( + &LocatedPrunableTree { + root_addr: r_addr, + root: right.as_ref().clone(), + }, + if r_addr.contains(&target_addr) { + target_addr + } else { + r_addr + }, + truncate_at, + )?) + }; + + // Compute the root value based on the child roots; these may contain the + // hashes of empty/truncated nodes. + let (root, new_left, new_right) = match (l_result, r_result) { + (Some((l_root, new_left)), Some((r_root, new_right))) => ( + S::H::combine(l_addr.level(), &l_root, &r_root), + new_left, + new_right, + ), + (Some((l_root, new_left)), None) => (l_root, new_left, None), + (None, Some((r_root, new_right))) => (r_root, None, new_right), + (None, None) => unreachable!(), + }; + + let new_parent = Tree(Node::Parent { + ann: new_left + .as_ref() + .and_then(|l| l.node_value()) + .zip(new_right.as_ref().and_then(|r| r.node_value())) + .map(|(l, r)| { + // the node values of child nodes cannot contain the hashes of + // empty nodes or nodes with positions greater than the + Arc::new(S::H::combine(l_addr.level(), l, r)) + }), + left: new_left.map_or_else(|| left.clone(), Arc::new), + right: new_right.map_or_else(|| right.clone(), Arc::new), + }); + + Ok((root, Some(new_parent))) + } + } + } + Tree(Node::Leaf { value }) => { + if truncate_at >= cap.root_addr.position_range_end() + && target_addr.contains(&cap.root_addr) + { + // no truncation or computation of child subtrees of this leaf is necessary, just use + // the cached leaf value + Ok((value.0.clone(), None)) + } else { + // since the tree was truncated below this level, recursively call with an + // empty parent node to trigger the continued traversal + let (root, replacement) = self.root_internal( + &LocatedPrunableTree { + root_addr: cap.root_addr(), + root: Tree::parent(None, Tree::empty(), Tree::empty()), + }, + target_addr, + truncate_at, + )?; + + Ok(( + root, + replacement.map(|r| r.reannotate_root(Some(Arc::new(value.0.clone())))), + )) + } + } + Tree(Node::Nil) => { + if cap.root_addr == target_addr + || cap.root_addr.level() == ShardTree::::subtree_level() + { + // We are at the leaf level or the target address; compute the root hash and + // return it as cacheable if it is not truncated. + let root = self.root_from_shards(target_addr, truncate_at)?; + Ok(( + root.clone(), + if truncate_at >= cap.root_addr.position_range_end() { + // return the compute root as a new leaf to be cached if it contains no + // empty hashes due to truncation + Some(Tree::leaf((root, RetentionFlags::EPHEMERAL))) + } else { + None + }, + )) + } else { + // Compute the result by recursively walking down the tree. By replacing + // the current node with a parent node, the `Parent` handler will take care + // of the branching recursive calls. + self.root_internal( + &LocatedPrunableTree { + root_addr: cap.root_addr, + root: Tree::parent(None, Tree::empty(), Tree::empty()), + }, + target_addr, + truncate_at, + ) + } + } + } + } + + fn root_from_shards( + &self, + address: Address, + truncate_at: Position, + ) -> Result> { + match address.context(Self::subtree_level()) { + Either::Left(subtree_addr) => { + // The requested root address is fully contained within one of the subtrees. + Ok(if truncate_at <= address.position_range_start() { + H::empty_root(address.level()) + } else { + // get the child of the subtree with its root at `address` + self.store + .get_shard(subtree_addr) + .map_err(ShardTreeError::Storage)? + .ok_or_else(|| vec![subtree_addr]) + .and_then(|subtree| { + subtree.subtree(address).map_or_else( + || Err(vec![address]), + |child| child.root_hash(truncate_at), + ) + }) + .map_err(QueryError::TreeIncomplete)? + }) + } + Either::Right(subtree_range) => { + // The requested root requires hashing together the roots of several subtrees. + let mut root_stack = vec![]; + let mut incomplete = vec![]; + + for subtree_idx in subtree_range { + let subtree_addr = Address::from_parts(Self::subtree_level(), subtree_idx); + if truncate_at <= subtree_addr.position_range_start() { + break; + } + + let subtree_root = self + .store + .get_shard(subtree_addr) + .map_err(ShardTreeError::Storage)? + .ok_or_else(|| vec![subtree_addr]) + .and_then(|s| s.root_hash(truncate_at)); + + match subtree_root { + Ok(mut cur_hash) => { + if subtree_addr.index() % 2 == 0 { + root_stack.push((subtree_addr, cur_hash)) + } else { + let mut cur_addr = subtree_addr; + while let Some((addr, hash)) = root_stack.pop() { + if addr.parent() == cur_addr.parent() { + cur_hash = H::combine(cur_addr.level(), &hash, &cur_hash); + cur_addr = cur_addr.parent(); + } else { + root_stack.push((addr, hash)); + break; + } + } + root_stack.push((cur_addr, cur_hash)); + } + } + Err(mut new_incomplete) => { + // Accumulate incomplete root information and continue, so that we can + // return the complete set of incomplete results. + incomplete.append(&mut new_incomplete); + } + } + } + + if !incomplete.is_empty() { + return Err(ShardTreeError::Query(QueryError::TreeIncomplete( + incomplete, + ))); + } + + // Now hash with empty roots to obtain the root at maximum height + if let Some((mut cur_addr, mut cur_hash)) = root_stack.pop() { + while let Some((addr, hash)) = root_stack.pop() { + while addr.level() > cur_addr.level() { + cur_hash = H::combine( + cur_addr.level(), + &cur_hash, + &H::empty_root(cur_addr.level()), + ); + cur_addr = cur_addr.parent(); + } + cur_hash = H::combine(cur_addr.level(), &hash, &cur_hash); + cur_addr = cur_addr.parent(); + } + + while cur_addr.level() < address.level() { + cur_hash = H::combine( + cur_addr.level(), + &cur_hash, + &H::empty_root(cur_addr.level()), + ); + cur_addr = cur_addr.parent(); + } + + Ok(cur_hash) + } else { + // if the stack is empty, we just return the default root at max height + Ok(H::empty_root(address.level())) + } + } + } + } + + /// Returns the position of the rightmost leaf inserted as of the given checkpoint. + /// + /// Returns the maximum leaf position if `checkpoint_depth == 0` (or `Ok(None)` in this + /// case if the tree is empty) or an error if the checkpointed position cannot be restored + /// because it has been pruned. Note that no actual level-0 leaf may exist at this position. + pub fn max_leaf_position( + &self, + checkpoint_depth: usize, + ) -> Result, ShardTreeError> { + Ok(if checkpoint_depth == 0 { + // TODO: This relies on the invariant that the last shard in the subtrees vector is + // never created without a leaf then being added to it. However, this may be a + // difficult invariant to maintain when adding empty roots, so perhaps we need a + // better way of tracking the actual max position of the tree; we might want to + // just store it directly. + self.store + .last_shard() + .map_err(ShardTreeError::Storage)? + .and_then(|t| t.max_position()) + } else { + match self + .store + .get_checkpoint_at_depth(checkpoint_depth) + .map_err(ShardTreeError::Storage)? + { + Some((_, c)) => Ok(c.position()), + None => { + // There is no checkpoint at the specified depth, so we report it as pruned. + Err(QueryError::CheckpointPruned) + } + }? + }) + } + + /// Computes the root of the tree as of the checkpointed position having the specified + /// checkpoint id. + pub fn root_at_checkpoint_id(&self, checkpoint: &C) -> Result> { + let position = self + .store + .get_checkpoint(checkpoint) + .map_err(ShardTreeError::Storage)? + .map(|c| c.position()) + .ok_or(QueryError::CheckpointPruned)?; + + position.map_or_else( + || Ok(H::empty_root(Self::root_addr().level())), + |pos| self.root(Self::root_addr(), pos + 1), + ) + } + + /// Computes the root of the tree as of the checkpointed position having the specified + /// checkpoint id, caching intermediate values produced while computing the root. + pub fn root_at_checkpoint_id_caching( + &mut self, + checkpoint: &C, + ) -> Result> { + let position = self + .store + .get_checkpoint(checkpoint) + .map_err(ShardTreeError::Storage)? + .map(|c| c.position()) + .ok_or(QueryError::CheckpointPruned)?; + + position.map_or_else( + || Ok(H::empty_root(Self::root_addr().level())), + |pos| self.root_caching(Self::root_addr(), pos + 1), + ) + } + + /// Computes the root of the tree as of the checkpointed position at the specified depth. + /// + /// Returns the root as of the most recently appended leaf if `checkpoint_depth == 0`. Note + /// that if the most recently appended leaf is also a checkpoint, this will return the same + /// result as `checkpoint_depth == 1`. + pub fn root_at_checkpoint_depth( + &self, + checkpoint_depth: usize, + ) -> Result> { + self.max_leaf_position(checkpoint_depth)?.map_or_else( + || Ok(H::empty_root(Self::root_addr().level())), + |pos| self.root(Self::root_addr(), pos + 1), + ) + } + + /// Computes the root of the tree as of the checkpointed position at the specified depth, + /// caching intermediate values produced while computing the root. + pub fn root_at_checkpoint_depth_caching( + &mut self, + checkpoint_depth: usize, + ) -> Result> { + self.max_leaf_position(checkpoint_depth)?.map_or_else( + || Ok(H::empty_root(Self::root_addr().level())), + |pos| self.root_caching(Self::root_addr(), pos + 1), + ) + } + + fn witness_helper( + mut ctx: Ctx, + position: Position, + as_of: Position, + get_shard: impl Fn(&Ctx, Address) -> Result>, S::Error>, + mut root: impl FnMut(&mut Ctx, Address, Position) -> Result>, + ) -> Result, ShardTreeError> { + let subtree_addr = Self::subtree_addr(position); + + // compute the witness for the specified position up to the subtree root + let mut witness = get_shard(&ctx, subtree_addr) + .map_err(ShardTreeError::Storage)? + .map_or_else( + || Err(QueryError::TreeIncomplete(vec![subtree_addr])), + |subtree| subtree.witness(position, as_of + 1), + )?; + + // compute the remaining parts of the witness up to the root + let root_addr = Self::root_addr(); + let mut cur_addr = subtree_addr; + while cur_addr != root_addr { + witness.push(root(&mut ctx, cur_addr.sibling(), as_of + 1)?); + cur_addr = cur_addr.parent(); + } + + Ok(MerklePath::from_parts(witness, position).unwrap()) + } + + fn witness_internal( + &self, + position: Position, + as_of: Position, + ) -> Result, ShardTreeError> { + Self::witness_helper( + self, + position, + as_of, + |ctx, shard_root| ctx.store.get_shard(shard_root), + |ctx, address, truncate_at| ctx.root(address, truncate_at), + ) + } + + fn witness_internal_caching( + &mut self, + position: Position, + as_of: Position, + ) -> Result, ShardTreeError> { + Self::witness_helper( + self, + position, + as_of, + |ctx, shard_root| ctx.store.get_shard(shard_root), + |ctx, address, truncate_at| ctx.root_caching(address, truncate_at), + ) + } + + /// Computes the witness for the leaf at the specified position, as of the given checkpoint + /// depth. + /// + /// Returns the witness as of the most recently appended leaf if `checkpoint_depth == 0`. Note + /// that if the most recently appended leaf is also a checkpoint, this will return the same + /// result as `checkpoint_depth == 1`. + pub fn witness_at_checkpoint_depth( + &self, + position: Position, + checkpoint_depth: usize, + ) -> Result, ShardTreeError> { + let as_of = self.max_leaf_position(checkpoint_depth).and_then(|v| { + v.ok_or_else(|| QueryError::TreeIncomplete(vec![Self::root_addr()]).into()) + })?; + + if position > as_of { + Err( + QueryError::NotContained(Address::from_parts(Level::from(0), position.into())) + .into(), + ) + } else { + self.witness_internal(position, as_of) + } + } + + /// Computes the witness for the leaf at the specified position, as of the given checkpoint + /// depth. + /// + /// This implementation will mutate the tree to cache intermediate root (ommer) values that are + /// computed in the process of constructing the witness, so as to avoid the need to recompute + /// those values from potentially large numbers of subtree roots in the future. + pub fn witness_at_checkpoint_depth_caching( + &mut self, + position: Position, + checkpoint_depth: usize, + ) -> Result, ShardTreeError> { + let as_of = self.max_leaf_position(checkpoint_depth).and_then(|v| { + v.ok_or_else(|| QueryError::TreeIncomplete(vec![Self::root_addr()]).into()) + })?; + + if position > as_of { + Err( + QueryError::NotContained(Address::from_parts(Level::from(0), position.into())) + .into(), + ) + } else { + self.witness_internal_caching(position, as_of) + } + } + + /// Computes the witness for the leaf at the specified position, as of the given checkpoint. + pub fn witness_at_checkpoint_id( + &self, + position: Position, + checkpoint_id: &C, + ) -> Result, ShardTreeError> { + let as_of = self + .store + .get_checkpoint(checkpoint_id) + .map_err(ShardTreeError::Storage)? + .and_then(|c| c.position()) + .ok_or(QueryError::CheckpointPruned)?; + + self.witness_internal(position, as_of) + } + + /// Computes the witness for the leaf at the specified position, as of the given checkpoint. + /// + /// This implementation will mutate the tree to cache intermediate root (ommer) values that are + /// computed in the process of constructing the witness, so as to avoid the need to recompute + /// those values from potentially large numbers of subtree roots in the future. + pub fn witness_at_checkpoint_id_caching( + &mut self, + position: Position, + checkpoint_id: &C, + ) -> Result, ShardTreeError> { + let as_of = self + .store + .get_checkpoint(checkpoint_id) + .map_err(ShardTreeError::Storage)? + .and_then(|c| c.position()) + .ok_or(QueryError::CheckpointPruned)?; + + self.witness_internal_caching(position, as_of) + } + + /// Make a marked leaf at a position eligible to be pruned. + /// + /// If the checkpoint associated with the specified identifier does not exist because the + /// corresponding checkpoint would have been more than `max_checkpoints` deep, the removal is + /// recorded as of the first existing checkpoint and the associated leaves will be pruned when + /// that checkpoint is subsequently removed. + /// + /// Returns `Ok(true)` if a mark was successfully removed from the leaf at the specified + /// position, `Ok(false)` if the tree does not contain a leaf at the specified position or is + /// not marked, or an error if one is produced by the underlying data store. + pub fn remove_mark( + &mut self, + position: Position, + as_of_checkpoint: Option<&C>, + ) -> Result> { + match self + .store + .get_shard(Self::subtree_addr(position)) + .map_err(ShardTreeError::Storage)? + { + Some(shard) + if shard + .value_at_position(position) + .iter() + .any(|(_, r)| r.is_marked()) => + { + match as_of_checkpoint { + Some(cid) + if Some(cid) + >= self + .store + .min_checkpoint_id() + .map_err(ShardTreeError::Storage)? + .as_ref() => + { + self.store + .update_checkpoint_with(cid, |checkpoint| { + checkpoint.mark_removed(position); + Ok(()) + }) + .map_err(ShardTreeError::Storage) + } + _ => { + // if no checkpoint was provided, or if the checkpoint is too far in the past, + // remove the mark directly. + self.store + .put_shard( + shard.clear_flags(BTreeMap::from([( + position, + RetentionFlags::MARKED, + )])), + ) + .map_err(ShardTreeError::Storage)?; + Ok(true) + } + } + } + _ => Ok(false), + } + } +} + +#[cfg(test)] +mod tests { + use assert_matches::assert_matches; + use proptest::prelude::*; + + use incrementalmerkletree::{ + frontier::NonEmptyFrontier, + testing::{ + arb_operation, check_append, check_checkpoint_rewind, check_operations, + check_remove_mark, check_rewind_remove_mark, check_root_hashes, + check_witness_consistency, check_witnesses, complete_tree::CompleteTree, CombinedTree, + SipHashable, + }, + Address, Hashable, Level, Position, Retention, + }; + + use crate::{ + store::memory::MemoryShardStore, + testing::{ + arb_char_str, arb_shardtree, check_shard_sizes, check_shardtree_insertion, + check_witness_with_pruned_subtrees, + }, + InsertionError, LocatedPrunableTree, ShardTree, + }; + + #[test] + fn shardtree_insertion() { + let tree: ShardTree, 4, 3> = + ShardTree::new(MemoryShardStore::empty(), 100); + + check_shardtree_insertion(tree) + } + + #[test] + fn shard_sizes() { + let tree: ShardTree, 4, 2> = + ShardTree::new(MemoryShardStore::empty(), 100); + + check_shard_sizes(tree) + } + + #[test] + fn witness_with_pruned_subtrees() { + let tree: ShardTree, 6, 3> = + ShardTree::new(MemoryShardStore::empty(), 100); + + check_witness_with_pruned_subtrees(tree) + } + + fn new_tree(m: usize) -> ShardTree, 4, 3> { + ShardTree::new(MemoryShardStore::empty(), m) + } + + #[test] + fn append() { + check_append(new_tree); + } + + #[test] + fn root_hashes() { + check_root_hashes(new_tree); + } + + #[test] + fn witnesses() { + check_witnesses(new_tree); + } + + #[test] + fn witness_consistency() { + check_witness_consistency(new_tree); + } + + #[test] + fn checkpoint_rewind() { + check_checkpoint_rewind(new_tree); + } + + #[test] + fn remove_mark() { + check_remove_mark(new_tree); + } + + #[test] + fn rewind_remove_mark() { + check_rewind_remove_mark(new_tree); + } + + #[test] + fn checkpoint_pruning_repeated() { + // Create a tree with some leaves. + let mut tree = new_tree(10); + for c in 'a'..='c' { + tree.append(c.to_string(), Retention::Ephemeral).unwrap(); + } + + // Repeatedly checkpoint the tree at the same position until the checkpoint cache + // is full (creating a sequence of checkpoints in between which no new leaves were + // appended to the tree). + for i in 0..10 { + assert_eq!(tree.checkpoint(i), Ok(true)); + } + + // Create one more checkpoint at the same position, causing the oldest in the + // cache to be pruned. + assert_eq!(tree.checkpoint(10), Ok(true)); + + // Append a leaf to the tree and checkpoint it, causing the next oldest in the + // cache to be pruned. + assert_eq!( + tree.append( + 'd'.to_string(), + Retention::Checkpoint { + id: 11, + is_marked: false + }, + ), + Ok(()), + ); + } + + // Combined tree tests + #[allow(clippy::type_complexity)] + fn new_combined_tree( + max_checkpoints: usize, + ) -> CombinedTree< + H, + usize, + CompleteTree, + ShardTree, 4, 3>, + > { + CombinedTree::new( + CompleteTree::new(max_checkpoints), + ShardTree::new(MemoryShardStore::empty(), max_checkpoints), + ) + } + + #[test] + fn combined_append() { + check_append::(new_combined_tree); + } + + #[test] + fn combined_rewind_remove_mark() { + check_rewind_remove_mark(new_combined_tree); + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(100000))] + + #[test] + fn check_randomized_u64_ops( + ops in proptest::collection::vec( + arb_operation( + (0..32u64).prop_map(SipHashable), + (0u64..100).prop_map(Position::from) + ), + 1..100 + ) + ) { + let tree = new_combined_tree(100); + let indexed_ops = ops.iter().enumerate().map(|(i, op)| op.map_checkpoint_id(|_| i)).collect::>(); + check_operations(tree, &indexed_ops)?; + } + + #[test] + fn check_randomized_str_ops( + ops in proptest::collection::vec( + arb_operation( + (97u8..123).prop_map(|c| char::from(c).to_string()), + (0u64..100).prop_map(Position::from) + ), + 1..100 + ) + ) { + let tree = new_combined_tree(100); + let indexed_ops = ops.iter().enumerate().map(|(i, op)| op.map_checkpoint_id(|_| i)).collect::>(); + check_operations(tree, &indexed_ops)?; + } + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(100))] + + #[test] + fn check_shardtree_caching( + (mut tree, _, marked_positions) in arb_shardtree(arb_char_str()) + ) { + if let Some(max_leaf_pos) = tree.max_leaf_position(0).unwrap() { + let max_complete_addr = Address::above_position(max_leaf_pos.root_level(), max_leaf_pos); + let root = tree.root(max_complete_addr, max_leaf_pos + 1); + let caching_root = tree.root_caching(max_complete_addr, max_leaf_pos + 1); + assert_matches!(root, Ok(_)); + assert_eq!(root, caching_root); + + for pos in marked_positions { + let witness = tree.witness_at_checkpoint_depth(pos, 0); + let caching_witness = tree.witness_at_checkpoint_depth_caching(pos, 0); + assert_matches!(witness, Ok(_)); + assert_eq!(witness, caching_witness); + } + } + } + } + + #[test] + fn insert_frontier_nodes() { + let mut frontier = NonEmptyFrontier::new("a".to_string()); + for c in 'b'..'z' { + frontier.append(c.to_string()); + } + + let root_addr = Address::from_parts(Level::from(4), 1); + let tree = LocatedPrunableTree::empty(root_addr); + let result = tree.insert_frontier_nodes::<()>(frontier.clone(), &Retention::Ephemeral); + assert_matches!(result, Ok(_)); + + let mut tree1 = LocatedPrunableTree::empty(root_addr); + for c in 'q'..'z' { + let (t, _, _) = tree1 + .append::<()>(c.to_string(), Retention::Ephemeral) + .unwrap(); + tree1 = t; + } + assert_matches!( + tree1.insert_frontier_nodes::<()>(frontier.clone(), &Retention::Ephemeral), + Ok(t) if t == result.unwrap() + ); + + let mut tree2 = LocatedPrunableTree::empty(root_addr); + for c in 'a'..'i' { + let (t, _, _) = tree2 + .append::<()>(c.to_string(), Retention::Ephemeral) + .unwrap(); + tree2 = t; + } + assert_matches!( + tree2.insert_frontier_nodes::<()>(frontier, &Retention::Ephemeral), + Err(InsertionError::Conflict(_)) + ); + } + + #[test] + fn insert_frontier_nodes_sub_shard_height() { + let mut frontier = NonEmptyFrontier::new("a".to_string()); + for c in 'b'..='c' { + frontier.append(c.to_string()); + } + + let root_addr = Address::from_parts(Level::from(3), 0); + let tree = LocatedPrunableTree::empty(root_addr); + let result = tree.insert_frontier_nodes::<()>(frontier.clone(), &Retention::Ephemeral); + assert_matches!(result, Ok((ref _t, None))); + + if let Ok((t, None)) = result { + // verify that the leaf at the tip is included + assert_eq!( + t.root.root_hash(root_addr, Position::from(3)), + Ok("abc_____".to_string()) + ); + } + } +} diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/prunable.rs b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/prunable.rs new file mode 100644 index 000000000000..673a80116c56 --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/prunable.rs @@ -0,0 +1,1133 @@ +//! Helpers for working with trees that support pruning unneeded leaves and branches. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use bitflags::bitflags; +use incrementalmerkletree::{ + frontier::NonEmptyFrontier, Address, Hashable, Level, Position, Retention, +}; +use tracing::trace; + +use crate::error::{InsertionError, QueryError}; +use crate::{LocatedTree, Node, Tree}; + +bitflags! { + /// Flags storing the [`Retention`] state of a leaf. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub struct RetentionFlags: u8 { + /// An leaf with `EPHEMERAL` retention can be pruned as soon as we are certain that it is not part + /// of the witness for a leaf with [`CHECKPOINT`] or [`MARKED`] retention. + /// + /// [`CHECKPOINT`]: RetentionFlags::CHECKPOINT + /// [`MARKED`]: RetentionFlags::MARKED + const EPHEMERAL = 0b00000000; + + /// A leaf with `CHECKPOINT` retention can be pruned when there are more than `max_checkpoints` + /// additional checkpoint leaves, if it is not also a marked leaf. + const CHECKPOINT = 0b00000001; + + /// A leaf with `MARKED` retention can be pruned only as a consequence of an explicit deletion + /// action. + const MARKED = 0b00000010; + } +} + +impl RetentionFlags { + pub fn is_checkpoint(&self) -> bool { + (*self & RetentionFlags::CHECKPOINT) == RetentionFlags::CHECKPOINT + } + + pub fn is_marked(&self) -> bool { + (*self & RetentionFlags::MARKED) == RetentionFlags::MARKED + } +} + +impl<'a, C> From<&'a Retention> for RetentionFlags { + fn from(retention: &'a Retention) -> Self { + match retention { + Retention::Ephemeral => RetentionFlags::EPHEMERAL, + Retention::Checkpoint { is_marked, .. } => { + if *is_marked { + RetentionFlags::CHECKPOINT | RetentionFlags::MARKED + } else { + RetentionFlags::CHECKPOINT + } + } + Retention::Marked => RetentionFlags::MARKED, + } + } +} + +impl From> for RetentionFlags { + fn from(retention: Retention) -> Self { + RetentionFlags::from(&retention) + } +} + +/// A [`Tree`] annotated with Merkle hashes. +pub type PrunableTree = Tree>, (H, RetentionFlags)>; + +impl PrunableTree { + /// Returns the the value if this is a leaf. + pub fn leaf_value(&self) -> Option<&H> { + self.0.leaf_value().map(|(h, _)| h) + } + + /// Returns the cached root value with which the tree has been annotated for this node if it is + /// available, otherwise return the value if this is a leaf. + pub fn node_value(&self) -> Option<&H> { + self.0.annotation().map_or_else( + || self.leaf_value(), + |rc_opt| rc_opt.as_ref().map(|rc| rc.as_ref()), + ) + } + + /// Returns whether or not this tree is a leaf with `Marked` retention. + pub fn is_marked_leaf(&self) -> bool { + self.0 + .leaf_value() + .map_or(false, |(_, retention)| retention.is_marked()) + } + + /// Determines whether a tree has any [`Retention::Marked`] nodes. + pub fn contains_marked(&self) -> bool { + match &self.0 { + Node::Parent { left, right, .. } => left.contains_marked() || right.contains_marked(), + Node::Leaf { value: (_, r) } => r.is_marked(), + Node::Nil => false, + } + } + + /// Returns the Merkle root of this tree, given the address of the root node, or + /// a vector of the addresses of `Nil` nodes that inhibited the computation of + /// such a root. + /// + /// # Parameters: + /// * `truncate_at` An inclusive lower bound on positions in the tree beyond which all leaf + /// values will be treated as `Nil`. + pub fn root_hash(&self, root_addr: Address, truncate_at: Position) -> Result> { + if truncate_at <= root_addr.position_range_start() { + // we are in the part of the tree where we're generating empty roots, + // so no need to inspect the tree + Ok(H::empty_root(root_addr.level())) + } else { + match self { + Tree(Node::Parent { ann, left, right }) => ann + .as_ref() + .filter(|_| truncate_at >= root_addr.position_range_end()) + .map_or_else( + || { + // Compute the roots of the left and right children and hash them + // together. + let (l_addr, r_addr) = root_addr.children().unwrap(); + accumulate_result_with( + left.root_hash(l_addr, truncate_at), + right.root_hash(r_addr, truncate_at), + |left_root, right_root| { + H::combine(l_addr.level(), &left_root, &right_root) + }, + ) + }, + |rc| { + // Since we have an annotation on the root, and we are not truncating + // within this subtree, we can just use the cached value. + Ok(rc.as_ref().clone()) + }, + ), + Tree(Node::Leaf { value }) => { + if truncate_at >= root_addr.position_range_end() { + // no truncation of this leaf is necessary, just use it + Ok(value.0.clone()) + } else { + // we have a leaf value that is a subtree root created by hashing together + // the roots of child subtrees, but truncation would require that that leaf + // value be "split" into its constituent parts, which we can't do so we + // return an error + Err(vec![root_addr]) + } + } + Tree(Node::Nil) => Err(vec![root_addr]), + } + } + } + + /// Returns a vector of the positions of [`Node::Leaf`] values in the tree having + /// [`MARKED`](RetentionFlags::MARKED) retention. + /// + /// Computing the set of marked positions requires a full traversal of the tree, and so should + /// be considered to be a somewhat expensive operation. + pub fn marked_positions(&self, root_addr: Address) -> BTreeSet { + match &self.0 { + Node::Parent { left, right, .. } => { + // We should never construct parent nodes where both children are Nil. + // While we could handle that here, if we encountered that case it would + // be indicative of a programming error elsewhere and so we assert instead. + assert!(!(left.0.is_nil() && right.0.is_nil())); + let (left_root, right_root) = root_addr + .children() + .expect("A parent node cannot appear at level 0"); + + let mut left_incomplete = left.marked_positions(left_root); + let mut right_incomplete = right.marked_positions(right_root); + left_incomplete.append(&mut right_incomplete); + left_incomplete + } + Node::Leaf { + value: (_, retention), + } => { + let mut result = BTreeSet::new(); + if root_addr.level() == 0.into() && retention.is_marked() { + result.insert(Position::from(root_addr.index())); + } + result + } + Node::Nil => BTreeSet::new(), + } + } + + /// Prunes the tree by hashing together ephemeral sibling nodes. + /// + /// `level` must be the level of the root of the node being pruned. + pub fn prune(self, level: Level) -> Self { + match self { + Tree(Node::Parent { ann, left, right }) => Tree::unite( + level, + ann, + left.as_ref().clone().prune(level - 1), + right.as_ref().clone().prune(level - 1), + ), + other => other, + } + } + + /// Merge two subtrees having the same root address. + /// + /// The merge operation is checked to be strictly additive and returns an error if merging + /// would cause information loss or if a conflict between root hashes occurs at a node. The + /// returned error contains the address of the node where such a conflict occurred. + pub fn merge_checked(self, root_addr: Address, other: Self) -> Result { + #[allow(clippy::type_complexity)] + fn go( + addr: Address, + t0: PrunableTree, + t1: PrunableTree, + ) -> Result, Address> { + // Require that any roots the we compute will not be default-filled by picking + // a starting valid fill point that is outside the range of leaf positions. + let no_default_fill = addr.position_range_end(); + match (t0, t1) { + (Tree(Node::Nil), other) | (other, Tree(Node::Nil)) => Ok(other), + (Tree(Node::Leaf { value: vl }), Tree(Node::Leaf { value: vr })) => { + if vl.0 == vr.0 { + // Merge the flags together. + Ok(Tree(Node::Leaf { + value: (vl.0, vl.1 | vr.1), + })) + } else { + trace!(left = ?vl.0, right = ?vr.0, "Merge conflict for leaves"); + Err(addr) + } + } + (Tree(Node::Leaf { value }), parent @ Tree(Node::Parent { .. })) + | (parent @ Tree(Node::Parent { .. }), Tree(Node::Leaf { value })) => { + let parent_hash = parent.root_hash(addr, no_default_fill); + if parent_hash.iter().all(|r| r == &value.0) { + Ok(parent.reannotate_root(Some(Arc::new(value.0)))) + } else { + trace!(leaf = ?value, node = ?parent_hash, "Merge conflict for leaf into node"); + Err(addr) + } + } + (lparent, rparent) => { + let lroot = lparent.root_hash(addr, no_default_fill).ok(); + let rroot = rparent.root_hash(addr, no_default_fill).ok(); + // If both parents share the same root hash (or if one of them is absent), + // they can be merged + if lroot.iter().zip(&rroot).all(|(l, r)| l == r) { + // using `if let` here to bind variables; we need to borrow the trees for + // root hash calculation but binding the children of the parent node + // interferes with binding a reference to the parent. + if let ( + Tree(Node::Parent { + ann: lann, + left: ll, + right: lr, + }), + Tree(Node::Parent { + ann: rann, + left: rl, + right: rr, + }), + ) = (lparent, rparent) + { + let (l_addr, r_addr) = addr.children().unwrap(); + Ok(Tree::unite( + addr.level() - 1, + lann.or(rann), + go(l_addr, ll.as_ref().clone(), rl.as_ref().clone())?, + go(r_addr, lr.as_ref().clone(), rr.as_ref().clone())?, + )) + } else { + unreachable!() + } + } else { + trace!(left = ?lroot, right = ?rroot, "Merge conflict for nodes"); + Err(addr) + } + } + } + } + + trace!(this = ?self, other = ?other, "Merging subtrees"); + go(root_addr, self, other) + } + + /// Unite two nodes by either constructing a new parent node, or, if both nodes are ephemeral + /// leaves or Nil, constructing a replacement root by hashing leaf values together (or a + /// replacement `Nil` value). + /// + /// `level` must be the level of the two nodes that are being joined. + pub(crate) fn unite(level: Level, ann: Option>, left: Self, right: Self) -> Self { + match (left, right) { + (Tree(Node::Nil), Tree(Node::Nil)) => Tree(Node::Nil), + (Tree(Node::Leaf { value: lv }), Tree(Node::Leaf { value: rv })) + // we can prune right-hand leaves that are not marked; if a leaf + // is a checkpoint then that information will be propagated to + // the replacement leaf + if lv.1 == RetentionFlags::EPHEMERAL && (rv.1 & RetentionFlags::MARKED) == RetentionFlags::EPHEMERAL => + { + Tree( + Node::Leaf { + value: (H::combine(level, &lv.0, &rv.0), rv.1), + }, + ) + } + (left, right) => Tree( + Node::Parent { + ann, + left: Arc::new(left), + right: Arc::new(right), + }, + ), + } + } +} + +/// A [`LocatedTree`] annotated with Merkle hashes. +pub type LocatedPrunableTree = LocatedTree>, (H, RetentionFlags)>; + +/// A data structure describing the nature of a [`Node::Nil`] node in the tree that was introduced +/// as the consequence of an insertion. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct IncompleteAt { + /// The address of the empty node. + pub address: Address, + /// A flag identifying whether or not the missing node is required in order to construct a + /// witness for a node with [`MARKED`] retention. + /// + /// [`MARKED`]: RetentionFlags::MARKED + pub required_for_witness: bool, +} + +impl LocatedPrunableTree { + /// Computes the root hash of this tree, truncated to the given position. + /// + /// If the tree contains any [`Node::Nil`] nodes corresponding to positions less than + /// `truncate_at`, this will return an error containing the addresses of those nodes within the + /// tree. + pub fn root_hash(&self, truncate_at: Position) -> Result> { + self.root.root_hash(self.root_addr, truncate_at) + } + + /// Compute the root hash of this subtree, filling empty nodes along the rightmost path of the + /// subtree with the empty root value for the given level. + /// + /// This should only be used for computing roots when it is known that no successor trees + /// exist. + /// + /// If the tree contains any [`Node::Nil`] nodes that are to the left of filled nodes in the + /// tree, this will return an error containing the addresses of those nodes. + pub fn right_filled_root(&self) -> Result> { + self.root_hash( + self.max_position() + .map_or_else(|| self.root_addr.position_range_start(), |pos| pos + 1), + ) + } + + /// Returns the positions of marked leaves in the tree. + pub fn marked_positions(&self) -> BTreeSet { + fn go( + root_addr: Address, + root: &PrunableTree, + acc: &mut BTreeSet, + ) { + match &root.0 { + Node::Parent { left, right, .. } => { + let (l_addr, r_addr) = root_addr.children().unwrap(); + go(l_addr, left.as_ref(), acc); + go(r_addr, right.as_ref(), acc); + } + Node::Leaf { value } => { + if value.1.is_marked() && root_addr.level() == 0.into() { + acc.insert(Position::from(root_addr.index())); + } + } + _ => {} + } + } + + let mut result = BTreeSet::new(); + go(self.root_addr, &self.root, &mut result); + result + } + + /// Compute the witness for the leaf at the specified position. + /// + /// This tree will be truncated to the `truncate_at` position, and then empty roots + /// corresponding to later positions will be filled by the [`Hashable::empty_root`] + /// implementation for `H`. + /// + /// Returns either the witness for the leaf at the specified position, or an error that + /// describes the causes of failure. + pub fn witness(&self, position: Position, truncate_at: Position) -> Result, QueryError> { + // traverse down to the desired leaf position, and then construct + // the authentication path on the way back up. + fn go( + root: &PrunableTree, + root_addr: Address, + position: Position, + truncate_at: Position, + ) -> Result, Vec
> { + match &root.0 { + Node::Parent { left, right, .. } => { + let (l_addr, r_addr) = root_addr.children().unwrap(); + if root_addr.level() > 1.into() { + let r_start = r_addr.position_range_start(); + if position < r_start { + accumulate_result_with( + go(left.as_ref(), l_addr, position, truncate_at), + right.as_ref().root_hash(r_addr, truncate_at), + |mut witness, sibling_root| { + witness.push(sibling_root); + witness + }, + ) + } else { + // if the position we're witnessing is down the right-hand branch then + // we always set the truncation bound outside the range of leaves on the + // left, because we don't allow any empty nodes to the left + accumulate_result_with( + left.as_ref().root_hash(l_addr, r_start), + go(right.as_ref(), r_addr, position, truncate_at), + |sibling_root, mut witness| { + witness.push(sibling_root); + witness + }, + ) + } + } else { + // we handle the level 0 leaves here by adding the sibling of our desired + // leaf to the witness + if position.is_right_child() { + if right.is_marked_leaf() { + left.leaf_value() + .map(|v| vec![v.clone()]) + .ok_or_else(|| vec![l_addr]) + } else { + Err(vec![l_addr]) + } + } else if left.is_marked_leaf() { + // If we have the left-hand leaf and the right-hand leaf is empty, we + // can fill it with the empty leaf, but only if we are truncating at + // a position to the left of the current position + if truncate_at <= position + 1 { + Ok(vec![H::empty_leaf()]) + } else { + right + .leaf_value() + .map_or_else(|| Err(vec![r_addr]), |v| Ok(vec![v.clone()])) + } + } else { + Err(vec![r_addr]) + } + } + } + _ => { + // if we encounter a nil or leaf node, we were unable to descend + // to the leaf at the desired position. + Err(vec![root_addr]) + } + } + } + + if self.root_addr.position_range().contains(&position) { + go(&self.root, self.root_addr, position, truncate_at) + .map_err(QueryError::TreeIncomplete) + } else { + Err(QueryError::NotContained(self.root_addr)) + } + } + + /// Prunes this tree by replacing all nodes that are right-hand children along the path + /// to the specified position with [`Node::Nil`]. + /// + /// The leaf at the specified position is retained. Returns the truncated tree if a leaf or + /// subtree root with the specified position as its maximum position exists, or `None` + /// otherwise. + pub fn truncate_to_position(&self, position: Position) -> Option { + fn go( + position: Position, + root_addr: Address, + root: &PrunableTree, + ) -> Option> { + match &root.0 { + Node::Parent { ann, left, right } => { + let (l_child, r_child) = root_addr.children().unwrap(); + if position < r_child.position_range_start() { + // we are truncating within the range of the left node, so recurse + // to the left to truncate the left child and then reconstruct the + // node with `Nil` as the right sibling + go(position, l_child, left.as_ref()).map(|left| { + Tree::unite(l_child.level(), ann.clone(), left, Tree(Node::Nil)) + }) + } else { + // we are truncating within the range of the right node, so recurse + // to the right to truncate the right child and then reconstruct the + // node with the left sibling unchanged + go(position, r_child, right.as_ref()).map(|right| { + Tree::unite(r_child.level(), ann.clone(), left.as_ref().clone(), right) + }) + } + } + Node::Leaf { .. } => { + if root_addr.max_position() <= position { + Some(root.clone()) + } else { + None + } + } + Node::Nil => None, + } + } + + if self.root_addr.position_range().contains(&position) { + go(position, self.root_addr, &self.root).map(|root| LocatedTree { + root_addr: self.root_addr, + root, + }) + } else { + None + } + } + + /// Inserts a descendant subtree into this subtree, creating empty sibling nodes as necessary + /// to fill out the tree. + /// + /// In the case that a leaf node would be replaced by an incomplete subtree, the resulting + /// parent node will be annotated with the existing leaf value. + /// + /// Returns the updated tree, along with the addresses of any [`Node::Nil`] nodes that were + /// inserted in the process of creating the parent nodes down to the insertion point, or an + /// error if the specified subtree's root address is not in the range of valid descendants of + /// the root node of this tree or if the insertion would result in a conflict between computed + /// root hashes of complete subtrees. + pub fn insert_subtree( + &self, + subtree: Self, + contains_marked: bool, + ) -> Result<(Self, Vec), InsertionError> { + // A function to recursively dig into the tree, creating a path downward and introducing + // empty nodes as necessary until we can insert the provided subtree. + #[allow(clippy::type_complexity)] + fn go( + root_addr: Address, + into: &PrunableTree, + subtree: LocatedPrunableTree, + is_complete: bool, + contains_marked: bool, + ) -> Result<(PrunableTree, Vec), InsertionError> { + // In the case that we are replacing a node entirely, we need to extend the + // subtree up to the level of the node being replaced, adding Nil siblings + // and recording the presence of those incomplete nodes when necessary + let replacement = |ann: Option>, mut node: LocatedPrunableTree| { + // construct the replacement node bottom-up + let mut incomplete = vec![]; + while node.root_addr.level() < root_addr.level() { + incomplete.push(IncompleteAt { + address: node.root_addr.sibling(), + required_for_witness: contains_marked, + }); + node = LocatedTree { + root_addr: node.root_addr.parent(), + root: if node.root_addr.is_right_child() { + Tree(Node::Parent { + ann: None, + left: Arc::new(Tree(Node::Nil)), + right: Arc::new(node.root), + }) + } else { + Tree(Node::Parent { + ann: None, + left: Arc::new(node.root), + right: Arc::new(Tree(Node::Nil)), + }) + }, + }; + } + (node.root.reannotate_root(ann), incomplete) + }; + + trace!( + "Node at {:?} contains subtree at {:?}", + root_addr, + subtree.root_addr(), + ); + match into { + Tree(Node::Nil) => Ok(replacement(None, subtree)), + Tree(Node::Leaf { value: (value, _) }) => { + if root_addr == subtree.root_addr { + if is_complete { + // It is safe to replace the existing root unannotated, because we + // can always recompute the root from a complete subtree. + Ok((subtree.root, vec![])) + } else if subtree.root.node_value().iter().all(|v| v == &value) { + Ok(( + // at this point we statically know the root to be a parent + subtree.root.reannotate_root(Some(Arc::new(value.clone()))), + vec![], + )) + } else { + trace!( + cur_root = ?value, + new_root = ?subtree.root.node_value(), + "Insertion conflict", + ); + Err(InsertionError::Conflict(root_addr)) + } + } else { + Ok(replacement(Some(Arc::new(value.clone())), subtree)) + } + } + parent if root_addr == subtree.root_addr => { + // Merge the existing subtree with the subtree being inserted. + // A merge operation can't introduce any new incomplete roots. + parent + .clone() + .merge_checked(root_addr, subtree.root) + .map_err(InsertionError::Conflict) + .map(|tree| (tree, vec![])) + } + Tree(Node::Parent { ann, left, right }) => { + // In this case, we have an existing parent but we need to dig down farther + // before we can insert the subtree that we're carrying for insertion. + let (l_addr, r_addr) = root_addr.children().unwrap(); + if l_addr.contains(&subtree.root_addr) { + let (new_left, incomplete) = + go(l_addr, left.as_ref(), subtree, is_complete, contains_marked)?; + Ok(( + Tree::unite( + root_addr.level() - 1, + ann.clone(), + new_left, + right.as_ref().clone(), + ), + incomplete, + )) + } else { + let (new_right, incomplete) = go( + r_addr, + right.as_ref(), + subtree, + is_complete, + contains_marked, + )?; + Ok(( + Tree::unite( + root_addr.level() - 1, + ann.clone(), + left.as_ref().clone(), + new_right, + ), + incomplete, + )) + } + } + } + } + + let LocatedTree { root_addr, root } = self; + if root_addr.contains(&subtree.root_addr) { + let complete = subtree.root.is_complete(); + go(*root_addr, root, subtree, complete, contains_marked).map(|(root, incomplete)| { + ( + LocatedTree { + root_addr: *root_addr, + root, + }, + incomplete, + ) + }) + } else { + Err(InsertionError::NotContained(subtree.root_addr)) + } + } + + /// Append a single value at the first available position in the tree. + /// + /// Prefer to use [`Self::batch_append`] or [`Self::batch_insert`] when appending multiple + /// values, as these operations require fewer traversals of the tree than are necessary when + /// performing multiple sequential calls to [`Self::append`]. + pub fn append( + &self, + value: H, + retention: Retention, + ) -> Result<(Self, Position, Option), InsertionError> { + let checkpoint_id = if let Retention::Checkpoint { id, .. } = &retention { + Some(id.clone()) + } else { + None + }; + + self.batch_append(Some((value, retention)).into_iter()) + // We know that the max insert position will have been incremented by one. + .and_then(|r| { + let mut r = r.expect("We know the iterator to have been nonempty."); + if r.remainder.next().is_some() { + Err(InsertionError::TreeFull) + } else { + Ok((r.subtree, r.max_insert_position.unwrap(), checkpoint_id)) + } + }) + } + + // Constructs a pair of trees that contain the leaf and ommers of the given frontier. The first + // element of the result is a tree with its root at a level less than or equal to `split_at`; + // the second element is a tree with its leaves at level `split_at` that is only returned if + // the frontier contains sufficient data to fill the first tree to the `split_at` level. + fn from_frontier( + frontier: NonEmptyFrontier, + leaf_retention: &Retention, + split_at: Level, + ) -> (Self, Option) { + let (position, leaf, ommers) = frontier.into_parts(); + Self::from_frontier_parts(position, leaf, ommers.into_iter(), leaf_retention, split_at) + } + + // Permits construction of a subtree from legacy `CommitmentTree` data that may + // have inaccurate position information (e.g. in the case that the tree is the + // cursor for an `IncrementalWitness`). + pub(crate) fn from_frontier_parts( + position: Position, + leaf: H, + mut ommers: impl Iterator, + leaf_retention: &Retention, + split_at: Level, + ) -> (Self, Option) { + let mut addr = Address::from(position); + let mut subtree = Tree(Node::Leaf { + value: (leaf, leaf_retention.into()), + }); + + while addr.level() < split_at { + if addr.is_left_child() { + // the current address is a left child, so create a parent with + // an empty right-hand tree + subtree = Tree::parent(None, subtree, Tree::empty()); + } else if let Some(left) = ommers.next() { + // the current address corresponds to a right child, so create a parent that + // takes the left sibling to that child from the ommers + subtree = + Tree::parent(None, Tree::leaf((left, RetentionFlags::EPHEMERAL)), subtree); + } else { + break; + } + + addr = addr.parent(); + } + + let located_subtree = LocatedTree { + root_addr: addr, + root: subtree, + }; + + let located_supertree = if located_subtree.root_addr().level() == split_at { + let mut addr = located_subtree.root_addr(); + let mut supertree = None; + for left in ommers { + // build up the left-biased tree until we get a right-hand node + while addr.is_left_child() { + supertree = supertree.map(|t| Tree::parent(None, t, Tree::empty())); + addr = addr.parent(); + } + + // once we have a right-hand root, add a parent with the current ommer as the + // left-hand sibling + supertree = Some(Tree::parent( + None, + Tree::leaf((left, RetentionFlags::EPHEMERAL)), + supertree.unwrap_or_else(Tree::empty), + )); + addr = addr.parent(); + } + + supertree.map(|t| LocatedTree { + root_addr: addr, + root: t, + }) + } else { + // if there were not enough ommers available from the frontier to reach the address + // of the root of this tree, there is no contribution to the cap + None + }; + + (located_subtree, located_supertree) + } + + /// Inserts leaves and subtree roots from the provided frontier into this tree, up to the level + /// of this tree's root. + /// + /// Returns the updated tree, along with a [`LocatedPrunableTree`] containing only the remainder + /// of the frontier's ommers that had addresses at levels greater than the root of this tree. + /// + /// Returns an error in the following cases: + /// * the leaf node of `frontier` is at a position that is not contained within this tree's + /// position range + /// * a conflict occurs where an ommer of the frontier being inserted does not match the + /// existing value for that node + pub fn insert_frontier_nodes( + &self, + frontier: NonEmptyFrontier, + leaf_retention: &Retention, + ) -> Result<(Self, Option), InsertionError> { + let subtree_range = self.root_addr.position_range(); + if subtree_range.contains(&frontier.position()) { + let leaf_is_marked = leaf_retention.is_marked(); + let (subtree, supertree) = + Self::from_frontier(frontier, leaf_retention, self.root_addr.level()); + + let subtree = self.insert_subtree(subtree, leaf_is_marked)?.0; + + Ok((subtree, supertree)) + } else { + Err(InsertionError::OutOfRange( + frontier.position(), + subtree_range, + )) + } + } + + /// Clears the specified retention flags at all positions specified, pruning any branches + /// that no longer need to be retained. + pub fn clear_flags(&self, to_clear: BTreeMap) -> Self { + fn go( + to_clear: &[(Position, RetentionFlags)], + root_addr: Address, + root: &PrunableTree, + ) -> PrunableTree { + if to_clear.is_empty() { + // nothing to do, so we just return the root + root.clone() + } else { + match root { + Tree(Node::Parent { ann, left, right }) => { + let (l_addr, r_addr) = root_addr.children().unwrap(); + + let p = to_clear.partition_point(|(p, _)| p < &l_addr.position_range_end()); + trace!( + "In {:?}, partitioned: {:?} {:?}", + root_addr, + &to_clear[0..p], + &to_clear[p..], + ); + Tree::unite( + l_addr.level(), + ann.clone(), + go(&to_clear[0..p], l_addr, left), + go(&to_clear[p..], r_addr, right), + ) + } + Tree(Node::Leaf { value: (h, r) }) => { + trace!("In {:?}, clearing {:?}", root_addr, to_clear); + // When we reach a leaf, we should be down to just a single position + // which should correspond to the last level-0 child of the address's + // subtree range; if it's a checkpoint this will always be the case for + // a partially-pruned branch, and if it's a marked node then it will + // be a level-0 leaf. + match to_clear { + [(pos, flags)] => { + assert_eq!(*pos, root_addr.max_position()); + Tree(Node::Leaf { + value: (h.clone(), *r & !*flags), + }) + } + _ => { + panic!("Tree state inconsistent with checkpoints."); + } + } + } + Tree(Node::Nil) => Tree(Node::Nil), + } + } + } + + let to_clear = to_clear.into_iter().collect::>(); + Self { + root_addr: self.root_addr, + root: go(&to_clear, self.root_addr, &self.root), + } + } +} + +// We need an applicative functor for Result for this function so that we can correctly +// accumulate errors, but we don't have one so we just write a special- cased version here. +fn accumulate_result_with( + left: Result>, + right: Result>, + combine_success: impl FnOnce(A, B) -> C, +) -> Result> { + match (left, right) { + (Ok(a), Ok(b)) => Ok(combine_success(a, b)), + (Err(mut xs), Err(mut ys)) => { + xs.append(&mut ys); + Err(xs) + } + (Ok(_), Err(xs)) => Err(xs), + (Err(xs), Ok(_)) => Err(xs), + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use incrementalmerkletree::{Address, Level, Position}; + + use super::{LocatedPrunableTree, PrunableTree, RetentionFlags}; + use crate::{ + error::QueryError, + tree::{ + tests::{leaf, nil, parent}, + LocatedTree, + }, + }; + + #[test] + fn root() { + let t: PrunableTree = parent( + leaf(("a".to_string(), RetentionFlags::EPHEMERAL)), + leaf(("b".to_string(), RetentionFlags::EPHEMERAL)), + ); + + assert_eq!( + t.root_hash(Address::from_parts(Level::from(1), 0), Position::from(2)), + Ok("ab".to_string()) + ); + + let t0 = parent(nil(), t.clone()); + assert_eq!( + t0.root_hash(Address::from_parts(Level::from(2), 0), Position::from(4)), + Err(vec![Address::from_parts(Level::from(1), 0)]) + ); + + // Check root computation with truncation + let t1 = parent(t, nil()); + assert_eq!( + t1.root_hash(Address::from_parts(Level::from(2), 0), Position::from(2)), + Ok("ab__".to_string()) + ); + assert_eq!( + t1.root_hash(Address::from_parts(Level::from(2), 0), Position::from(3)), + Err(vec![Address::from_parts(Level::from(1), 1)]) + ); + } + + #[test] + fn marked_positions() { + let t: PrunableTree = parent( + leaf(("a".to_string(), RetentionFlags::EPHEMERAL)), + leaf(("b".to_string(), RetentionFlags::MARKED)), + ); + assert_eq!( + t.marked_positions(Address::from_parts(Level::from(1), 0)), + BTreeSet::from([Position::from(1)]) + ); + + let t0 = parent(t.clone(), t); + assert_eq!( + t0.marked_positions(Address::from_parts(Level::from(2), 1)), + BTreeSet::from([Position::from(5), Position::from(7)]) + ); + } + + #[test] + fn prune() { + let t: PrunableTree = parent( + leaf(("a".to_string(), RetentionFlags::EPHEMERAL)), + leaf(("b".to_string(), RetentionFlags::EPHEMERAL)), + ); + + assert_eq!( + t.clone().prune(Level::from(1)), + leaf(("ab".to_string(), RetentionFlags::EPHEMERAL)) + ); + + let t0 = parent(leaf(("c".to_string(), RetentionFlags::MARKED)), t); + assert_eq!( + t0.prune(Level::from(2)), + parent( + leaf(("c".to_string(), RetentionFlags::MARKED)), + leaf(("ab".to_string(), RetentionFlags::EPHEMERAL)) + ) + ); + } + + #[test] + fn merge_checked() { + let t0: PrunableTree = + parent(leaf(("a".to_string(), RetentionFlags::EPHEMERAL)), nil()); + + let t1: PrunableTree = + parent(nil(), leaf(("b".to_string(), RetentionFlags::EPHEMERAL))); + + assert_eq!( + t0.clone() + .merge_checked(Address::from_parts(1.into(), 0), t1.clone()), + Ok(leaf(("ab".to_string(), RetentionFlags::EPHEMERAL))) + ); + + let t2: PrunableTree = + parent(leaf(("c".to_string(), RetentionFlags::EPHEMERAL)), nil()); + assert_eq!( + t0.clone() + .merge_checked(Address::from_parts(1.into(), 0), t2.clone()), + Err(Address::from_parts(0.into(), 0)) + ); + + let t3: PrunableTree = parent(t0, t2); + let t4: PrunableTree = parent(t1.clone(), t1); + + assert_eq!( + t3.merge_checked(Address::from_parts(2.into(), 0), t4), + Ok(leaf(("abcb".to_string(), RetentionFlags::EPHEMERAL))) + ); + } + + #[test] + fn merge_checked_flags() { + let t0: PrunableTree = leaf(("a".to_string(), RetentionFlags::EPHEMERAL)); + let t1: PrunableTree = leaf(("a".to_string(), RetentionFlags::MARKED)); + let t2: PrunableTree = leaf(("a".to_string(), RetentionFlags::CHECKPOINT)); + + assert_eq!( + t0.merge_checked(Address::from_parts(1.into(), 0), t1.clone()), + Ok(t1.clone()), + ); + + assert_eq!( + t1.merge_checked(Address::from_parts(1.into(), 0), t2), + Ok(leaf(( + "a".to_string(), + RetentionFlags::MARKED | RetentionFlags::CHECKPOINT, + ))), + ); + } + + #[test] + fn located_insert_subtree() { + let t: LocatedPrunableTree = LocatedTree { + root_addr: Address::from_parts(3.into(), 1), + root: parent( + leaf(("abcd".to_string(), RetentionFlags::EPHEMERAL)), + parent(nil(), leaf(("gh".to_string(), RetentionFlags::EPHEMERAL))), + ), + }; + + assert_eq!( + t.insert_subtree( + LocatedTree { + root_addr: Address::from_parts(1.into(), 6), + root: parent(leaf(("e".to_string(), RetentionFlags::MARKED)), nil()) + }, + true + ), + Ok(( + LocatedTree { + root_addr: Address::from_parts(3.into(), 1), + root: parent( + leaf(("abcd".to_string(), RetentionFlags::EPHEMERAL)), + parent( + parent(leaf(("e".to_string(), RetentionFlags::MARKED)), nil()), + leaf(("gh".to_string(), RetentionFlags::EPHEMERAL)) + ) + ) + }, + vec![] + )) + ); + } + + #[test] + fn located_insert_subtree_leaf_overwrites() { + let t: LocatedPrunableTree = LocatedTree { + root_addr: Address::from_parts(2.into(), 1), + root: parent(leaf(("a".to_string(), RetentionFlags::MARKED)), nil()), + }; + + assert_eq!( + t.insert_subtree( + LocatedTree { + root_addr: Address::from_parts(1.into(), 2), + root: leaf(("b".to_string(), RetentionFlags::EPHEMERAL)), + }, + false, + ), + Ok(( + LocatedTree { + root_addr: Address::from_parts(2.into(), 1), + root: parent(leaf(("b".to_string(), RetentionFlags::EPHEMERAL)), nil()), + }, + vec![], + )), + ); + } + + #[test] + fn located_witness() { + let t: LocatedPrunableTree = LocatedTree { + root_addr: Address::from_parts(3.into(), 0), + root: parent( + leaf(("abcd".to_string(), RetentionFlags::EPHEMERAL)), + parent( + parent( + leaf(("e".to_string(), RetentionFlags::MARKED)), + leaf(("f".to_string(), RetentionFlags::EPHEMERAL)), + ), + leaf(("gh".to_string(), RetentionFlags::EPHEMERAL)), + ), + ), + }; + + assert_eq!( + t.witness(4.into(), 8.into()), + Ok(vec!["f", "gh", "abcd"] + .into_iter() + .map(|s| s.to_string()) + .collect()) + ); + assert_eq!( + t.witness(4.into(), 6.into()), + Ok(vec!["f", "__", "abcd"] + .into_iter() + .map(|s| s.to_string()) + .collect()) + ); + assert_eq!( + t.witness(4.into(), 7.into()), + Err(QueryError::TreeIncomplete(vec![Address::from_parts( + 1.into(), + 3 + )])) + ); + } +} diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/store.rs b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/store.rs new file mode 100644 index 000000000000..eadd16862577 --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/store.rs @@ -0,0 +1,307 @@ +//! Traits and structs for storing [`ShardTree`]s. +//! +//! [`ShardTree`]: crate::ShardTree +//! +//! # Structure +//! +//! The tree is represented as an ordered collection of fixed-depth subtrees, or "shards". +//! Each shard is a [`LocatedPrunableTree`]. The roots of the shards form the leaves in +//! the "cap", which is a [`PrunableTree`]. +//! +//! ```text +//! Level +//! 3 root \ +//! / \ | +//! / \ | +//! 2 / \ } cap +//! / \ / \ | +//! / \ / \ | +//! 1 A B C D / \ +//! / \ / \ / \ / \ } shards +//! 0 /\ /\ /\ /\ /\ /\ /\ /\ / +//! ``` + +use std::collections::BTreeSet; + +use incrementalmerkletree::{Address, Position}; + +use crate::{LocatedPrunableTree, PrunableTree}; + +pub mod caching; +pub mod memory; + +/// A capability for storage of fragment subtrees of the [`ShardTree`] type. +/// +/// All fragment subtrees must have roots at level `SHARD_HEIGHT`. +/// +/// [`ShardTree`]: crate::ShardTree +pub trait ShardStore { + /// The type used for leaves and nodes in the tree. + type H; + + /// The type used to identify checkpointed positions in the tree. + type CheckpointId; + + /// The error type for operations on this store. + type Error: std::error::Error; + + /// Returns the subtree at the given root address, if any such subtree exists. + fn get_shard( + &self, + shard_root: Address, + ) -> Result>, Self::Error>; + + /// Returns the subtree containing the maximum inserted leaf position. + fn last_shard(&self) -> Result>, Self::Error>; + + /// Inserts or replaces the subtree having the same root address as the provided tree. + /// + /// Implementations of this method MUST enforce the constraint that the root address + /// of the provided subtree has level `SHARD_HEIGHT`. + fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error>; + + /// Returns the vector of addresses corresponding to the roots of subtrees stored in this + /// store. + fn get_shard_roots(&self) -> Result, Self::Error>; + + /// Removes subtrees from the underlying store having root addresses at indices greater + /// than or equal to that of the specified address. + /// + /// Implementations of this method MUST enforce the constraint that the root address + /// provided has level `SHARD_HEIGHT`. + fn truncate(&mut self, from: Address) -> Result<(), Self::Error>; + + /// A tree that is used to cache the known roots of subtrees in the "cap" - the top part of the + /// tree, which contains parent nodes produced by hashing the roots of the individual shards. + /// Nodes in the cap have levels in the range `SHARD_HEIGHT..DEPTH`. Note that the cap may be + /// sparse, in the same way that individual shards may be sparse. + fn get_cap(&self) -> Result, Self::Error>; + + /// Persists the provided cap to the data store. + fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error>; + + /// Returns the identifier for the checkpoint with the lowest associated position value. + fn min_checkpoint_id(&self) -> Result, Self::Error>; + + /// Returns the identifier for the checkpoint with the highest associated position value. + fn max_checkpoint_id(&self) -> Result, Self::Error>; + + /// Adds a checkpoint to the data store. + fn add_checkpoint( + &mut self, + checkpoint_id: Self::CheckpointId, + checkpoint: Checkpoint, + ) -> Result<(), Self::Error>; + + /// Returns the number of checkpoints maintained by the data store + fn checkpoint_count(&self) -> Result; + + /// Returns the id and position of the checkpoint, if any. Returns `None` if + /// `checkpoint_depth == 0` or if insufficient checkpoints exist to seek back + /// to the requested depth. + fn get_checkpoint_at_depth( + &self, + checkpoint_depth: usize, + ) -> Result, Self::Error>; + + /// Returns the checkpoint corresponding to the specified checkpoint identifier. + fn get_checkpoint( + &self, + checkpoint_id: &Self::CheckpointId, + ) -> Result, Self::Error>; + + /// Iterates in checkpoint ID order over the first `limit` checkpoints, applying the + /// given callback to each. + fn with_checkpoints(&mut self, limit: usize, callback: F) -> Result<(), Self::Error> + where + F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>; + + /// Update the checkpoint having the given identifier by mutating it with the provided + /// function, and persist the updated checkpoint to the data store. + /// + /// Returns `Ok(true)` if the checkpoint was found, `Ok(false)` if no checkpoint with the + /// provided identifier exists in the data store, or an error if a storage error occurred. + fn update_checkpoint_with( + &mut self, + checkpoint_id: &Self::CheckpointId, + update: F, + ) -> Result + where + F: Fn(&mut Checkpoint) -> Result<(), Self::Error>; + + /// Removes a checkpoint from the data store. + /// + /// If no checkpoint exists with the given ID, this does nothing. + fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error>; + + /// Removes checkpoints with identifiers greater than or equal to the given identifier. + fn truncate_checkpoints( + &mut self, + checkpoint_id: &Self::CheckpointId, + ) -> Result<(), Self::Error>; +} + +impl ShardStore for &mut S { + type H = S::H; + type CheckpointId = S::CheckpointId; + type Error = S::Error; + + fn get_shard( + &self, + shard_root: Address, + ) -> Result>, Self::Error> { + S::get_shard(*self, shard_root) + } + + fn last_shard(&self) -> Result>, Self::Error> { + S::last_shard(*self) + } + + fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { + S::put_shard(*self, subtree) + } + + fn get_shard_roots(&self) -> Result, Self::Error> { + S::get_shard_roots(*self) + } + + fn get_cap(&self) -> Result, Self::Error> { + S::get_cap(*self) + } + + fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error> { + S::put_cap(*self, cap) + } + + fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { + S::truncate(*self, from) + } + + fn min_checkpoint_id(&self) -> Result, Self::Error> { + S::min_checkpoint_id(self) + } + + fn max_checkpoint_id(&self) -> Result, Self::Error> { + S::max_checkpoint_id(self) + } + + fn add_checkpoint( + &mut self, + checkpoint_id: Self::CheckpointId, + checkpoint: Checkpoint, + ) -> Result<(), Self::Error> { + S::add_checkpoint(self, checkpoint_id, checkpoint) + } + + fn checkpoint_count(&self) -> Result { + S::checkpoint_count(self) + } + + fn get_checkpoint_at_depth( + &self, + checkpoint_depth: usize, + ) -> Result, Self::Error> { + S::get_checkpoint_at_depth(self, checkpoint_depth) + } + + fn get_checkpoint( + &self, + checkpoint_id: &Self::CheckpointId, + ) -> Result, Self::Error> { + S::get_checkpoint(self, checkpoint_id) + } + + fn with_checkpoints(&mut self, limit: usize, callback: F) -> Result<(), Self::Error> + where + F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, + { + S::with_checkpoints(self, limit, callback) + } + + fn update_checkpoint_with( + &mut self, + checkpoint_id: &Self::CheckpointId, + update: F, + ) -> Result + where + F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, + { + S::update_checkpoint_with(self, checkpoint_id, update) + } + + fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error> { + S::remove_checkpoint(self, checkpoint_id) + } + + fn truncate_checkpoints( + &mut self, + checkpoint_id: &Self::CheckpointId, + ) -> Result<(), Self::Error> { + S::truncate_checkpoints(self, checkpoint_id) + } +} + +/// An enumeration of possible checkpoint locations. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum TreeState { + /// Checkpoints of the empty tree. + Empty, + /// Checkpoint at a (possibly pruned) leaf state corresponding to the + /// wrapped leaf position. + AtPosition(Position), +} + +/// The information required to save the state of a [`ShardTree`] at some [`Position`]. +/// +/// [`ShardTree`]: crate::ShardTree +#[derive(Clone, Debug)] +pub struct Checkpoint { + tree_state: TreeState, + marks_removed: BTreeSet, +} + +impl Checkpoint { + pub fn tree_empty() -> Self { + Checkpoint { + tree_state: TreeState::Empty, + marks_removed: BTreeSet::new(), + } + } + + pub fn at_position(position: Position) -> Self { + Checkpoint { + tree_state: TreeState::AtPosition(position), + marks_removed: BTreeSet::new(), + } + } + + pub fn from_parts(tree_state: TreeState, marks_removed: BTreeSet) -> Self { + Checkpoint { + tree_state, + marks_removed, + } + } + + pub fn tree_state(&self) -> TreeState { + self.tree_state + } + + pub fn marks_removed(&self) -> &BTreeSet { + &self.marks_removed + } + + pub fn is_tree_empty(&self) -> bool { + matches!(self.tree_state, TreeState::Empty) + } + + pub fn position(&self) -> Option { + match self.tree_state { + TreeState::Empty => None, + TreeState::AtPosition(pos) => Some(pos), + } + } + + pub(crate) fn mark_removed(&mut self, position: Position) { + self.marks_removed.insert(position); + } +} diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/store/caching.rs b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/store/caching.rs new file mode 100644 index 000000000000..ab898fe4a7df --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/store/caching.rs @@ -0,0 +1,1252 @@ +//! Implementation of an in-memory shard store with persistence. + +use std::convert::Infallible; + +use incrementalmerkletree::Address; + +use super::{memory::MemoryShardStore, Checkpoint, ShardStore}; +use crate::{LocatedPrunableTree, PrunableTree}; + +#[derive(Debug)] +enum Action { + Truncate(Address), + RemoveCheckpoint(C), + TruncateCheckpoints(C), +} + +/// An implementation of [`ShardStore`] that caches all state in memory. +/// +/// Cache state is flushed to the backend via [`Self::flush`]. Dropping will instead drop +/// the cached state and not make any changes to the backend. +#[derive(Debug)] +pub struct CachingShardStore +where + S: ShardStore, + S::H: Clone, + S::CheckpointId: Clone + Ord, +{ + backend: S, + cache: MemoryShardStore, + deferred_actions: Vec>, +} + +impl CachingShardStore +where + S: ShardStore, + S::H: Clone, + S::CheckpointId: Clone + Ord, +{ + /// Loads a `CachingShardStore` from the given backend. + pub fn load(mut backend: S) -> Result { + let mut cache = MemoryShardStore::empty(); + + for shard_root in backend.get_shard_roots()? { + let _ = cache.put_shard(backend.get_shard(shard_root)?.expect("known address")); + } + let _ = cache.put_cap(backend.get_cap()?); + + backend.with_checkpoints(backend.checkpoint_count()?, |checkpoint_id, checkpoint| { + cache + .add_checkpoint(checkpoint_id.clone(), checkpoint.clone()) + .unwrap(); + Ok(()) + })?; + + Ok(Self { + backend, + cache, + deferred_actions: vec![], + }) + } + + /// Flushes the current cache state to the backend and returns it. + pub fn flush(mut self) -> Result { + for action in &self.deferred_actions { + match action { + Action::Truncate(from) => self.backend.truncate(*from), + Action::RemoveCheckpoint(checkpoint_id) => { + self.backend.remove_checkpoint(checkpoint_id) + } + Action::TruncateCheckpoints(checkpoint_id) => { + self.backend.truncate_checkpoints(checkpoint_id) + } + }?; + } + self.deferred_actions.clear(); + + for shard_root in self.cache.get_shard_roots().unwrap() { + self.backend.put_shard( + self.cache + .get_shard(shard_root) + .unwrap() + .expect("known address"), + )?; + } + self.backend.put_cap(self.cache.get_cap().unwrap())?; + + let mut checkpoints = Vec::with_capacity(self.cache.checkpoint_count().unwrap()); + self.cache + .with_checkpoints( + self.cache.checkpoint_count().unwrap(), + |checkpoint_id, checkpoint| { + checkpoints.push((checkpoint_id.clone(), checkpoint.clone())); + Ok(()) + }, + ) + .unwrap(); + for (checkpoint_id, checkpoint) in checkpoints { + self.backend.add_checkpoint(checkpoint_id, checkpoint)?; + } + + Ok(self.backend) + } +} + +impl ShardStore for CachingShardStore +where + S: ShardStore, + S::H: Clone, + S::CheckpointId: Clone + Ord, +{ + type H = S::H; + type CheckpointId = S::CheckpointId; + type Error = Infallible; + + fn get_shard( + &self, + shard_root: Address, + ) -> Result>, Self::Error> { + self.cache.get_shard(shard_root) + } + + fn last_shard(&self) -> Result>, Self::Error> { + self.cache.last_shard() + } + + fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { + self.cache.put_shard(subtree) + } + + fn get_shard_roots(&self) -> Result, Self::Error> { + self.cache.get_shard_roots() + } + + fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { + self.deferred_actions.push(Action::Truncate(from)); + self.cache.truncate(from) + } + + fn get_cap(&self) -> Result, Self::Error> { + self.cache.get_cap() + } + + fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error> { + self.cache.put_cap(cap) + } + + fn add_checkpoint( + &mut self, + checkpoint_id: Self::CheckpointId, + checkpoint: Checkpoint, + ) -> Result<(), Self::Error> { + self.cache.add_checkpoint(checkpoint_id, checkpoint) + } + + fn checkpoint_count(&self) -> Result { + self.cache.checkpoint_count() + } + + fn get_checkpoint( + &self, + checkpoint_id: &Self::CheckpointId, + ) -> Result, Self::Error> { + self.cache.get_checkpoint(checkpoint_id) + } + + fn get_checkpoint_at_depth( + &self, + checkpoint_depth: usize, + ) -> Result, Self::Error> { + self.cache.get_checkpoint_at_depth(checkpoint_depth) + } + + fn min_checkpoint_id(&self) -> Result, Self::Error> { + self.cache.min_checkpoint_id() + } + + fn max_checkpoint_id(&self) -> Result, Self::Error> { + self.cache.max_checkpoint_id() + } + + fn with_checkpoints(&mut self, limit: usize, callback: F) -> Result<(), Self::Error> + where + F: FnMut(&Self::CheckpointId, &Checkpoint) -> Result<(), Self::Error>, + { + self.cache.with_checkpoints(limit, callback) + } + + fn update_checkpoint_with( + &mut self, + checkpoint_id: &Self::CheckpointId, + update: F, + ) -> Result + where + F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, + { + self.cache.update_checkpoint_with(checkpoint_id, update) + } + + fn remove_checkpoint(&mut self, checkpoint_id: &Self::CheckpointId) -> Result<(), Self::Error> { + self.deferred_actions + .push(Action::RemoveCheckpoint(checkpoint_id.clone())); + self.cache.remove_checkpoint(checkpoint_id) + } + + fn truncate_checkpoints( + &mut self, + checkpoint_id: &Self::CheckpointId, + ) -> Result<(), Self::Error> { + self.deferred_actions + .push(Action::TruncateCheckpoints(checkpoint_id.clone())); + self.cache.truncate_checkpoints(checkpoint_id) + } +} + +#[cfg(test)] +mod tests { + use incrementalmerkletree::{ + testing::{ + append_str, check_operations, unmark, witness, CombinedTree, Operation, TestHashable, + Tree, + }, + Hashable, Position, Retention, + }; + + use super::CachingShardStore; + use crate::{ + store::{memory::MemoryShardStore, ShardStore}, + ShardTree, + }; + + fn check_equal( + mut lhs: MemoryShardStore, + rhs: CachingShardStore>, + ) { + let rhs = rhs.flush().unwrap(); + assert_eq!(lhs.get_shard_roots(), rhs.get_shard_roots()); + for shard_root in lhs.get_shard_roots().unwrap() { + assert_eq!(lhs.get_shard(shard_root), rhs.get_shard(shard_root)); + } + assert_eq!( + lhs.checkpoint_count().unwrap(), + rhs.checkpoint_count().unwrap(), + ); + lhs.with_checkpoints( + lhs.checkpoint_count().unwrap(), + |checkpoint_id, lhs_checkpoint| { + let rhs_checkpoint = rhs.get_checkpoint(checkpoint_id).unwrap().unwrap(); + assert_eq!(lhs_checkpoint.tree_state, rhs_checkpoint.tree_state); + assert_eq!(lhs_checkpoint.marks_removed, rhs_checkpoint.marks_removed); + Ok(()) + }, + ) + .unwrap(); + } + + #[test] + fn root_hashes() { + use Retention::*; + + { + let mut lhs = MemoryShardStore::<_, u64>::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert_eq!( + tree.root(0).unwrap(), + String::combine_all(tree.depth(), &[]), + ); + assert!(tree.append(String::from_u64(0), Ephemeral)); + assert_eq!( + tree.root(0).unwrap(), + String::combine_all(tree.depth(), &[0]), + ); + assert!(tree.append(String::from_u64(1), Ephemeral)); + assert_eq!( + tree.root(0).unwrap(), + String::combine_all(tree.depth(), &[0, 1]), + ); + assert!(tree.append(String::from_u64(2), Ephemeral)); + assert_eq!( + tree.root(0).unwrap(), + String::combine_all(tree.depth(), &[0, 1, 2]), + ); + + check_equal(lhs, rhs); + } + + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut t = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert!(t.append( + String::from_u64(0), + Checkpoint { + id: 1, + is_marked: true, + }, + )); + for _ in 0..3 { + assert!(t.append(String::from_u64(0), Ephemeral)); + } + assert_eq!( + t.root(0).unwrap(), + String::combine_all(t.depth(), &[0, 0, 0, 0]) + ); + + check_equal(lhs, rhs); + } + } + + #[test] + fn append() { + use Retention::*; + + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert_eq!(tree.depth(), 4); + + // 16 appends should succeed + for i in 0..16 { + assert!(tree.append(String::from_u64(i), Ephemeral)); + assert_eq!(tree.current_position(), Some(Position::from(i))); + } + + // 17th append should fail + assert!(!tree.append(String::from_u64(16), Ephemeral)); + + check_equal(lhs, rhs); + } + + { + // The following checks a condition on state restoration in the case that an append fails. + // We want to ensure that a failed append does not cause a loss of information. + let ops = (0..17) + .map(|i| Operation::Append(String::from_u64(i), Ephemeral)) + .collect::>(); + + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + check_operations(tree, &ops).unwrap(); + check_equal(lhs, rhs); + } + } + + #[test] + fn check_witnesses() { + use Operation::{Append, Rewind, Witness}; + use Retention::*; + + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert!(tree.append(String::from_u64(0), Ephemeral)); + assert!(tree.append(String::from_u64(1), Marked)); + assert_eq!(tree.witness(Position::from(0), 0), None); + + check_equal(lhs, rhs); + } + + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert!(tree.append(String::from_u64(0), Marked)); + assert_eq!( + tree.witness(Position::from(0), 0), + Some(vec![ + String::empty_root(0.into()), + String::empty_root(1.into()), + String::empty_root(2.into()), + String::empty_root(3.into()) + ]) + ); + + assert!(tree.append(String::from_u64(1), Ephemeral)); + assert_eq!( + tree.witness(0.into(), 0), + Some(vec![ + String::from_u64(1), + String::empty_root(1.into()), + String::empty_root(2.into()), + String::empty_root(3.into()) + ]) + ); + + assert!(tree.append(String::from_u64(2), Marked)); + assert_eq!( + tree.witness(Position::from(2), 0), + Some(vec![ + String::empty_root(0.into()), + String::combine_all(1, &[0, 1]), + String::empty_root(2.into()), + String::empty_root(3.into()) + ]) + ); + + assert!(tree.append(String::from_u64(3), Ephemeral)); + assert_eq!( + tree.witness(Position::from(2), 0), + Some(vec![ + String::from_u64(3), + String::combine_all(1, &[0, 1]), + String::empty_root(2.into()), + String::empty_root(3.into()) + ]) + ); + + assert!(tree.append(String::from_u64(4), Ephemeral)); + assert_eq!( + tree.witness(Position::from(2), 0), + Some(vec![ + String::from_u64(3), + String::combine_all(1, &[0, 1]), + String::combine_all(2, &[4]), + String::empty_root(3.into()) + ]) + ); + + check_equal(lhs, rhs); + } + + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert!(tree.append(String::from_u64(0), Marked)); + for i in 1..6 { + assert!(tree.append(String::from_u64(i), Ephemeral)); + } + assert!(tree.append(String::from_u64(6), Marked)); + assert!(tree.append(String::from_u64(7), Ephemeral)); + + assert_eq!( + tree.witness(0.into(), 0), + Some(vec![ + String::from_u64(1), + String::combine_all(1, &[2, 3]), + String::combine_all(2, &[4, 5, 6, 7]), + String::empty_root(3.into()) + ]) + ); + + check_equal(lhs, rhs); + } + + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert!(tree.append(String::from_u64(0), Marked)); + assert!(tree.append(String::from_u64(1), Ephemeral)); + assert!(tree.append(String::from_u64(2), Ephemeral)); + assert!(tree.append(String::from_u64(3), Marked)); + assert!(tree.append(String::from_u64(4), Marked)); + assert!(tree.append(String::from_u64(5), Marked)); + assert!(tree.append(String::from_u64(6), Ephemeral)); + + assert_eq!( + tree.witness(Position::from(5), 0), + Some(vec![ + String::from_u64(4), + String::combine_all(1, &[6]), + String::combine_all(2, &[0, 1, 2, 3]), + String::empty_root(3.into()) + ]) + ); + + check_equal(lhs, rhs); + } + + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + for i in 0..10 { + assert!(tree.append(String::from_u64(i), Ephemeral)); + } + assert!(tree.append(String::from_u64(10), Marked)); + assert!(tree.append(String::from_u64(11), Ephemeral)); + + assert_eq!( + tree.witness(Position::from(10), 0), + Some(vec![ + String::from_u64(11), + String::combine_all(1, &[8, 9]), + String::empty_root(2.into()), + String::combine_all(3, &[0, 1, 2, 3, 4, 5, 6, 7]) + ]) + ); + + check_equal(lhs, rhs); + } + + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert!(tree.append( + String::from_u64(0), + Checkpoint { + id: 1, + is_marked: true, + }, + )); + assert!(tree.rewind()); + for i in 1..4 { + assert!(tree.append(String::from_u64(i), Ephemeral)); + } + assert!(tree.append(String::from_u64(4), Marked)); + for i in 5..8 { + assert!(tree.append(String::from_u64(i), Ephemeral)); + } + assert_eq!( + tree.witness(0.into(), 0), + Some(vec![ + String::from_u64(1), + String::combine_all(1, &[2, 3]), + String::combine_all(2, &[4, 5, 6, 7]), + String::empty_root(3.into()), + ]) + ); + + check_equal(lhs, rhs); + } + + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert!(tree.append(String::from_u64(0), Ephemeral)); + assert!(tree.append(String::from_u64(1), Ephemeral)); + assert!(tree.append(String::from_u64(2), Marked)); + assert!(tree.append(String::from_u64(3), Ephemeral)); + assert!(tree.append(String::from_u64(4), Ephemeral)); + assert!(tree.append(String::from_u64(5), Ephemeral)); + assert!(tree.append( + String::from_u64(6), + Checkpoint { + id: 1, + is_marked: true, + }, + )); + assert!(tree.append(String::from_u64(7), Ephemeral)); + assert!(tree.rewind()); + assert_eq!( + tree.witness(Position::from(2), 0), + Some(vec![ + String::from_u64(3), + String::combine_all(1, &[0, 1]), + String::combine_all(2, &[4, 5, 6]), + String::empty_root(3.into()) + ]) + ); + + check_equal(lhs, rhs); + } + + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + for i in 0..12 { + assert!(tree.append(String::from_u64(i), Ephemeral)); + } + assert!(tree.append(String::from_u64(12), Marked)); + assert!(tree.append(String::from_u64(13), Marked)); + assert!(tree.append(String::from_u64(14), Ephemeral)); + assert!(tree.append(String::from_u64(15), Ephemeral)); + + assert_eq!( + tree.witness(Position::from(12), 0), + Some(vec![ + String::from_u64(13), + String::combine_all(1, &[14, 15]), + String::combine_all(2, &[8, 9, 10, 11]), + String::combine_all(3, &[0, 1, 2, 3, 4, 5, 6, 7]), + ]) + ); + + check_equal(lhs, rhs); + } + + { + let ops = (0..=11) + .map(|i| Append(String::from_u64(i), Marked)) + .chain(Some(Append(String::from_u64(12), Ephemeral))) + .chain(Some(Append(String::from_u64(13), Ephemeral))) + .chain(Some(Witness(11u64.into(), 0))) + .collect::>(); + + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert_eq!( + Operation::apply_all(&ops, &mut tree), + Some(( + Position::from(11), + vec![ + String::from_u64(10), + String::combine_all(1, &[8, 9]), + String::combine_all(2, &[12, 13]), + String::combine_all(3, &[0, 1, 2, 3, 4, 5, 6, 7]), + ] + )) + ); + + check_equal(lhs, rhs); + } + + { + let ops = vec![ + Append(String::from_u64(0), Ephemeral), + Append(String::from_u64(1), Ephemeral), + Append(String::from_u64(2), Ephemeral), + Append( + String::from_u64(3), + Checkpoint { + id: 1, + is_marked: true, + }, + ), + Append(String::from_u64(4), Marked), + Operation::Checkpoint(2), + Append( + String::from_u64(5), + Checkpoint { + id: 3, + is_marked: false, + }, + ), + Append( + String::from_u64(6), + Checkpoint { + id: 4, + is_marked: false, + }, + ), + Append( + String::from_u64(7), + Checkpoint { + id: 5, + is_marked: false, + }, + ), + Witness(3u64.into(), 5), + ]; + + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert_eq!( + Operation::apply_all(&ops, &mut tree), + Some(( + Position::from(3), + vec![ + String::from_u64(2), + String::combine_all(1, &[0, 1]), + String::combine_all(2, &[]), + String::combine_all(3, &[]), + ] + )) + ); + + check_equal(lhs, rhs); + } + + { + let ops = vec![ + Append(String::from_u64(0), Ephemeral), + Append(String::from_u64(0), Ephemeral), + Append(String::from_u64(0), Ephemeral), + Append( + String::from_u64(0), + Checkpoint { + id: 1, + is_marked: true, + }, + ), + Append(String::from_u64(0), Ephemeral), + Append(String::from_u64(0), Ephemeral), + Append(String::from_u64(0), Ephemeral), + Append( + String::from_u64(0), + Checkpoint { + id: 2, + is_marked: false, + }, + ), + Append(String::from_u64(0), Ephemeral), + Append(String::from_u64(0), Ephemeral), + Witness(Position::from(3), 1), + ]; + + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert_eq!( + Operation::apply_all(&ops, &mut tree), + Some(( + Position::from(3), + vec![ + String::from_u64(0), + String::combine_all(1, &[0, 0]), + String::combine_all(2, &[0, 0, 0, 0]), + String::combine_all(3, &[]), + ] + )) + ); + + check_equal(lhs, rhs); + } + + { + let ops = vec![ + Append(String::from_u64(0), Marked), + Append(String::from_u64(0), Ephemeral), + Append(String::from_u64(0), Ephemeral), + Append(String::from_u64(0), Ephemeral), + Append(String::from_u64(0), Ephemeral), + Append(String::from_u64(0), Ephemeral), + Append(String::from_u64(0), Ephemeral), + Operation::Checkpoint(1), + Append(String::from_u64(0), Marked), + Operation::Checkpoint(2), + Operation::Checkpoint(3), + Append( + String::from_u64(0), + Checkpoint { + id: 4, + is_marked: false, + }, + ), + Rewind, + Rewind, + Witness(Position::from(7), 2), + ]; + + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert_eq!(Operation::apply_all(&ops, &mut tree), None); + + check_equal(lhs, rhs); + } + + { + let ops = vec![ + Append(String::from_u64(0), Marked), + Append(String::from_u64(0), Ephemeral), + Append( + String::from_u64(0), + Checkpoint { + id: 1, + is_marked: true, + }, + ), + Append( + String::from_u64(0), + Checkpoint { + id: 4, + is_marked: false, + }, + ), + Witness(Position::from(2), 2), + ]; + + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert_eq!( + Operation::apply_all(&ops, &mut tree), + Some(( + Position::from(2), + vec![ + String::empty_leaf(), + String::combine_all(1, &[0, 0]), + String::combine_all(2, &[]), + String::combine_all(3, &[]), + ] + )) + ); + + check_equal(lhs, rhs); + } + } + + #[test] + fn check_checkpoint_rewind() { + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut t = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert!(!t.rewind()); + + check_equal(lhs, rhs); + } + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut t = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + assert!(t.checkpoint(1)); + assert!(t.rewind()); + + check_equal(lhs, rhs); + } + + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut t = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + t.append("a".to_string(), Retention::Ephemeral); + assert!(t.checkpoint(1)); + t.append("b".to_string(), Retention::Marked); + assert!(t.rewind()); + assert_eq!(Some(Position::from(0)), t.current_position()); + + check_equal(lhs, rhs); + } + + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut t = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + t.append("a".to_string(), Retention::Marked); + assert!(t.checkpoint(1)); + assert!(t.rewind()); + + check_equal(lhs, rhs); + } + + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut t = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + t.append("a".to_string(), Retention::Marked); + assert!(t.checkpoint(1)); + t.append("a".to_string(), Retention::Ephemeral); + assert!(t.rewind()); + assert_eq!(Some(Position::from(0)), t.current_position()); + + check_equal(lhs, rhs); + } + + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut t = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + t.append("a".to_string(), Retention::Ephemeral); + assert!(t.checkpoint(1)); + assert!(t.checkpoint(2)); + assert!(t.rewind()); + t.append("b".to_string(), Retention::Ephemeral); + assert!(t.rewind()); + t.append("b".to_string(), Retention::Ephemeral); + assert_eq!(t.root(0).unwrap(), "ab______________"); + + check_equal(lhs, rhs); + } + } + + #[test] + fn check_remove_mark() { + let samples = vec![ + vec![ + append_str("a", Retention::Ephemeral), + append_str( + "a", + Retention::Checkpoint { + id: 1, + is_marked: true, + }, + ), + witness(1, 1), + ], + vec![ + append_str("a", Retention::Ephemeral), + append_str("a", Retention::Ephemeral), + append_str("a", Retention::Ephemeral), + append_str("a", Retention::Marked), + Operation::Checkpoint(1), + unmark(3), + witness(3, 0), + ], + ]; + + for (i, sample) in samples.iter().enumerate() { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + let result = check_operations(tree, sample); + assert!( + matches!(result, Ok(())), + "Reference/Test mismatch at index {}: {:?}", + i, + result + ); + + check_equal(lhs, rhs); + } + } + + #[test] + fn check_rewind_remove_mark() { + use Operation::*; + + // rewinding doesn't remove a mark + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + tree.append("e".to_string(), Retention::Marked); + assert!(tree.checkpoint(1)); + assert!(tree.rewind()); + assert!(tree.remove_mark(0u64.into())); + + check_equal(lhs, rhs); + } + + // use a maximum number of checkpoints of 1 + { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let mut tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 1), + ShardTree::<_, 4, 3>::new(&mut rhs, 1), + ); + + assert!(tree.append("e".to_string(), Retention::Marked)); + assert!(tree.checkpoint(1)); + assert!(tree.marked_positions().contains(&0u64.into())); + assert!(tree.append("f".to_string(), Retention::Ephemeral)); + // simulate a spend of `e` at `f` + assert!(tree.remove_mark(0u64.into())); + // even though the mark has been staged for removal, it's not gone yet + assert!(tree.marked_positions().contains(&0u64.into())); + assert!(tree.checkpoint(2)); + // the newest checkpoint will have caused the oldest to roll off, and + // so the forgotten node will be unmarked + assert!(!tree.marked_positions().contains(&0u64.into())); + assert!(!tree.remove_mark(0u64.into())); + + check_equal(lhs, rhs); + } + + // The following check_operations tests cover errors where the + // test framework itself previously did not correctly handle + // chain state restoration. + + let samples = vec![ + vec![ + append_str("x", Retention::Marked), + Checkpoint(1), + Rewind, + unmark(0), + ], + vec![ + append_str("d", Retention::Marked), + Checkpoint(1), + unmark(0), + Rewind, + unmark(0), + ], + vec![ + append_str("o", Retention::Marked), + Checkpoint(1), + Checkpoint(2), + unmark(0), + Rewind, + Rewind, + ], + vec![ + append_str("s", Retention::Marked), + append_str("m", Retention::Ephemeral), + Checkpoint(1), + unmark(0), + Rewind, + unmark(0), + unmark(0), + ], + vec![ + append_str("a", Retention::Marked), + Checkpoint(1), + Rewind, + append_str("a", Retention::Marked), + ], + ]; + + for (i, sample) in samples.iter().enumerate() { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + let result = check_operations(tree, sample); + assert!( + matches!(result, Ok(())), + "Reference/Test mismatch at index {}: {:?}", + i, + result + ); + + check_equal(lhs, rhs); + } + } + + #[test] + fn check_witness_consistency() { + use Operation::*; + + let samples = vec![ + // Reduced examples + vec![ + append_str("a", Retention::Ephemeral), + append_str("b", Retention::Marked), + Checkpoint(1), + witness(0, 1), + ], + vec![ + append_str("c", Retention::Ephemeral), + append_str("d", Retention::Marked), + Checkpoint(1), + witness(1, 1), + ], + vec![ + append_str("e", Retention::Marked), + Checkpoint(1), + append_str("f", Retention::Ephemeral), + witness(0, 1), + ], + vec![ + append_str("g", Retention::Marked), + Checkpoint(1), + unmark(0), + append_str("h", Retention::Ephemeral), + witness(0, 0), + ], + vec![ + append_str("i", Retention::Marked), + Checkpoint(1), + unmark(0), + append_str("j", Retention::Ephemeral), + witness(0, 0), + ], + vec![ + append_str("i", Retention::Marked), + append_str("j", Retention::Ephemeral), + Checkpoint(1), + append_str("k", Retention::Ephemeral), + witness(0, 1), + ], + vec![ + append_str("l", Retention::Marked), + Checkpoint(1), + Checkpoint(2), + append_str("m", Retention::Ephemeral), + Checkpoint(3), + witness(0, 2), + ], + vec![ + Checkpoint(1), + append_str("n", Retention::Marked), + witness(0, 1), + ], + vec![ + append_str("a", Retention::Marked), + Checkpoint(1), + unmark(0), + Checkpoint(2), + append_str("b", Retention::Ephemeral), + witness(0, 1), + ], + vec![ + append_str("a", Retention::Marked), + append_str("b", Retention::Ephemeral), + unmark(0), + Checkpoint(1), + witness(0, 0), + ], + vec![ + append_str("a", Retention::Marked), + Checkpoint(1), + unmark(0), + Checkpoint(2), + Rewind, + append_str("b", Retention::Ephemeral), + witness(0, 0), + ], + vec![ + append_str("a", Retention::Marked), + Checkpoint(1), + Checkpoint(2), + Rewind, + append_str("a", Retention::Ephemeral), + unmark(0), + witness(0, 1), + ], + // Unreduced examples + vec![ + append_str("o", Retention::Ephemeral), + append_str("p", Retention::Marked), + append_str("q", Retention::Ephemeral), + Checkpoint(1), + unmark(1), + witness(1, 1), + ], + vec![ + append_str("r", Retention::Ephemeral), + append_str("s", Retention::Ephemeral), + append_str("t", Retention::Marked), + Checkpoint(1), + unmark(2), + Checkpoint(2), + witness(2, 2), + ], + vec![ + append_str("u", Retention::Marked), + append_str("v", Retention::Ephemeral), + append_str("w", Retention::Ephemeral), + Checkpoint(1), + unmark(0), + append_str("x", Retention::Ephemeral), + Checkpoint(2), + Checkpoint(3), + witness(0, 3), + ], + ]; + + for (i, sample) in samples.iter().enumerate() { + let mut lhs = MemoryShardStore::empty(); + let mut rhs = CachingShardStore::load(MemoryShardStore::empty()).unwrap(); + let tree = CombinedTree::new( + ShardTree::<_, 4, 3>::new(&mut lhs, 100), + ShardTree::<_, 4, 3>::new(&mut rhs, 100), + ); + + let result = check_operations(tree, sample); + assert!( + matches!(result, Ok(())), + "Reference/Test mismatch at index {}: {:?}", + i, + result + ); + + check_equal(lhs, rhs); + } + } +} diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/store/memory.rs b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/store/memory.rs new file mode 100644 index 000000000000..aa16f52e4e6f --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/store/memory.rs @@ -0,0 +1,165 @@ +//! Implementation of an in-memory shard store with no persistence. + +use std::collections::BTreeMap; +use std::convert::{Infallible, TryFrom}; + +use incrementalmerkletree::Address; + +use super::{Checkpoint, ShardStore}; +use crate::{LocatedPrunableTree, LocatedTree, Node, PrunableTree, Tree}; + +/// An implementation of [`ShardStore`] that stores all state in memory. +/// +/// State is not persisted anywhere, and will be lost when the struct is dropped. +#[derive(Debug)] +pub struct MemoryShardStore { + shards: Vec>, + checkpoints: BTreeMap, + cap: PrunableTree, +} + +impl MemoryShardStore { + /// Constructs a new empty `MemoryShardStore`. + pub fn empty() -> Self { + Self { + shards: vec![], + checkpoints: BTreeMap::new(), + cap: PrunableTree::empty(), + } + } +} + +impl ShardStore for MemoryShardStore { + type H = H; + type CheckpointId = C; + type Error = Infallible; + + fn get_shard( + &self, + shard_root: Address, + ) -> Result>, Self::Error> { + let shard_idx = + usize::try_from(shard_root.index()).expect("SHARD_HEIGHT > 64 is unsupported"); + Ok(self.shards.get(shard_idx).cloned()) + } + + fn last_shard(&self) -> Result>, Self::Error> { + Ok(self.shards.last().cloned()) + } + + fn put_shard(&mut self, subtree: LocatedPrunableTree) -> Result<(), Self::Error> { + let subtree_addr = subtree.root_addr; + for subtree_idx in + self.shards.last().map_or(0, |s| s.root_addr.index() + 1)..=subtree_addr.index() + { + self.shards.push(LocatedTree { + root_addr: Address::from_parts(subtree_addr.level(), subtree_idx), + root: Tree(Node::Nil), + }) + } + + let shard_idx = + usize::try_from(subtree_addr.index()).expect("SHARD_HEIGHT > 64 is unsupported"); + self.shards[shard_idx] = subtree; + Ok(()) + } + + fn get_shard_roots(&self) -> Result, Self::Error> { + Ok(self.shards.iter().map(|s| s.root_addr).collect()) + } + + fn truncate(&mut self, from: Address) -> Result<(), Self::Error> { + let shard_idx = usize::try_from(from.index()).expect("SHARD_HEIGHT > 64 is unsupported"); + self.shards.truncate(shard_idx); + Ok(()) + } + + fn get_cap(&self) -> Result, Self::Error> { + Ok(self.cap.clone()) + } + + fn put_cap(&mut self, cap: PrunableTree) -> Result<(), Self::Error> { + self.cap = cap; + Ok(()) + } + + fn add_checkpoint( + &mut self, + checkpoint_id: C, + checkpoint: Checkpoint, + ) -> Result<(), Self::Error> { + self.checkpoints.insert(checkpoint_id, checkpoint); + Ok(()) + } + + fn checkpoint_count(&self) -> Result { + Ok(self.checkpoints.len()) + } + + fn get_checkpoint( + &self, + checkpoint_id: &Self::CheckpointId, + ) -> Result, Self::Error> { + Ok(self.checkpoints.get(checkpoint_id).cloned()) + } + + fn get_checkpoint_at_depth( + &self, + checkpoint_depth: usize, + ) -> Result, Self::Error> { + Ok(if checkpoint_depth == 0 { + None + } else { + self.checkpoints + .iter() + .rev() + .nth(checkpoint_depth - 1) + .map(|(id, c)| (id.clone(), c.clone())) + }) + } + + fn min_checkpoint_id(&self) -> Result, Self::Error> { + Ok(self.checkpoints.keys().next().cloned()) + } + + fn max_checkpoint_id(&self) -> Result, Self::Error> { + Ok(self.checkpoints.keys().last().cloned()) + } + + fn with_checkpoints(&mut self, limit: usize, mut callback: F) -> Result<(), Self::Error> + where + F: FnMut(&C, &Checkpoint) -> Result<(), Self::Error>, + { + for (cid, checkpoint) in self.checkpoints.iter().take(limit) { + callback(cid, checkpoint)? + } + + Ok(()) + } + + fn update_checkpoint_with( + &mut self, + checkpoint_id: &C, + update: F, + ) -> Result + where + F: Fn(&mut Checkpoint) -> Result<(), Self::Error>, + { + if let Some(c) = self.checkpoints.get_mut(checkpoint_id) { + update(c)?; + return Ok(true); + } + + Ok(false) + } + + fn remove_checkpoint(&mut self, checkpoint_id: &C) -> Result<(), Self::Error> { + self.checkpoints.remove(checkpoint_id); + Ok(()) + } + + fn truncate_checkpoints(&mut self, checkpoint_id: &C) -> Result<(), Self::Error> { + self.checkpoints.split_off(checkpoint_id); + Ok(()) + } +} diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/testing.rs b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/testing.rs new file mode 100644 index 000000000000..4771a663983d --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/testing.rs @@ -0,0 +1,413 @@ +use std::convert::TryFrom; + +use assert_matches::assert_matches; +use proptest::bool::weighted; +use proptest::collection::vec; +use proptest::prelude::*; +use proptest::sample::select; + +use incrementalmerkletree::{testing, Hashable}; + +use super::*; +use crate::store::{memory::MemoryShardStore, ShardStore}; + +pub fn arb_retention_flags() -> impl Strategy + Clone { + select(vec![ + RetentionFlags::EPHEMERAL, + RetentionFlags::CHECKPOINT, + RetentionFlags::MARKED, + RetentionFlags::MARKED | RetentionFlags::CHECKPOINT, + ]) +} + +pub fn arb_tree( + arb_annotation: A, + arb_leaf: V, + depth: u32, + size: u32, +) -> impl Strategy> + Clone +where + A::Value: Clone + 'static, + V::Value: Clone + 'static, +{ + let leaf = prop_oneof![ + Just(Tree(Node::Nil)), + arb_leaf.prop_map(|value| Tree(Node::Leaf { value })) + ]; + + leaf.prop_recursive(depth, size, 2, move |inner| { + (arb_annotation.clone(), inner.clone(), inner).prop_map(|(ann, left, right)| { + Tree(if left.is_nil() && right.is_nil() { + Node::Nil + } else { + Node::Parent { + ann, + left: Arc::new(left), + right: Arc::new(right), + } + }) + }) + }) +} + +pub fn arb_prunable_tree( + arb_leaf: H, + depth: u32, + size: u32, +) -> impl Strategy> + Clone +where + H::Value: Clone + 'static, +{ + arb_tree( + proptest::option::of(arb_leaf.clone().prop_map(Arc::new)), + (arb_leaf, arb_retention_flags()), + depth, + size, + ) +} + +/// Constructs a random sequence of leaves that form a tree of size up to 2^6. +pub fn arb_leaves( + arb_leaf: H, +) -> impl Strategy)>> +where + H::Value: Hashable + Clone + PartialEq, +{ + vec( + (arb_leaf, weighted(0.1), weighted(0.2)), + 0..=(2usize.pow(6)), + ) + .prop_map(|leaves| { + leaves + .into_iter() + .enumerate() + .map(|(id, (leaf, is_marked, is_checkpoint))| { + ( + leaf, + match (is_checkpoint, is_marked) { + (false, false) => Retention::Ephemeral, + (true, is_marked) => Retention::Checkpoint { id, is_marked }, + (false, true) => Retention::Marked, + }, + ) + }) + .collect() + }) +} + +/// Constructs a random shardtree of size up to 2^6 with shards of size 2^3. Returns the tree, +/// along with vectors of the checkpoint and mark positions. +pub fn arb_shardtree( + arb_leaf: H, +) -> impl Strategy< + Value = ( + ShardTree, 6, 3>, + Vec, + Vec, + ), +> +where + H::Value: Hashable + Clone + PartialEq, +{ + arb_leaves(arb_leaf).prop_map(|leaves| { + let mut tree = ShardTree::new(MemoryShardStore::empty(), 10); + let mut checkpoint_positions = vec![]; + let mut marked_positions = vec![]; + tree.batch_insert( + Position::from(0), + leaves + .into_iter() + .enumerate() + .map(|(id, (leaf, retention))| { + let pos = Position::try_from(id).unwrap(); + match retention { + Retention::Ephemeral => (), + Retention::Checkpoint { is_marked, .. } => { + checkpoint_positions.push(pos); + if is_marked { + marked_positions.push(pos); + } + } + Retention::Marked => marked_positions.push(pos), + } + (leaf, retention) + }), + ) + .unwrap(); + (tree, checkpoint_positions, marked_positions) + }) +} + +pub fn arb_char_str() -> impl Strategy + Clone { + let chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + (0usize..chars.len()).prop_map(move |i| chars.get(i..=i).unwrap().to_string()) +} + +impl< + H: Hashable + Ord + Clone + core::fmt::Debug, + C: Clone + Ord + core::fmt::Debug, + S: ShardStore, + const DEPTH: u8, + const SHARD_HEIGHT: u8, + > testing::Tree for ShardTree +{ + fn depth(&self) -> u8 { + DEPTH + } + + fn append(&mut self, value: H, retention: Retention) -> bool { + match ShardTree::append(self, value, retention) { + Ok(_) => true, + Err(ShardTreeError::Insert(InsertionError::TreeFull)) => false, + Err(other) => panic!("append failed due to error: {:?}", other), + } + } + + fn current_position(&self) -> Option { + match ShardTree::max_leaf_position(self, 0) { + Ok(v) => v, + Err(err) => panic!("current position query failed: {:?}", err), + } + } + + fn get_marked_leaf(&self, position: Position) -> Option { + match ShardTree::get_marked_leaf(self, position) { + Ok(v) => v, + Err(err) => panic!("marked leaf query failed: {:?}", err), + } + } + + fn marked_positions(&self) -> BTreeSet { + match ShardTree::marked_positions(self) { + Ok(v) => v, + Err(err) => panic!("marked positions query failed: {:?}", err), + } + } + + fn root(&self, checkpoint_depth: usize) -> Option { + match ShardTree::root_at_checkpoint_depth(self, checkpoint_depth) { + Ok(v) => Some(v), + Err(err) => panic!("root computation failed: {:?}", err), + } + } + + fn witness(&self, position: Position, checkpoint_depth: usize) -> Option> { + match ShardTree::witness_at_checkpoint_depth(self, position, checkpoint_depth) { + Ok(p) => Some(p.path_elems().to_vec()), + Err(ShardTreeError::Query( + QueryError::NotContained(_) + | QueryError::TreeIncomplete(_) + | QueryError::CheckpointPruned, + )) => None, + Err(err) => panic!("witness computation failed: {:?}", err), + } + } + + fn remove_mark(&mut self, position: Position) -> bool { + let max_checkpoint = self + .store + .max_checkpoint_id() + .unwrap_or_else(|err| panic!("checkpoint retrieval failed: {:?}", err)); + + match ShardTree::remove_mark(self, position, max_checkpoint.as_ref()) { + Ok(result) => result, + Err(err) => panic!("mark removal failed: {:?}", err), + } + } + + fn checkpoint(&mut self, checkpoint_id: C) -> bool { + ShardTree::checkpoint(self, checkpoint_id).unwrap() + } + + fn rewind(&mut self) -> bool { + ShardTree::truncate_to_depth(self, 1).unwrap() + } +} + +pub fn check_shardtree_insertion< + E: Debug, + S: ShardStore, +>( + mut tree: ShardTree, +) { + assert_matches!( + tree.batch_insert( + Position::from(1), + vec![ + ("b".to_string(), Retention::Checkpoint { id: 1, is_marked: false }), + ("c".to_string(), Retention::Ephemeral), + ("d".to_string(), Retention::Marked), + ].into_iter() + ), + Ok(Some((pos, incomplete))) if + pos == Position::from(3) && + incomplete == vec![ + IncompleteAt { + address: Address::from_parts(Level::from(0), 0), + required_for_witness: true + }, + IncompleteAt { + address: Address::from_parts(Level::from(2), 1), + required_for_witness: true + } + ] + ); + + assert_matches!( + tree.root_at_checkpoint_depth(1), + Err(ShardTreeError::Query(QueryError::TreeIncomplete(v))) if v == vec![Address::from_parts(Level::from(0), 0)] + ); + + assert_matches!( + tree.batch_insert( + Position::from(0), + vec![ + ("a".to_string(), Retention::Ephemeral), + ].into_iter() + ), + Ok(Some((pos, incomplete))) if + pos == Position::from(0) && + incomplete == vec![] + ); + + assert_matches!( + tree.root_at_checkpoint_depth(0), + Ok(h) if h == *"abcd____________" + ); + + assert_matches!( + tree.root_at_checkpoint_depth(1), + Ok(h) if h == *"ab______________" + ); + + assert_matches!( + tree.batch_insert( + Position::from(10), + vec![ + ("k".to_string(), Retention::Ephemeral), + ("l".to_string(), Retention::Checkpoint { id: 2, is_marked: false }), + ("m".to_string(), Retention::Ephemeral), + ].into_iter() + ), + Ok(Some((pos, incomplete))) if + pos == Position::from(12) && + incomplete == vec![ + IncompleteAt { + address: Address::from_parts(Level::from(0), 13), + required_for_witness: false + }, + IncompleteAt { + address: Address::from_parts(Level::from(1), 7), + required_for_witness: false + }, + IncompleteAt { + address: Address::from_parts(Level::from(1), 4), + required_for_witness: false + }, + ] + ); + + assert_matches!( + tree.root_at_checkpoint_depth(0), + // The (0, 13) and (1, 7) incomplete subtrees are + // not considered incomplete here because they appear + // at the tip of the tree. + Err(ShardTreeError::Query(QueryError::TreeIncomplete(xs))) if xs == vec![ + Address::from_parts(Level::from(2), 1), + Address::from_parts(Level::from(1), 4), + ] + ); + + assert_matches!(tree.truncate_to_depth(1), Ok(true)); + + assert_matches!( + tree.batch_insert( + Position::from(4), + ('e'..'k') + .into_iter() + .map(|c| (c.to_string(), Retention::Ephemeral)) + ), + Ok(_) + ); + + assert_matches!( + tree.root_at_checkpoint_depth(0), + Ok(h) if h == *"abcdefghijkl____" + ); + + assert_matches!( + tree.root_at_checkpoint_depth(1), + Ok(h) if h == *"ab______________" + ); +} + +pub fn check_shard_sizes>( + mut tree: ShardTree, +) { + for c in 'a'..'p' { + tree.append(c.to_string(), Retention::Ephemeral).unwrap(); + } + + assert_eq!(tree.store.get_shard_roots().unwrap().len(), 4); + assert_eq!( + tree.store + .get_shard(Address::from_parts(Level::from(2), 3)) + .unwrap() + .and_then(|t| t.max_position()), + Some(Position::from(14)) + ); +} + +pub fn check_witness_with_pruned_subtrees< + E: Debug, + S: ShardStore, +>( + mut tree: ShardTree, +) { + // introduce some roots + let shard_root_level = Level::from(3); + for idx in 0u64..4 { + let root = if idx == 3 { + "abcdefgh".to_string() + } else { + idx.to_string() + }; + tree.insert(Address::from_parts(shard_root_level, idx), root) + .unwrap(); + } + + // simulate discovery of a note + tree.batch_insert( + Position::from(24), + ('a'..='h').into_iter().map(|c| { + ( + c.to_string(), + match c { + 'c' => Retention::Marked, + 'h' => Retention::Checkpoint { + id: 3, + is_marked: false, + }, + _ => Retention::Ephemeral, + }, + ) + }), + ) + .unwrap(); + + // construct a witness for the note + let witness = tree + .witness_at_checkpoint_depth(Position::from(26), 0) + .unwrap(); + assert_eq!( + witness.path_elems(), + &[ + "d", + "ab", + "efgh", + "2", + "01", + "________________________________" + ] + ); +} diff --git a/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/tree.rs b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/tree.rs new file mode 100644 index 000000000000..5a87eae25ce5 --- /dev/null +++ b/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/tree.rs @@ -0,0 +1,429 @@ +//! The core tree types. + +use std::ops::Deref; +use std::sync::Arc; + +use incrementalmerkletree::{Address, Level, Position}; + +/// A "pattern functor" for a single layer of a binary tree. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Node { + /// A parent node in the tree, annotated with a value of type `A` and with left and right + /// children of type `C`. + Parent { ann: A, left: C, right: C }, + /// A node of the tree that contains a value (usually a hash, sometimes with additional + /// metadata) and that has no children. + /// + /// Note that leaf nodes may appear at any position in the tree; i.e. they may contain computed + /// subtree root values and not just level-0 leaves. + Leaf { value: V }, + /// The empty tree; a subtree or leaf for which no information is available. + Nil, +} + +impl Node { + /// Returns whether or not this is the `Nil` tree. + /// + /// This is useful for cases where the compiler can automatically dereference an [`Arc`], where + /// one would otherwise need additional ceremony to make an equality check. + pub fn is_nil(&self) -> bool { + matches!(self, Node::Nil) + } + + /// Returns the contained leaf value, if this is a leaf node. + pub fn leaf_value(&self) -> Option<&V> { + match self { + Node::Parent { .. } => None, + Node::Leaf { value } => Some(value), + Node::Nil { .. } => None, + } + } + + /// Returns the annotation, if this is a parent node. + pub fn annotation(&self) -> Option<&A> { + match self { + Node::Parent { ann, .. } => Some(ann), + Node::Leaf { .. } => None, + Node::Nil => None, + } + } + + /// Replaces the annotation on this node, if it is a [`Node::Parent`]; otherwise + /// returns this node unaltered. + pub fn reannotate(self, ann: A) -> Self { + match self { + Node::Parent { left, right, .. } => Node::Parent { ann, left, right }, + other => other, + } + } +} + +impl<'a, C: Clone, A: Clone, V: Clone> Node { + /// Maps a `Node` to a `Node` by cloning the contents of the node. + pub fn cloned(&self) -> Node { + match self { + Node::Parent { ann, left, right } => Node::Parent { + ann: (*ann).clone(), + left: left.clone(), + right: right.clone(), + }, + Node::Leaf { value } => Node::Leaf { + value: (*value).clone(), + }, + Node::Nil => Node::Nil, + } + } +} + +/// An immutable binary tree with each of its nodes tagged with an annotation value. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Tree(pub(crate) Node>, A, V>); + +impl Deref for Tree { + type Target = Node>, A, V>; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Tree { + /// Constructs the empty tree. + pub fn empty() -> Self { + Tree(Node::Nil) + } + + /// Constructs a tree containing a single leaf. + pub fn leaf(value: V) -> Self { + Tree(Node::Leaf { value }) + } + + /// Constructs a tree containing a pair of leaves. + pub fn parent(ann: A, left: Self, right: Self) -> Self { + Tree(Node::Parent { + ann, + left: Arc::new(left), + right: Arc::new(right), + }) + } + + /// Returns `true` if the tree has no leaves. + pub fn is_empty(&self) -> bool { + self.0.is_nil() + } + + /// Replaces the annotation at the root of the tree, if the root is a [`Node::Parent`]; + /// otherwise returns this tree unaltered. + pub fn reannotate_root(self, ann: A) -> Self { + Tree(self.0.reannotate(ann)) + } + + /// Returns `true` if no [`Node::Nil`] nodes are present in the tree, `false` otherwise. + pub fn is_complete(&self) -> bool { + match &self.0 { + Node::Parent { left, right, .. } => { + left.as_ref().is_complete() && right.as_ref().is_complete() + } + Node::Leaf { .. } => true, + Node::Nil { .. } => false, + } + } + + /// Returns a vector of the addresses of [`Node::Nil`] subtree roots within this tree. + /// + /// The given address must correspond to the root of this tree, or this method will + /// yield incorrect results or may panic. + pub fn incomplete_nodes(&self, root_addr: Address) -> Vec
{ + match &self.0 { + Node::Parent { left, right, .. } => { + // We should never construct parent nodes where both children are Nil. + // While we could handle that here, if we encountered that case it would + // be indicative of a programming error elsewhere and so we assert instead. + assert!(!(left.0.is_nil() && right.0.is_nil())); + let (left_root, right_root) = root_addr + .children() + .expect("A parent node cannot appear at level 0"); + + let mut left_incomplete = left.incomplete_nodes(left_root); + let mut right_incomplete = right.incomplete_nodes(right_root); + left_incomplete.append(&mut right_incomplete); + left_incomplete + } + Node::Leaf { .. } => vec![], + Node::Nil => vec![root_addr], + } + } +} + +/// A binary Merkle tree with its root at the given address. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct LocatedTree { + pub(crate) root_addr: Address, + pub(crate) root: Tree, +} + +impl LocatedTree { + /// Constructs a new LocatedTree from its constituent parts + pub fn from_parts(root_addr: Address, root: Tree) -> Self { + LocatedTree { root_addr, root } + } + + /// Returns the root address of this tree. + pub fn root_addr(&self) -> Address { + self.root_addr + } + + /// Returns a reference to the root of the tree. + pub fn root(&self) -> &Tree { + &self.root + } + + /// Consumes this tree and returns its root as an owned value. + pub fn take_root(self) -> Tree { + self.root + } + + /// Returns a new [`LocatedTree`] with the provided value replacing the annotation of its root + /// node, if that root node is a [`Node::Parent`]. Otherwise returns this tree unaltered. + pub fn reannotate_root(self, value: A) -> Self { + LocatedTree { + root_addr: self.root_addr, + root: self.root.reannotate_root(value), + } + } + + /// Returns the set of incomplete subtree roots contained within this tree, ordered by + /// increasing position. + pub fn incomplete_nodes(&self) -> Vec
{ + self.root.incomplete_nodes(self.root_addr) + } + + /// Returns the maximum position at which a non-`Nil` leaf has been observed in the tree. + /// + /// Note that no actual leaf value may exist at this position, as it may have previously been + /// pruned. + pub fn max_position(&self) -> Option { + fn go(addr: Address, root: &Tree) -> Option { + match &root.0 { + Node::Nil => None, + Node::Leaf { .. } => Some(addr.position_range_end() - 1), + Node::Parent { left, right, .. } => { + let (l_addr, r_addr) = addr.children().unwrap(); + go(r_addr, right.as_ref()).or_else(|| go(l_addr, left.as_ref())) + } + } + } + + go(self.root_addr, &self.root) + } + + /// Returns the value at the specified position, if any. + pub fn value_at_position(&self, position: Position) -> Option<&V> { + fn go(pos: Position, addr: Address, root: &Tree) -> Option<&V> { + match &root.0 { + Node::Parent { left, right, .. } => { + let (l_addr, r_addr) = addr.children().unwrap(); + if l_addr.position_range().contains(&pos) { + go(pos, l_addr, left) + } else { + go(pos, r_addr, right) + } + } + Node::Leaf { value } if addr.level() == Level::from(0) => Some(value), + _ => None, + } + } + + if self.root_addr.position_range().contains(&position) { + go(position, self.root_addr, &self.root) + } else { + None + } + } +} + +impl LocatedTree { + /// Constructs a new empty tree with its root at the provided address. + pub fn empty(root_addr: Address) -> Self { + Self { + root_addr, + root: Tree(Node::Nil), + } + } + + /// Constructs a new tree consisting of a single leaf with the provided value, and the + /// specified root address. + pub fn with_root_value(root_addr: Address, value: V) -> Self { + Self { + root_addr, + root: Tree(Node::Leaf { value }), + } + } + + /// Traverses this tree to find the child node at the specified address and returns it. + /// + /// Returns `None` if the specified address is not a descendant of this tree's root address, or + /// if the tree is terminated by a [`Node::Nil`] or leaf node before the specified address can + /// be reached. + pub fn subtree(&self, addr: Address) -> Option { + fn go( + root_addr: Address, + root: &Tree, + addr: Address, + ) -> Option> { + if root_addr == addr { + Some(LocatedTree { + root_addr, + root: root.clone(), + }) + } else { + match &root.0 { + Node::Parent { left, right, .. } => { + let (l_addr, r_addr) = root_addr.children().unwrap(); + if l_addr.contains(&addr) { + go(l_addr, left.as_ref(), addr) + } else { + go(r_addr, right.as_ref(), addr) + } + } + _ => None, + } + } + } + + if self.root_addr.contains(&addr) { + go(self.root_addr, &self.root, addr) + } else { + None + } + } + + /// Decomposes this tree into the vector of its subtrees having height `level + 1`. + /// + /// If this root address of this tree is lower down in the tree than the level specified, + /// the entire tree is returned as the sole element of the result vector. + pub fn decompose_to_level(self, level: Level) -> Vec { + fn go( + level: Level, + root_addr: Address, + root: Tree, + ) -> Vec> { + if root_addr.level() == level { + vec![LocatedTree { root_addr, root }] + } else { + match root.0 { + Node::Parent { left, right, .. } => { + let (l_addr, r_addr) = root_addr.children().unwrap(); + let mut l_decomposed = go( + level, + l_addr, + Arc::try_unwrap(left).unwrap_or_else(|rc| (*rc).clone()), + ); + let mut r_decomposed = go( + level, + r_addr, + Arc::try_unwrap(right).unwrap_or_else(|rc| (*rc).clone()), + ); + l_decomposed.append(&mut r_decomposed); + l_decomposed + } + _ => vec![], + } + } + } + + if level >= self.root_addr.level() { + vec![self] + } else { + go(level, self.root_addr, self.root) + } + } +} + +#[cfg(test)] +pub(crate) mod tests { + use incrementalmerkletree::{Address, Level}; + + use super::{LocatedTree, Node, Tree}; + + pub(crate) fn str_leaf(c: &str) -> Tree { + Tree(Node::Leaf { + value: c.to_string(), + }) + } + + pub(crate) fn nil() -> Tree { + Tree::empty() + } + + pub(crate) fn leaf(value: B) -> Tree { + Tree::leaf(value) + } + + pub(crate) fn parent(left: Tree, right: Tree) -> Tree { + Tree::parent(A::default(), left, right) + } + + #[test] + fn incomplete_nodes() { + let t: Tree<(), String> = parent(nil(), str_leaf("a")); + assert_eq!( + t.incomplete_nodes(Address::from_parts(Level::from(1), 0)), + vec![Address::from_parts(Level::from(0), 0)] + ); + + let t0 = parent(str_leaf("b"), t.clone()); + assert_eq!( + t0.incomplete_nodes(Address::from_parts(Level::from(2), 1)), + vec![Address::from_parts(Level::from(0), 6)] + ); + + let t1 = parent(nil(), t); + assert_eq!( + t1.incomplete_nodes(Address::from_parts(Level::from(2), 1)), + vec![ + Address::from_parts(Level::from(1), 2), + Address::from_parts(Level::from(0), 6) + ] + ); + } + + #[test] + fn located() { + let l = parent(str_leaf("a"), str_leaf("b")); + let r = parent(str_leaf("c"), str_leaf("d")); + + let t: LocatedTree<(), String> = LocatedTree { + root_addr: Address::from_parts(2.into(), 1), + root: parent(l.clone(), r.clone()), + }; + + assert_eq!(t.max_position(), Some(7.into())); + assert_eq!(t.value_at_position(5.into()), Some(&"b".to_string())); + assert_eq!(t.value_at_position(8.into()), None); + assert_eq!(t.subtree(Address::from_parts(0.into(), 1)), None); + assert_eq!(t.subtree(Address::from_parts(3.into(), 0)), None); + + let subtree_addr = Address::from_parts(1.into(), 3); + assert_eq!( + t.subtree(subtree_addr), + Some(LocatedTree { + root_addr: subtree_addr, + root: r.clone() + }) + ); + + assert_eq!( + t.decompose_to_level(1.into()), + vec![ + LocatedTree { + root_addr: Address::from_parts(1.into(), 2), + root: l, + }, + LocatedTree { + root_addr: Address::from_parts(1.into(), 3), + root: r, + } + ] + ); + } +} diff --git a/third_party/rust/shardtree/v0_3/BUILD.gn b/third_party/rust/shardtree/v0_3/BUILD.gn new file mode 100644 index 000000000000..e39ac5ac745f --- /dev/null +++ b/third_party/rust/shardtree/v0_3/BUILD.gn @@ -0,0 +1,50 @@ +# Copyright (c) 2024 The Brave Authors. All rights reserved. +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at https://mozilla.org/MPL/2.0/. + +# @generated from third_party/rust/chromium_crates_io/BUILD.gn.hbs by +# tools/crates/gnrt. +# Do not edit! + +import("//build/rust/cargo_crate.gni") + +cargo_crate("lib") { + crate_name = "shardtree" + epoch = "0.3" + crate_type = "rlib" + crate_root = "//brave/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/lib.rs" + sources = [ + "//brave/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/batch.rs", + "//brave/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/error.rs", + "//brave/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/legacy.rs", + "//brave/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/lib.rs", + "//brave/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/prunable.rs", + "//brave/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/store.rs", + "//brave/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/store/caching.rs", + "//brave/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/store/memory.rs", + "//brave/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/testing.rs", + "//brave/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/src/tree.rs", + ] + inputs = [] + + build_native_rust_unit_tests = false + edition = "2021" + cargo_pkg_version = "0.3.1" + cargo_pkg_authors = "Kris Nuttycombe " + cargo_pkg_name = "shardtree" + cargo_pkg_description = "A space-efficient Merkle tree with witnessing of marked leaves, checkpointing & state restoration." + library_configs -= [ "//build/config/compiler:chromium_code" ] + library_configs += [ "//build/config/compiler:no_chromium_code" ] + executable_configs -= [ "//build/config/compiler:chromium_code" ] + executable_configs += [ "//build/config/compiler:no_chromium_code" ] + proc_macro_configs -= [ "//build/config/compiler:chromium_code" ] + proc_macro_configs += [ "//build/config/compiler:no_chromium_code" ] + deps = [ + "//brave/third_party/rust/bitflags/v2:lib", + "//brave/third_party/rust/either/v1:lib", + "//brave/third_party/rust/incrementalmerkletree/v0_5:lib", + "//brave/third_party/rust/tracing/v0_1:lib", + ] + features = [ "legacy-api" ] +} diff --git a/third_party/rust/shardtree/v0_3/README.chromium b/third_party/rust/shardtree/v0_3/README.chromium new file mode 100644 index 000000000000..c684f6aa6733 --- /dev/null +++ b/third_party/rust/shardtree/v0_3/README.chromium @@ -0,0 +1,9 @@ +Name: shardtree +URL: https://crates.io/crates/shardtree +Description: A space-efficient Merkle tree with witnessing of marked leaves, checkpointing & state restoration. +Version: 0.3.1 +Security Critical: yes +Shipped: yes +License: Apache 2.0 +License File: //brave/third_party/rust/chromium_crates_io/vendor/shardtree-0.3.1/LICENSE-APACHE +Revision: 7c862c16b31aa3883fcbb61f30af9930eb01cc0b diff --git a/third_party/rust/zcash_client_backend/v0_12/README.chromium b/third_party/rust/zcash_client_backend/v0_12/README.chromium new file mode 100644 index 000000000000..fc60e9e2d520 --- /dev/null +++ b/third_party/rust/zcash_client_backend/v0_12/README.chromium @@ -0,0 +1,8 @@ +Name: zcash_client_backend +URL: https://crates.io/crates/zcash_client_backend +Description: APIs for creating shielded Zcash light clients +Version: 0.12.1 +Security Critical: yes +Shipped: yes +License: Apache 2.0 +License File: //brave/third_party/rust/chromium_crates_io/vendor/zcash_client_backend-0.12.1/../../../../../common/licenses/Apache-2.0