diff --git a/.github/actions/build-dependencies/action.yml b/.github/actions/build-dependencies/action.yml index 45d242931..32e7b3321 100644 --- a/.github/actions/build-dependencies/action.yml +++ b/.github/actions/build-dependencies/action.yml @@ -10,7 +10,7 @@ inputs: rust-toolchain: description: "Rust toolchain to install" required: false - default: stable + default: 1.72.1 rust-components: description: "Rust components to install" diff --git a/.github/nightly-version b/.github/nightly-version index 2540aa02c..e205afa2c 100644 --- a/.github/nightly-version +++ b/.github/nightly-version @@ -1 +1 @@ -nightly-2023-09-01 +nightly-2023-10-01 diff --git a/.github/workflows/daily-deny.yml b/.github/workflows/daily-deny.yml index 2a5a6b212..a15c6c7e7 100644 --- a/.github/workflows/daily-deny.yml +++ b/.github/workflows/daily-deny.yml @@ -20,7 +20,7 @@ jobs: - name: Install cargo uses: dtolnay/rust-toolchain@5cb429dd810e16ff67df78472fa81cf760f4d1c0 with: - toolchain: stable + toolchain: 1.72.1 - name: Install cargo deny run: cargo install --locked cargo-deny diff --git a/.github/workflows/full-stack-tests.yml b/.github/workflows/full-stack-tests.yml index f764bc838..7fd71a6b9 100644 --- a/.github/workflows/full-stack-tests.yml +++ b/.github/workflows/full-stack-tests.yml @@ -15,10 +15,25 @@ jobs: steps: - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac - - name: Install Build Dependencies - uses: ./.github/actions/build-dependencies + # - name: Install Build Dependencies + # uses: ./.github/actions/build-dependencies + # with: + # github-token: ${{ inputs.github-token }} + + # Inlined build-dependencies action to minimize disk usage + - name: Install Protobuf + uses: arduino/setup-protoc@a8b67ba40b37d35169e222f3bb352603327985b6 + with: + repo-token: ${{ inputs.github-token }} + + - name: Install Rust + uses: dtolnay/rust-toolchain@5cb429dd810e16ff67df78472fa81cf760f4d1c0 with: - github-token: ${{ inputs.github-token }} + toolchain: stable + targets: wasm32-unknown-unknown + + - name: Remove unused packages + run: sudo apt remove -y *powershell* *bazel* *nodejs* *npm* *yarn* *terraform* *firefox* *chromium* *texinfo* *sqlite3* *imagemagick* && sudo apt autoremove -y - name: Run Full Stack Docker tests run: cd tests/full-stack && GITHUB_CI=true RUST_BACKTRACE=1 cargo test diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6b427387c..9646cbb5f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -41,7 +41,7 @@ jobs: - name: Install cargo uses: dtolnay/rust-toolchain@5cb429dd810e16ff67df78472fa81cf760f4d1c0 with: - toolchain: stable + toolchain: 1.72.1 - name: Install cargo deny run: cargo install --locked cargo-deny diff --git a/.github/workflows/mini-tests.yml b/.github/workflows/mini-tests.yml new file mode 100644 index 000000000..634d30b69 --- /dev/null +++ b/.github/workflows/mini-tests.yml @@ -0,0 +1,28 @@ +name: mini/ Tests + +on: + push: + branches: + - develop + paths: + - "mini/**" + + pull_request: + paths: + - "mini/**" + + workflow_dispatch: + +jobs: + test-common: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac + + - name: Test Dependencies + uses: ./.github/actions/test-dependencies + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Run Tests + run: GITHUB_CI=true RUST_BACKTRACE=1 cargo test --all-features -p mini-serai diff --git a/Cargo.lock b/Cargo.lock index c1b9bbdf1..e8cda73bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -96,9 +96,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.5" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" +checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" dependencies = [ "memchr", ] @@ -208,15 +208,6 @@ version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b1c5a481ec30a5abd8dfbd94ab5cf1bb4e9a66be7f1b3b322f2f1170c200fd" -[[package]] -name = "array-init" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23589ecb866b460d3a0f1278834750268c607e8e28a1b982c907219f3178cd72" -dependencies = [ - "nodrop", -] - [[package]] name = "arrayref" version = "0.3.7" @@ -325,7 +316,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -473,7 +464,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -601,16 +592,15 @@ dependencies = [ [[package]] name = "blake3" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "199c42ab6972d92c9f8995f086273d25c42fc0f7b2a1fcefba465c1352d25ba5" +checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "digest 0.10.7", ] [[package]] @@ -725,9 +715,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "byte-slice-cast" @@ -793,7 +783,7 @@ checksum = "e7daec1a2a2129eeba1644b220b4647ec537b0b5d4bfd6876fcc5a540056b592" dependencies = [ "camino", "cargo-platform", - "semver 1.0.18", + "semver 1.0.19", "serde", "serde_json", "thiserror", @@ -824,7 +814,7 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03915af431787e6ffdcc74c645077518c6b6e01f80b761e0fbbfa288536311b3" dependencies = [ - "smallvec 1.11.0", + "smallvec", ] [[package]] @@ -877,9 +867,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.30" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defd4e7873dbddba6c7c91e199c7fcb946abc4a6a4ac3195400bcfb01b5de877" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ "android-tzdata", "iana-time-zone", @@ -958,9 +948,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.2" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" +checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" dependencies = [ "clap_builder", "clap_derive", @@ -968,9 +958,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.2" +version = "4.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" dependencies = [ "anstream", "anstyle", @@ -987,7 +977,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -1086,9 +1076,9 @@ dependencies = [ [[package]] name = "const-hex" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08849ed393c907c90016652a01465a12d86361cd38ad2a7de026c56a520cc259" +checksum = "aa72a10d0e914cad6bcad4e7409e68d230c1c2db67896e19a37f758b1fcbdab5" dependencies = [ "cfg-if", "cpufeatures", @@ -1208,7 +1198,7 @@ dependencies = [ "hashbrown 0.13.2", "log", "regalloc2", - "smallvec 1.11.0", + "smallvec", "target-lexicon", ] @@ -1253,7 +1243,7 @@ checksum = "b7a94c4c5508b7407e125af9d5320694b7423322e59a4ac0d07919ae254347ca" dependencies = [ "cranelift-codegen", "log", - "smallvec 1.11.0", + "smallvec", "target-lexicon", ] @@ -1285,7 +1275,7 @@ dependencies = [ "cranelift-frontend", "itertools 0.10.5", "log", - "smallvec 1.11.0", + "smallvec", "wasmparser", "wasmtime-types", ] @@ -1314,16 +1304,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" version = "0.8.3" @@ -1418,9 +1398,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.0" +version = "4.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622178105f911d937a42cdb140730ba4a3ed2becd8ae6ce39c7d28b5d75d4588" +checksum = "e89b8c6a2e4b1f45971ad09761aafb85514a84744b67a95e32c3cc1352d1f65c" dependencies = [ "cfg-if", "cpufeatures", @@ -1443,7 +1423,7 @@ checksum = "83fdaf97f4804dcebfa5862639bc9ce4121e82140bec2a987ac5140294865b5b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -1470,7 +1450,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -1487,7 +1467,7 @@ checksum = "2fa16a70dd58129e4dfffdff535fb1bce66673f7bbeec4a5a1765a504e1ccd84" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -1495,7 +1475,7 @@ name = "dalek-ff-group" version = "0.4.1" dependencies = [ "crypto-bigint", - "curve25519-dalek 4.1.0", + "curve25519-dalek 4.1.1", "digest 0.10.7", "ff", "ff-group-tests", @@ -1576,7 +1556,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -1609,7 +1589,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core 0.20.3", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -1825,7 +1805,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -1927,9 +1907,9 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfc4744c1b8f2a09adc0e55242f60b1af195d88596bd8700be74418c056c555" +checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" [[package]] name = "ecdsa" @@ -1961,7 +1941,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7277392b266383ef8396db7fdeb1e77b6c52fed775f5df15bb24f35b72156980" dependencies = [ - "curve25519-dalek 4.1.0", + "curve25519-dalek 4.1.1", "ed25519", "rand_core 0.6.4", "serde", @@ -1975,7 +1955,7 @@ version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d9ce6874da5d4415896cd45ffbc4d1cfc0c4f9c079427bd870742c30f2f65a9" dependencies = [ - "curve25519-dalek 4.1.0", + "curve25519-dalek 4.1.1", "ed25519", "hashbrown 0.14.0", "hex", @@ -2029,9 +2009,9 @@ dependencies = [ [[package]] name = "enr" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0be7b2ac146c1f99fe245c02d16af0696450d8e06c135db75e10eeb9e642c20d" +checksum = "fe81b5c06ecfdbc71dd845216f225f53b62a10cb8a16c946836a3467f701d05b" dependencies = [ "base64 0.21.4", "bytes", @@ -2041,7 +2021,6 @@ dependencies = [ "rand", "rlp", "serde", - "serde-hex", "sha3", "zeroize", ] @@ -2257,7 +2236,7 @@ dependencies = [ "regex", "serde", "serde_json", - "syn 2.0.32", + "syn 2.0.37", "toml 0.7.8", "walkdir", ] @@ -2275,7 +2254,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -2301,7 +2280,7 @@ dependencies = [ "serde", "serde_json", "strum 0.25.0", - "syn 2.0.32", + "syn 2.0.37", "tempfile", "thiserror", "tiny-keccak", @@ -2317,7 +2296,7 @@ dependencies = [ "ethers-core", "ethers-solc", "reqwest", - "semver 1.0.18", + "semver 1.0.19", "serde", "serde_json", "thiserror", @@ -2425,7 +2404,7 @@ dependencies = [ "path-slash", "rayon", "regex", - "semver 1.0.18", + "semver 1.0.19", "serde", "serde_json", "solang-parser", @@ -2462,7 +2441,7 @@ dependencies = [ "fs-err", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -2788,7 +2767,7 @@ dependencies = [ "paste", "scale-info", "serde", - "smallvec 1.11.0", + "smallvec", "sp-api", "sp-arithmetic", "sp-core", @@ -2820,7 +2799,7 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -2832,7 +2811,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -2842,7 +2821,7 @@ source = "git+https://github.com/serai-dex/substrate#98ab693fdf71f371d5059aa6924 dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -3003,7 +2982,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -3089,6 +3068,19 @@ dependencies = [ "serde_json", ] +[[package]] +name = "generator" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc16584ff22b460a382b7feec54b23d2908d858152e5739a120b949293bd74e" +dependencies = [ + "cc", + "libc", + "log", + "rustversion", + "windows 0.48.0", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -3293,9 +3285,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "hex" @@ -3676,7 +3668,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix 0.38.13", + "rustix 0.38.14", "windows-sys", ] @@ -3897,7 +3889,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7d770dcb02bf6835887c3a979b5107a04ff4bbde97a5f0928d27404a155add9" dependencies = [ - "smallvec 1.11.0", + "smallvec", ] [[package]] @@ -3921,7 +3913,7 @@ dependencies = [ "parking_lot 0.12.1", "regex", "rocksdb", - "smallvec 1.11.0", + "smallvec", ] [[package]] @@ -3975,9 +3967,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" [[package]] name = "libloading" @@ -4055,9 +4047,9 @@ dependencies = [ [[package]] name = "libp2p-core" -version = "0.40.0" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef7dd7b09e71aac9271c60031d0e558966cdb3253ba0308ab369bb2de80630d0" +checksum = "dd44289ab25e4c9230d9246c475a22241e301b23e8f4061d3bdef304a1a99713" dependencies = [ "either", "fnv", @@ -4075,7 +4067,7 @@ dependencies = [ "quick-protobuf", "rand", "rw-stream-sink", - "smallvec 1.11.0", + "smallvec", "thiserror", "unsigned-varint", "void", @@ -4092,7 +4084,7 @@ dependencies = [ "libp2p-identity", "log", "parking_lot 0.12.1", - "smallvec 1.11.0", + "smallvec", "trust-dns-resolver", ] @@ -4123,7 +4115,7 @@ dependencies = [ "rand", "regex", "sha2", - "smallvec 1.11.0", + "smallvec", "unsigned-varint", "void", ] @@ -4145,7 +4137,7 @@ dependencies = [ "lru", "quick-protobuf", "quick-protobuf-codec", - "smallvec 1.11.0", + "smallvec", "thiserror", "void", ] @@ -4169,9 +4161,9 @@ dependencies = [ [[package]] name = "libp2p-kad" -version = "0.44.4" +version = "0.44.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc125f83d8f75322c79e4ade74677d299b34aa5c9d9b5251c03ec28c683cb765" +checksum = "41c5c483b1e90e79409711f515c5bea5de9c4d772a245b1ac01a2233fbcb67fe" dependencies = [ "arrayvec", "asynchronous-codec", @@ -4186,9 +4178,10 @@ dependencies = [ "libp2p-swarm", "log", "quick-protobuf", + "quick-protobuf-codec", "rand", "sha2", - "smallvec 1.11.0", + "smallvec", "thiserror", "uint", "unsigned-varint", @@ -4209,7 +4202,7 @@ dependencies = [ "libp2p-swarm", "log", "rand", - "smallvec 1.11.0", + "smallvec", "socket2 0.5.4", "tokio", "trust-dns-proto", @@ -4241,7 +4234,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71ce70757f2c0d82e9a3ef738fb10ea0723d16cec37f078f719e2c247704c1bb" dependencies = [ "bytes", - "curve25519-dalek 4.1.0", + "curve25519-dalek 4.1.1", "futures", "libp2p-core", "libp2p-identity", @@ -4261,9 +4254,9 @@ dependencies = [ [[package]] name = "libp2p-ping" -version = "0.43.0" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd5ee3270229443a2b34b27ed0cb7470ef6b4a6e45e54e89a8771fa683bab48" +checksum = "e702d75cd0827dfa15f8fd92d15b9932abe38d10d21f47c50438c71dd1b5dae3" dependencies = [ "either", "futures", @@ -4314,15 +4307,15 @@ dependencies = [ "libp2p-swarm", "log", "rand", - "smallvec 1.11.0", + "smallvec", "void", ] [[package]] name = "libp2p-swarm" -version = "0.43.3" +version = "0.43.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28016944851bd73526d3c146aabf0fa9bbe27c558f080f9e5447da3a1772c01a" +checksum = "f0cf749abdc5ca1dce6296dc8ea0f012464dfcfd3ddd67ffc0cabd8241c4e1da" dependencies = [ "either", "fnv", @@ -4336,7 +4329,7 @@ dependencies = [ "multistream-select", "once_cell", "rand", - "smallvec 1.11.0", + "smallvec", "tokio", "void", ] @@ -4351,7 +4344,7 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -4384,7 +4377,7 @@ dependencies = [ "rcgen", "ring", "rustls", - "rustls-webpki 0.101.5", + "rustls-webpki 0.101.6", "thiserror", "x509-parser", "yasna", @@ -4525,6 +4518,19 @@ version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +[[package]] +name = "loom" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a17963e5073acf8d3e2637402657c6b467218f36fe10d696b3e1095ae019bf" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber 0.3.17", +] + [[package]] name = "lru" version = "0.10.1" @@ -4581,7 +4587,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -4595,7 +4601,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -4606,7 +4612,7 @@ checksum = "c12469fc165526520dff2807c2975310ab47cf7190a45b99b49a7dc8befab17b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -4617,7 +4623,7 @@ checksum = "b8fb85ec1620619edf2984a7693497d4ec88a9665d8b87e942856884c92dbf2a" dependencies = [ "macro_magic_core", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -4635,6 +4641,15 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", +] + [[package]] name = "matches" version = "0.1.10" @@ -4643,26 +4658,21 @@ checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" [[package]] name = "matrixmultiply" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090126dc04f95dc0d1c1c91f61bdd474b3930ca064c1edc8a849da2c6cbe1e77" +checksum = "7574c1cf36da4798ab73da5b215bbf444f50718207754cb522201d78d1cd0ff2" dependencies = [ "autocfg", "rawpointer", ] -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest 0.10.7", ] @@ -4674,11 +4684,11 @@ checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" [[package]] name = "memfd" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc89ccdc6e10d6907450f753537ebc5c5d3460d2e4e62ea74bd571db62c0f9e" +checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" dependencies = [ - "rustix 0.37.23", + "rustix 0.38.14", ] [[package]] @@ -4726,6 +4736,13 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mini-serai" +version = "0.1.0" +dependencies = [ + "loom", +] + [[package]] name = "minimal-ed448" version = "0.4.0" @@ -4821,7 +4838,7 @@ dependencies = [ name = "monero-generators" version = "0.4.0" dependencies = [ - "curve25519-dalek 4.1.0", + "curve25519-dalek 4.1.1", "dalek-ff-group", "group", "sha3", @@ -4836,7 +4853,7 @@ dependencies = [ "async-trait", "base58-monero", "crc", - "curve25519-dalek 4.1.0", + "curve25519-dalek 4.1.1", "dalek-ff-group", "digest_auth", "dleq", @@ -5009,7 +5026,7 @@ dependencies = [ "futures", "log", "pin-project", - "smallvec 1.11.0", + "smallvec", "unsigned-varint", ] @@ -5150,12 +5167,6 @@ dependencies = [ "libc", ] -[[package]] -name = "nodrop" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" - [[package]] name = "nohash-hasher" version = "0.2.0" @@ -5178,6 +5189,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint" version = "0.4.4" @@ -5268,7 +5289,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -5361,7 +5382,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -5388,6 +5409,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "p256" version = "0.13.2" @@ -5600,9 +5627,9 @@ dependencies = [ [[package]] name = "parity-db" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78f19d20a0d2cc52327a88d131fa1c4ea81ea4a04714aedcfeca2dd410049cf8" +checksum = "ab512a34b3c2c5e465731cc7668edf79208bbe520be03484eeb05e63ed221735" dependencies = [ "blake2", "crc32fast", @@ -5688,7 +5715,7 @@ dependencies = [ "instant", "libc", "redox_syscall 0.2.16", - "smallvec 1.11.0", + "smallvec", "winapi", ] @@ -5701,7 +5728,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall 0.3.5", - "smallvec 1.11.0", + "smallvec", "windows-targets", ] @@ -5822,7 +5849,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -5886,7 +5913,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -5924,7 +5951,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -6065,7 +6092,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -6139,14 +6166,14 @@ checksum = "3d1eaa7fa0aa1929ffdf7eeb6eac234dde6268914a14ad44d23521ab6a9b258e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" dependencies = [ "unicode-ident", ] @@ -6185,7 +6212,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -6310,9 +6337,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13f81c9a9d574310b8351f8666f5a93ac3b0069c45c28ad52c10291389a7cf9" +checksum = "2c78e758510582acc40acb90458401172d41f1016f8c9dde89e49677afb7eec1" dependencies = [ "bytes", "rand", @@ -6419,9 +6446,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "rayon" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ "either", "rayon-core", @@ -6429,14 +6456,12 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] @@ -6497,7 +6522,7 @@ checksum = "7f7473c2cfcf90008193dd0e3e16599455cb601a9fce322b5bb55de799664925" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -6510,7 +6535,7 @@ dependencies = [ "log", "rustc-hash", "slice-group-by", - "smallvec 1.11.0", + "smallvec", ] [[package]] @@ -6730,7 +6755,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.18", + "semver 1.0.19", ] [[package]] @@ -6758,9 +6783,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.13" +version = "0.38.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" +checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" dependencies = [ "bitflags 2.4.0", "errno", @@ -6777,7 +6802,7 @@ checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", "ring", - "rustls-webpki 0.101.5", + "rustls-webpki 0.101.6", "sct", ] @@ -6804,9 +6829,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.100.2" +version = "0.100.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e98ff011474fa39949b7e5c0428f9b4937eda7da7848bbb947786b7be0b27dab" +checksum = "5f6a5fc258f1c1276dfe3016516945546e2d5383911efc0fc4f1cdc5df3a4ae3" dependencies = [ "ring", "untrusted", @@ -6814,9 +6839,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.5" +version = "0.101.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a27e3b59326c16e23d30aeb7a36a24cc0d29e71d68ff611cdfb4a01d013bed" +checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" dependencies = [ "ring", "untrusted", @@ -6976,7 +7001,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -7250,7 +7275,7 @@ dependencies = [ "cfg-if", "libc", "log", - "rustix 0.38.13", + "rustix 0.38.14", "sc-allocator", "sc-executor-common", "sp-runtime-interface", @@ -7317,7 +7342,7 @@ dependencies = [ "sc-utils", "serde", "serde_json", - "smallvec 1.11.0", + "smallvec", "sp-arithmetic", "sp-blockchain", "sp-core", @@ -7430,7 +7455,7 @@ dependencies = [ "sc-network-common", "sc-utils", "schnellru", - "smallvec 1.11.0", + "smallvec", "sp-arithmetic", "sp-blockchain", "sp-consensus", @@ -7727,7 +7752,7 @@ dependencies = [ "thiserror", "tracing", "tracing-log", - "tracing-subscriber", + "tracing-subscriber 0.2.25", ] [[package]] @@ -7738,7 +7763,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -7820,7 +7845,7 @@ dependencies = [ "scale-bits", "scale-decode-derive", "scale-info", - "smallvec 1.11.0", + "smallvec", "thiserror", ] @@ -7848,7 +7873,7 @@ dependencies = [ "scale-bits", "scale-encode-derive", "scale-info", - "smallvec 1.11.0", + "smallvec", "thiserror", ] @@ -7954,7 +7979,7 @@ checksum = "6b3cebf217f367b9d6f2f27ca0ebd14c7d1dfb1ae3cdbf6f3fa1e5c3e4f67bb8" dependencies = [ "arrayref", "arrayvec", - "curve25519-dalek 4.1.0", + "curve25519-dalek 4.1.1", "merlin", "rand", "rand_core 0.6.4", @@ -7964,6 +7989,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -8076,9 +8107,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" dependencies = [ "serde", ] @@ -8106,6 +8137,7 @@ name = "serai-client" version = "0.1.0" dependencies = [ "bitcoin", + "blake2", "ciphersuite", "frost-schnorrkel", "futures", @@ -8135,7 +8167,6 @@ dependencies = [ "frost-schnorrkel", "futures", "hex", - "lazy_static", "libp2p", "log", "modular-frost", @@ -8160,10 +8191,12 @@ dependencies = [ name = "serai-coordinator-tests" version = "0.1.0" dependencies = [ + "blake2", "ciphersuite", "dkg", "dockertest", "hex", + "parity-scale-codec", "rand_core 0.6.4", "schnorrkel", "serai-client", @@ -8200,7 +8233,7 @@ name = "serai-full-stack-tests" version = "0.1.0" dependencies = [ "bitcoin-serai", - "curve25519-dalek 4.1.0", + "curve25519-dalek 4.1.1", "dockertest", "hex", "monero-serai", @@ -8232,6 +8265,7 @@ dependencies = [ "serai-validator-sets-pallet", "sp-application-crypto", "sp-core", + "sp-io", "sp-runtime", "thiserror", ] @@ -8418,7 +8452,7 @@ version = "0.1.0" dependencies = [ "bitcoin-serai", "ciphersuite", - "curve25519-dalek 4.1.0", + "curve25519-dalek 4.1.1", "dkg", "dockertest", "hex", @@ -8580,17 +8614,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde-hex" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca37e3e4d1b39afd7ff11ee4e947efae85adfddf4841787bfa47c470e96dc26d" -dependencies = [ - "array-init", - "serde", - "smallvec 0.6.14", -] - [[package]] name = "serde_bytes" version = "0.11.12" @@ -8608,14 +8631,14 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] name = "serde_json" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc66a619ed80bf7a0f6b17dd063a84b88f6dea1813737cf469aef1d081142c2" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -8792,18 +8815,9 @@ checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" [[package]] name = "smallvec" -version = "0.6.14" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97fcaeba89edba30f044a10c6a3cc39df9c3f17d7cd829dd1446cab35f890e0" -dependencies = [ - "maybe-uninit", -] - -[[package]] -name = "smallvec" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "snap" @@ -8820,7 +8834,7 @@ dependencies = [ "aes-gcm", "blake2", "chacha20poly1305", - "curve25519-dalek 4.1.0", + "curve25519-dalek 4.1.1", "rand_core 0.6.4", "ring", "rustc_version", @@ -8910,7 +8924,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -9106,7 +9120,7 @@ source = "git+https://github.com/serai-dex/substrate#98ab693fdf71f371d5059aa6924 dependencies = [ "quote", "sp-core-hashing", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -9125,7 +9139,7 @@ source = "git+https://github.com/serai-dex/substrate#98ab693fdf71f371d5059aa6924 dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -9297,7 +9311,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -9339,7 +9353,7 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.12.1", "rand", - "smallvec 1.11.0", + "smallvec", "sp-core", "sp-externalities", "sp-panic-handler", @@ -9390,7 +9404,7 @@ dependencies = [ "sp-std", "tracing", "tracing-core", - "tracing-subscriber", + "tracing-subscriber 0.2.25", ] [[package]] @@ -9450,7 +9464,7 @@ dependencies = [ "parity-scale-codec", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -9474,7 +9488,7 @@ dependencies = [ "parity-scale-codec", "scale-info", "serde", - "smallvec 1.11.0", + "smallvec", "sp-arithmetic", "sp-core", "sp-debug-derive", @@ -9645,7 +9659,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -9767,7 +9781,7 @@ dependencies = [ "quote", "scale-info", "subxt-metadata", - "syn 2.0.32", + "syn 2.0.37", "thiserror", "tokio", ] @@ -9781,7 +9795,7 @@ dependencies = [ "darling 0.20.3", "proc-macro-error", "subxt-codegen", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -9810,9 +9824,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.32" +version = "2.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" dependencies = [ "proc-macro2", "quote", @@ -9873,7 +9887,7 @@ dependencies = [ "cfg-if", "fastrand 2.0.0", "redox_syscall 0.3.5", - "rustix 0.38.13", + "rustix 0.38.14", "windows-sys", ] @@ -9903,9 +9917,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" +checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" dependencies = [ "winapi-util", ] @@ -9933,7 +9947,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -9973,9 +9987,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +checksum = "426f806f4089c493dcac0d24c29c01e2c38baf8e30f1b716ee37e83d200b18fe" dependencies = [ "deranged", "itoa", @@ -9986,15 +10000,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" dependencies = [ "time-core", ] @@ -10069,7 +10083,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -10106,9 +10120,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", @@ -10224,7 +10238,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -10277,13 +10291,13 @@ dependencies = [ "ansi_term", "chrono", "lazy_static", - "matchers", + "matchers 0.0.1", "parking_lot 0.11.2", "regex", "serde", "serde_json", "sharded-slab", - "smallvec 1.11.0", + "smallvec", "thread_local", "tracing", "tracing-core", @@ -10291,6 +10305,24 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "tracing-subscriber" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +dependencies = [ + "matchers 0.1.0", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + [[package]] name = "tributary-chain" version = "0.1.0" @@ -10324,7 +10356,7 @@ dependencies = [ "hashbrown 0.13.2", "log", "rustc-hex", - "smallvec 1.11.0", + "smallvec", ] [[package]] @@ -10353,7 +10385,7 @@ dependencies = [ "ipnet", "lazy_static", "rand", - "smallvec 1.11.0", + "smallvec", "socket2 0.4.9", "thiserror", "tinyvec", @@ -10375,7 +10407,7 @@ dependencies = [ "lru-cache", "parking_lot 0.12.1", "resolv-conf", - "smallvec 1.11.0", + "smallvec", "thiserror", "tokio", "tracing", @@ -10408,9 +10440,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ucd-trie" @@ -10438,9 +10470,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -10453,9 +10485,9 @@ dependencies = [ [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unicode-xid" @@ -10606,7 +10638,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", "wasm-bindgen-shared", ] @@ -10640,7 +10672,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -10731,7 +10763,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dfcdb72d96f01e6c85b6bf20102e7423bdbaad5c337301bab2bbf253d26413c" dependencies = [ "indexmap 2.0.0", - "semver 1.0.18", + "semver 1.0.19", ] [[package]] @@ -10787,7 +10819,7 @@ dependencies = [ "directories-next", "file-per-thread-logger", "log", - "rustix 0.38.13", + "rustix 0.38.14", "serde", "sha2", "toml 0.5.11", @@ -10869,7 +10901,7 @@ dependencies = [ "log", "object 0.31.1", "rustc-demangle", - "rustix 0.38.13", + "rustix 0.38.14", "serde", "target-lexicon", "wasmtime-environ", @@ -10887,7 +10919,7 @@ checksum = "aef27ea6c34ef888030d15560037fe7ef27a5609fbbba8e1e3e41dc4245f5bb2" dependencies = [ "object 0.31.1", "once_cell", - "rustix 0.38.13", + "rustix 0.38.14", "wasmtime-versioned-export-macros", ] @@ -10919,7 +10951,7 @@ dependencies = [ "memoffset", "paste", "rand", - "rustix 0.38.13", + "rustix 0.38.14", "sptr", "wasm-encoder", "wasmtime-asm-macros", @@ -10949,7 +10981,7 @@ checksum = "ca7af9bb3ee875c4907835e607a275d10b04d15623d3aebe01afe8fbd3f85050" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] @@ -10968,7 +11000,7 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" dependencies = [ - "rustls-webpki 0.100.2", + "rustls-webpki 0.100.3", ] [[package]] @@ -10986,7 +11018,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.13", + "rustix 0.38.14", ] [[package]] @@ -11023,9 +11055,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -11289,7 +11321,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.37", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 41535fea0..dbf6910e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,8 @@ members = [ "substrate/client", + "mini", + "tests/no-std", "tests/docker", diff --git a/coins/bitcoin/src/wallet/mod.rs b/coins/bitcoin/src/wallet/mod.rs index abc10cab8..7cff854b1 100644 --- a/coins/bitcoin/src/wallet/mod.rs +++ b/coins/bitcoin/src/wallet/mod.rs @@ -71,6 +71,11 @@ impl ReceivedOutput { self.offset } + /// The Bitcoin output for this output. + pub fn output(&self) -> &TxOut { + &self.output + } + /// The outpoint for this output. pub fn outpoint(&self) -> &OutPoint { &self.outpoint diff --git a/coins/bitcoin/src/wallet/send.rs b/coins/bitcoin/src/wallet/send.rs index 007ba527a..7dde21867 100644 --- a/coins/bitcoin/src/wallet/send.rs +++ b/coins/bitcoin/src/wallet/send.rs @@ -116,6 +116,12 @@ impl SignableTransaction { self.needed_fee } + /// Returns the fee this transaction will use. + pub fn fee(&self) -> u64 { + self.prevouts.iter().map(|prevout| prevout.value).sum::() - + self.tx.output.iter().map(|prevout| prevout.value).sum::() + } + /// Create a new SignableTransaction. /// /// If a change address is specified, any leftover funds will be sent to it if the leftover funds diff --git a/coins/monero/build.rs b/coins/monero/build.rs index a54a3f2d3..db15c1cfd 100644 --- a/coins/monero/build.rs +++ b/coins/monero/build.rs @@ -44,10 +44,10 @@ fn generators(prefix: &'static str, path: &str) { pub(crate) static GENERATORS_CELL: OnceLock = OnceLock::new(); pub fn GENERATORS() -> &'static Generators {{ GENERATORS_CELL.get_or_init(|| Generators {{ - G: [ + G: vec![ {G_str} ], - H: [ + H: vec![ {H_str} ], }}) diff --git a/coins/monero/generators/src/lib.rs b/coins/monero/generators/src/lib.rs index 7f630f36e..e0377dca7 100644 --- a/coins/monero/generators/src/lib.rs +++ b/coins/monero/generators/src/lib.rs @@ -5,7 +5,7 @@ #![cfg_attr(not(feature = "std"), no_std)] -use std_shims::sync::OnceLock; +use std_shims::{sync::OnceLock, vec::Vec}; use sha3::{Digest, Keccak256}; @@ -56,14 +56,13 @@ const MAX_MN: usize = MAX_M * N; /// Container struct for Bulletproofs(+) generators. #[allow(non_snake_case)] pub struct Generators { - pub G: [EdwardsPoint; MAX_MN], - pub H: [EdwardsPoint; MAX_MN], + pub G: Vec, + pub H: Vec, } /// Generate generators as needed for Bulletproofs(+), as Monero does. pub fn bulletproofs_generators(dst: &'static [u8]) -> Generators { - let mut res = - Generators { G: [EdwardsPoint::identity(); MAX_MN], H: [EdwardsPoint::identity(); MAX_MN] }; + let mut res = Generators { G: Vec::with_capacity(MAX_MN), H: Vec::with_capacity(MAX_MN) }; for i in 0 .. MAX_MN { let i = 2 * i; @@ -73,8 +72,8 @@ pub fn bulletproofs_generators(dst: &'static [u8]) -> Generators { write_varint(&i.try_into().unwrap(), &mut even).unwrap(); write_varint(&(i + 1).try_into().unwrap(), &mut odd).unwrap(); - res.H[i / 2] = EdwardsPoint(hash_to_point(hash(&even))); - res.G[i / 2] = EdwardsPoint(hash_to_point(hash(&odd))); + res.H.push(EdwardsPoint(hash_to_point(hash(&even)))); + res.G.push(EdwardsPoint(hash_to_point(hash(&odd)))); } res } diff --git a/coordinator/Cargo.toml b/coordinator/Cargo.toml index 810d12a83..d00e9abe2 100644 --- a/coordinator/Cargo.toml +++ b/coordinator/Cargo.toml @@ -15,7 +15,6 @@ rustdoc-args = ["--cfg", "docsrs"] [dependencies] async-trait = "0.1" -lazy_static = "1" zeroize = "^1.5" rand_core = "0.6" diff --git a/coordinator/src/db.rs b/coordinator/src/db.rs index 67566e53b..fd9380253 100644 --- a/coordinator/src/db.rs +++ b/coordinator/src/db.rs @@ -1,26 +1,43 @@ +use core::marker::PhantomData; + +use blake2::{ + digest::{consts::U32, Digest}, + Blake2b, +}; + use scale::{Encode, Decode}; -use serai_client::{primitives::NetworkId, in_instructions::primitives::SignedBatch}; +use serai_client::{ + primitives::NetworkId, + in_instructions::primitives::{Batch, SignedBatch}, +}; pub use serai_db::*; -use crate::tributary::TributarySpec; +use ::tributary::ReadWrite; +use crate::tributary::{TributarySpec, Transaction}; #[derive(Debug)] -pub struct MainDb<'a, D: Db>(&'a mut D); -impl<'a, D: Db> MainDb<'a, D> { - pub fn new(db: &'a mut D) -> Self { - Self(db) - } - +pub struct MainDb(PhantomData); +impl MainDb { fn main_key(dst: &'static [u8], key: impl AsRef<[u8]>) -> Vec { D::key(b"coordinator_main", dst, key) } + fn handled_message_key(network: NetworkId, id: u64) -> Vec { + Self::main_key(b"handled_message", (network, id).encode()) + } + pub fn save_handled_message(txn: &mut D::Transaction<'_>, network: NetworkId, id: u64) { + txn.put(Self::handled_message_key(network, id), []); + } + pub fn handled_message(getter: &G, network: NetworkId, id: u64) -> bool { + getter.get(Self::handled_message_key(network, id)).is_some() + } + fn acive_tributaries_key() -> Vec { Self::main_key(b"active_tributaries", []) } - pub fn active_tributaries(&self) -> (Vec, Vec) { - let bytes = self.0.get(Self::acive_tributaries_key()).unwrap_or(vec![]); + pub fn active_tributaries(getter: &G) -> (Vec, Vec) { + let bytes = getter.get(Self::acive_tributaries_key()).unwrap_or(vec![]); let mut bytes_ref: &[u8] = bytes.as_ref(); let mut tributaries = vec![]; @@ -30,9 +47,9 @@ impl<'a, D: Db> MainDb<'a, D> { (bytes, tributaries) } - pub fn add_active_tributary(&mut self, spec: &TributarySpec) { + pub fn add_active_tributary(txn: &mut D::Transaction<'_>, spec: &TributarySpec) { let key = Self::acive_tributaries_key(); - let (mut existing_bytes, existing) = self.active_tributaries(); + let (mut existing_bytes, existing) = Self::active_tributaries(txn); for tributary in &existing { if tributary == spec { return; @@ -40,38 +57,78 @@ impl<'a, D: Db> MainDb<'a, D> { } spec.write(&mut existing_bytes).unwrap(); - let mut txn = self.0.txn(); txn.put(key, existing_bytes); - txn.commit(); } - fn first_preprocess_key(id: [u8; 32]) -> Vec { - Self::main_key(b"first_preprocess", id) + fn signed_transaction_key(nonce: u32) -> Vec { + Self::main_key(b"signed_transaction", nonce.to_le_bytes()) + } + pub fn save_signed_transaction(txn: &mut D::Transaction<'_>, nonce: u32, tx: Transaction) { + txn.put(Self::signed_transaction_key(nonce), tx.serialize()); + } + pub fn take_signed_transaction(txn: &mut D::Transaction<'_>, nonce: u32) -> Option { + let key = Self::signed_transaction_key(nonce); + let res = txn.get(&key).map(|bytes| Transaction::read(&mut bytes.as_slice()).unwrap()); + if res.is_some() { + txn.del(&key); + } + res } - pub fn save_first_preprocess(txn: &mut D::Transaction<'_>, id: [u8; 32], preprocess: Vec) { - let key = Self::first_preprocess_key(id); + + fn first_preprocess_key(network: NetworkId, id: [u8; 32]) -> Vec { + Self::main_key(b"first_preprocess", (network, id).encode()) + } + pub fn save_first_preprocess( + txn: &mut D::Transaction<'_>, + network: NetworkId, + id: [u8; 32], + preprocess: Vec, + ) { + let key = Self::first_preprocess_key(network, id); if let Some(existing) = txn.get(&key) { assert_eq!(existing, preprocess, "saved a distinct first preprocess"); return; } txn.put(key, preprocess); } - pub fn first_preprocess(getter: &G, id: [u8; 32]) -> Option> { - getter.get(Self::first_preprocess_key(id)) + pub fn first_preprocess(getter: &G, network: NetworkId, id: [u8; 32]) -> Option> { + getter.get(Self::first_preprocess_key(network, id)) + } + + fn expected_batch_key(network: NetworkId, id: u32) -> Vec { + Self::main_key(b"expected_batch", (network, id).encode()) + } + pub fn save_expected_batch(txn: &mut D::Transaction<'_>, batch: &Batch) { + txn.put( + Self::expected_batch_key(batch.network, batch.id), + Blake2b::::digest(batch.instructions.encode()), + ); + } + pub fn expected_batch(getter: &G, network: NetworkId, id: u32) -> Option<[u8; 32]> { + getter.get(Self::expected_batch_key(network, id)).map(|batch| batch.try_into().unwrap()) } fn batch_key(network: NetworkId, id: u32) -> Vec { Self::main_key(b"batch", (network, id).encode()) } - pub fn save_batch(&mut self, batch: SignedBatch) { - let mut txn = self.0.txn(); + pub fn save_batch(txn: &mut D::Transaction<'_>, batch: SignedBatch) { txn.put(Self::batch_key(batch.batch.network, batch.batch.id), batch.encode()); - txn.commit(); } - pub fn batch(&self, network: NetworkId, id: u32) -> Option { - self - .0 + pub fn batch(getter: &G, network: NetworkId, id: u32) -> Option { + getter .get(Self::batch_key(network, id)) .map(|batch| SignedBatch::decode(&mut batch.as_ref()).unwrap()) } + + fn last_verified_batch_key(network: NetworkId) -> Vec { + Self::main_key(b"last_verified_batch", network.encode()) + } + pub fn save_last_verified_batch(txn: &mut D::Transaction<'_>, network: NetworkId, id: u32) { + txn.put(Self::last_verified_batch_key(network), id.to_le_bytes()); + } + pub fn last_verified_batch(getter: &G, network: NetworkId) -> Option { + getter + .get(Self::last_verified_batch_key(network)) + .map(|id| u32::from_le_bytes(id.try_into().unwrap())) + } } diff --git a/coordinator/src/main.rs b/coordinator/src/main.rs index a242bde46..013781f19 100644 --- a/coordinator/src/main.rs +++ b/coordinator/src/main.rs @@ -1,7 +1,3 @@ -#![allow(unused_variables)] -#![allow(unreachable_code)] -#![allow(clippy::diverging_sub_expression)] - use core::{ops::Deref, future::Future}; use std::{ sync::Arc, @@ -27,15 +23,17 @@ use serai_client::{primitives::NetworkId, Public, Serai}; use message_queue::{Service, client::MessageQueue}; use futures::stream::StreamExt; -use tokio::{sync::RwLock, time::sleep}; - -use ::tributary::{ - ReadWrite, ProvidedError, TransactionKind, TransactionTrait, Block, Tributary, TributaryReader, +use tokio::{ + sync::{RwLock, mpsc, broadcast}, + time::sleep, }; +use ::tributary::{ReadWrite, ProvidedError, TransactionKind, TransactionTrait, Block, Tributary}; + mod tributary; -#[rustfmt::skip] -use crate::tributary::{TributarySpec, SignData, Transaction, TributaryDb, scanner::RecognizedIdType}; +use crate::tributary::{ + TributarySpec, SignData, Transaction, TributaryDb, NonceDecider, scanner::RecognizedIdType, +}; mod db; use db::MainDb; @@ -49,30 +47,26 @@ pub mod processors; use processors::Processors; mod substrate; +use substrate::SubstrateDb; #[cfg(test)] pub mod tests; -lazy_static::lazy_static! { - // This is a static to satisfy lifetime expectations - static ref NEW_TRIBUTARIES: RwLock> = RwLock::new(VecDeque::new()); -} - +#[derive(Clone)] pub struct ActiveTributary { pub spec: TributarySpec, - pub tributary: Arc>>, + pub tributary: Arc>, } -type Tributaries = HashMap<[u8; 32], ActiveTributary>; - -// Adds a tributary into the specified HahMap -async fn add_tributary( +// Adds a tributary into the specified HashMap +async fn add_tributary( db: D, key: Zeroizing<::F>, + processors: &Pro, p2p: P, - tributaries: &mut Tributaries, + tributaries: &broadcast::Sender>, spec: TributarySpec, -) -> TributaryReader { +) { log::info!("adding tributary {:?}", spec.set()); let tributary = Tributary::<_, Transaction, _>::new( @@ -80,21 +74,38 @@ async fn add_tributary( db, spec.genesis(), spec.start_time(), - key, + key.clone(), spec.validators(), p2p, ) .await .unwrap(); - let reader = tributary.reader(); - - tributaries.insert( - tributary.genesis(), - ActiveTributary { spec, tributary: Arc::new(RwLock::new(tributary)) }, - ); + // Trigger a DKG for the newly added Tributary + // If we're rebooting, we'll re-fire this message + // This is safe due to the message-queue deduplicating based off the intent system + let set = spec.set(); + processors + .send( + set.network, + processor_messages::key_gen::CoordinatorMessage::GenerateKey { + id: processor_messages::key_gen::KeyGenId { set, attempt: 0 }, + params: frost::ThresholdParams::new( + spec.t(), + spec.n(), + spec + .i(Ristretto::generator() * key.deref()) + .expect("adding a tributary for a set we aren't in set for"), + ) + .unwrap(), + }, + ) + .await; - reader + tributaries + .send(ActiveTributary { spec, tributary: Arc::new(tributary) }) + .map_err(|_| "all ActiveTributary recipients closed") + .unwrap(); } pub async fn scan_substrate( @@ -102,10 +113,11 @@ pub async fn scan_substrate( key: Zeroizing<::F>, processors: Pro, serai: Arc, + new_tributary_spec: mpsc::UnboundedSender, ) { log::info!("scanning substrate"); - let mut db = substrate::SubstrateDb::new(db); + let mut db = SubstrateDb::new(db); let mut next_substrate_block = db.next_block(); let new_substrate_block_notifier = { @@ -156,14 +168,13 @@ pub async fn scan_substrate( log::info!("creating new tributary for {:?}", spec.set()); // Save it to the database - MainDb::new(db).add_active_tributary(&spec); + let mut txn = db.txn(); + MainDb::::add_active_tributary(&mut txn, &spec); + txn.commit(); - // Add it to the queue - // If we reboot before this is read from the queue, the fact it was saved to the database - // means it'll be handled on reboot - async { - NEW_TRIBUTARIES.write().await.push_back(spec); - } + // If we reboot before this is read, the fact it was saved to the database means it'll be + // handled on reboot + new_tributary_spec.send(spec).unwrap(); }, &processors, &serai, @@ -180,131 +191,131 @@ pub async fn scan_substrate( } } -#[allow(clippy::type_complexity)] -pub async fn scan_tributaries< +pub(crate) trait RIDTrait: + Clone + Fn(NetworkId, [u8; 32], RecognizedIdType, [u8; 32], u32) -> FRid +{ +} +impl FRid> + RIDTrait for F +{ +} + +pub(crate) async fn scan_tributaries< D: Db, Pro: Processors, P: P2p, - FRid: Future, - RID: Clone + Fn(NetworkId, [u8; 32], RecognizedIdType, [u8; 32]) -> FRid, + FRid: Send + Future, + RID: 'static + Send + Sync + RIDTrait, >( raw_db: D, key: Zeroizing<::F>, recognized_id: RID, - p2p: P, processors: Pro, serai: Arc, - tributaries: Arc>>, + mut new_tributary: broadcast::Receiver>, ) { log::info!("scanning tributaries"); - let mut tributary_readers = vec![]; - for ActiveTributary { spec, tributary } in tributaries.read().await.values() { - tributary_readers.push((spec.clone(), tributary.read().await.reader())); - } - - // Handle new Tributary blocks - let mut tributary_db = tributary::TributaryDb::new(raw_db.clone()); loop { - // The following handle_new_blocks function may take an arbitrary amount of time - // Accordingly, it may take a long time to acquire a write lock on the tributaries table - // By definition of NEW_TRIBUTARIES, we allow tributaries to be added almost immediately, - // meaning the Substrate scanner won't become blocked on this - { - let mut new_tributaries = NEW_TRIBUTARIES.write().await; - while let Some(spec) = new_tributaries.pop_front() { - let reader = add_tributary( - raw_db.clone(), - key.clone(), - p2p.clone(), - // This is a short-lived write acquisition, which is why it should be fine - &mut *tributaries.write().await, - spec.clone(), - ) - .await; - - // Trigger a DKG for the newly added Tributary - let set = spec.set(); - processors - .send( - set.network, - processor_messages::CoordinatorMessage::KeyGen( - processor_messages::key_gen::CoordinatorMessage::GenerateKey { - id: processor_messages::key_gen::KeyGenId { set, attempt: 0 }, - params: frost::ThresholdParams::new( - spec.t(), - spec.n(), - spec - .i(Ristretto::generator() * key.deref()) - .expect("adding a tribuary for a set we aren't in set for"), - ) - .unwrap(), - }, - ), - ) - .await; - - tributary_readers.push((spec, reader)); - } - } - - for (spec, reader) in &tributary_readers { - tributary::scanner::handle_new_blocks::<_, _, _, _, _, _, P>( - &mut tributary_db, - &key, - recognized_id.clone(), - &processors, - |set, tx| { + match new_tributary.recv().await { + Ok(ActiveTributary { spec, tributary }) => { + // For each Tributary, spawn a dedicated scanner task + tokio::spawn({ + let raw_db = raw_db.clone(); + let key = key.clone(); + let recognized_id = recognized_id.clone(); + let processors = processors.clone(); let serai = serai.clone(); async move { + let spec = &spec; + let reader = tributary.reader(); + let mut tributary_db = tributary::TributaryDb::new(raw_db.clone()); loop { - match serai.publish(&tx).await { - Ok(_) => { - log::info!("set key pair for {set:?}"); - break; - } - // This is assumed to be some ephemeral error due to the assumed fault-free - // creation - // TODO2: Differentiate connection errors from invariants - Err(e) => { - // Check if this failed because the keys were already set by someone else - if matches!(serai.get_keys(spec.set()).await, Ok(Some(_))) { - log::info!("another coordinator set key pair for {:?}", set); - break; + // Obtain the next block notification now to prevent obtaining it immediately after + // the next block occurs + let next_block_notification = tributary.next_block_notification().await; + + tributary::scanner::handle_new_blocks::<_, _, _, _, _, _, P>( + &mut tributary_db, + &key, + recognized_id.clone(), + &processors, + |set, tx| { + let serai = serai.clone(); + async move { + loop { + match serai.publish(&tx).await { + Ok(_) => { + log::info!("set key pair for {set:?}"); + break; + } + // This is assumed to be some ephemeral error due to the assumed fault-free + // creation + // TODO2: Differentiate connection errors from invariants + Err(e) => { + // Check if this failed because the keys were already set by someone else + if matches!(serai.get_keys(spec.set()).await, Ok(Some(_))) { + log::info!("another coordinator set key pair for {:?}", set); + break; + } + + log::error!( + "couldn't connect to Serai node to publish set_keys TX: {:?}", + e + ); + tokio::time::sleep(Duration::from_secs(10)).await; + } + } + } } - - log::error!("couldn't connect to Serai node to publish set_keys TX: {:?}", e); - tokio::time::sleep(Duration::from_secs(10)).await; - } - } + }, + spec, + &reader, + ) + .await; + + next_block_notification + .await + .map_err(|_| "") + .expect("tributary dropped its notifications?"); } } - }, - spec, - reader, - ) - .await; + }); + } + Err(broadcast::error::RecvError::Lagged(_)) => { + panic!("scan_tributaries lagged to handle new_tributary") + } + Err(broadcast::error::RecvError::Closed) => panic!("new_tributary sender closed"), } - - // Sleep for half the block time - // TODO2: Define a notification system for when a new block occurs - sleep(Duration::from_secs((Tributary::::block_time() / 2).into())).await; } } pub async fn heartbeat_tributaries( p2p: P, - tributaries: Arc>>, + mut new_tributary: broadcast::Receiver>, ) { let ten_blocks_of_time = Duration::from_secs((10 * Tributary::::block_time()).into()); + let mut readers = vec![]; loop { - for ActiveTributary { spec: _, tributary } in tributaries.read().await.values() { - let tributary = tributary.read().await; - let tip = tributary.tip().await; - let block_time = SystemTime::UNIX_EPOCH + - Duration::from_secs(tributary.reader().time_of_block(&tip).unwrap_or(0)); + while let Ok(ActiveTributary { spec: _, tributary }) = { + match new_tributary.try_recv() { + Ok(tributary) => Ok(tributary), + Err(broadcast::error::TryRecvError::Empty) => Err(()), + Err(broadcast::error::TryRecvError::Lagged(_)) => { + panic!("heartbeat_tributaries lagged to handle new_tributary") + } + Err(broadcast::error::TryRecvError::Closed) => panic!("new_tributary sender closed"), + } + } { + readers.push(tributary.reader()); + } + + for tributary in &readers { + let tip = tributary.tip(); + let block_time = + SystemTime::UNIX_EPOCH + Duration::from_secs(tributary.time_of_block(&tip).unwrap_or(0)); // Only trigger syncing if the block is more than a minute behind if SystemTime::now() > (block_time + Duration::from_secs(60)) { @@ -331,458 +342,647 @@ pub async fn heartbeat_tributaries( pub async fn handle_p2p( our_key: ::G, p2p: P, - tributaries: Arc>>, + mut new_tributary: broadcast::Receiver>, ) { - loop { - let mut msg = p2p.receive().await; - // Spawn a dedicated task to handle this message, ensuring any singularly latent message - // doesn't hold everything up - // TODO2: Move to one task per tributary (or two. One for Tendermint, one for Tributary) - tokio::spawn({ - let p2p = p2p.clone(); - let tributaries = tributaries.clone(); - async move { - match msg.kind { - P2pMessageKind::KeepAlive => {} - - P2pMessageKind::Tributary(genesis) => { - let tributaries = tributaries.read().await; - let Some(tributary) = tributaries.get(&genesis) else { - log::debug!("received p2p message for unknown network"); - return; - }; + let channels = Arc::new(RwLock::new(HashMap::new())); + tokio::spawn({ + let p2p = p2p.clone(); + let channels = channels.clone(); + async move { + loop { + let tributary = new_tributary.recv().await.unwrap(); + let genesis = tributary.spec.genesis(); - log::trace!("handling message for tributary {:?}", tributary.spec.set()); - if tributary.tributary.read().await.handle_message(&msg.msg).await { - P2p::broadcast(&p2p, msg.kind, msg.msg).await; - } - } + let (send, mut recv) = mpsc::unbounded_channel(); + channels.write().await.insert(genesis, send); - // TODO2: Rate limit this per timestamp - // And/or slash on Heartbeat which justifies a response, since the node obviously was - // offline and we must now use our bandwidth to compensate for them? - P2pMessageKind::Heartbeat(genesis) => { - if msg.msg.len() != 40 { - log::error!("validator sent invalid heartbeat"); - return; - } + tokio::spawn({ + let p2p = p2p.clone(); + async move { + loop { + let mut msg: Message

= recv.recv().await.unwrap(); + match msg.kind { + P2pMessageKind::KeepAlive => {} + + P2pMessageKind::Tributary(msg_genesis) => { + assert_eq!(msg_genesis, genesis); + log::trace!("handling message for tributary {:?}", tributary.spec.set()); + if tributary.tributary.handle_message(&msg.msg).await { + P2p::broadcast(&p2p, msg.kind, msg.msg).await; + } + } - let tributaries = tributaries.read().await; - let Some(tributary) = tributaries.get(&genesis) else { - log::debug!("received heartbeat message for unknown network"); - return; - }; - let tributary_read = tributary.tributary.read().await; - - /* - // Have sqrt(n) nodes reply with the blocks - let mut responders = (tributary.spec.n() as f32).sqrt().floor() as u64; - // Try to have at least 3 responders - if responders < 3 { - responders = tributary.spec.n().min(3).into(); - } - */ - - // Have up to three nodes respond - let responders = u64::from(tributary.spec.n().min(3)); - - // Decide which nodes will respond by using the latest block's hash as a mutually agreed - // upon entropy source - // This isn't a secure source of entropy, yet it's fine for this - let entropy = u64::from_le_bytes(tributary_read.tip().await[.. 8].try_into().unwrap()); - // If n = 10, responders = 3, we want start to be 0 ..= 7 (so the highest is 7, 8, 9) - // entropy % (10 + 1) - 3 = entropy % 8 = 0 ..= 7 - let start = - usize::try_from(entropy % (u64::from(tributary.spec.n() + 1) - responders)).unwrap(); - let mut selected = false; - for validator in - &tributary.spec.validators()[start .. (start + usize::try_from(responders).unwrap())] - { - if our_key == validator.0 { - selected = true; - break; - } - } - if !selected { - log::debug!("received heartbeat and not selected to respond"); - return; - } + // TODO2: Rate limit this per timestamp + // And/or slash on Heartbeat which justifies a response, since the node obviously + // was offline and we must now use our bandwidth to compensate for them? + P2pMessageKind::Heartbeat(msg_genesis) => { + assert_eq!(msg_genesis, genesis); + if msg.msg.len() != 40 { + log::error!("validator sent invalid heartbeat"); + continue; + } + + let p2p = p2p.clone(); + let spec = tributary.spec.clone(); + let reader = tributary.tributary.reader(); + // Spawn a dedicated task as this may require loading large amounts of data from + // disk and take a notable amount of time + tokio::spawn(async move { + /* + // Have sqrt(n) nodes reply with the blocks + let mut responders = (tributary.spec.n() as f32).sqrt().floor() as u64; + // Try to have at least 3 responders + if responders < 3 { + responders = tributary.spec.n().min(3).into(); + } + */ + + // Have up to three nodes respond + let responders = u64::from(spec.n().min(3)); + + // Decide which nodes will respond by using the latest block's hash as a + // mutually agreed upon entropy source + // This isn't a secure source of entropy, yet it's fine for this + let entropy = u64::from_le_bytes(reader.tip()[.. 8].try_into().unwrap()); + // If n = 10, responders = 3, we want `start` to be 0 ..= 7 + // (so the highest is 7, 8, 9) + // entropy % (10 + 1) - 3 = entropy % 8 = 0 ..= 7 + let start = + usize::try_from(entropy % (u64::from(spec.n() + 1) - responders)).unwrap(); + let mut selected = false; + for validator in + &spec.validators()[start .. (start + usize::try_from(responders).unwrap())] + { + if our_key == validator.0 { + selected = true; + break; + } + } + if !selected { + log::debug!("received heartbeat and not selected to respond"); + return; + } - log::debug!("received heartbeat and selected to respond"); + log::debug!("received heartbeat and selected to respond"); - let reader = tributary_read.reader(); - drop(tributary_read); + let mut latest = msg.msg[.. 32].try_into().unwrap(); + while let Some(next) = reader.block_after(&latest) { + let mut res = reader.block(&next).unwrap().serialize(); + res.extend(reader.commit(&next).unwrap()); + // Also include the timestamp used within the Heartbeat + res.extend(&msg.msg[32 .. 40]); + p2p.send(msg.sender, P2pMessageKind::Block(spec.genesis()), res).await; + latest = next; + } + }); + } - let mut latest = msg.msg[.. 32].try_into().unwrap(); - while let Some(next) = reader.block_after(&latest) { - let mut res = reader.block(&next).unwrap().serialize(); - res.extend(reader.commit(&next).unwrap()); - // Also include the timestamp used within the Heartbeat - res.extend(&msg.msg[32 .. 40]); - p2p.send(msg.sender, P2pMessageKind::Block(tributary.spec.genesis()), res).await; - latest = next; + P2pMessageKind::Block(msg_genesis) => { + assert_eq!(msg_genesis, genesis); + let mut msg_ref: &[u8] = msg.msg.as_ref(); + let Ok(block) = Block::::read(&mut msg_ref) else { + log::error!("received block message with an invalidly serialized block"); + continue; + }; + // Get just the commit + msg.msg.drain(.. (msg.msg.len() - msg_ref.len())); + msg.msg.drain((msg.msg.len() - 8) ..); + + let res = tributary.tributary.sync_block(block, msg.msg).await; + log::debug!("received block from {:?}, sync_block returned {}", msg.sender, res); + } + } } } + }); + } + } + }); - P2pMessageKind::Block(genesis) => { - let mut msg_ref: &[u8] = msg.msg.as_ref(); - let Ok(block) = Block::::read(&mut msg_ref) else { - log::error!("received block message with an invalidly serialized block"); - return; - }; - // Get just the commit - msg.msg.drain(.. (msg.msg.len() - msg_ref.len())); - msg.msg.drain((msg.msg.len() - 8) ..); - - let tributaries = tributaries.read().await; - let Some(tributary) = tributaries.get(&genesis) else { - log::debug!("received block message for unknown network"); - return; - }; - - let res = tributary.tributary.read().await.sync_block(block, msg.msg).await; - log::debug!("received block from {:?}, sync_block returned {}", msg.sender, res); - } + loop { + let msg = p2p.receive().await; + match msg.kind { + P2pMessageKind::KeepAlive => {} + P2pMessageKind::Tributary(genesis) => { + if let Some(channel) = channels.read().await.get(&genesis) { + channel.send(msg).unwrap(); } } - }); + P2pMessageKind::Heartbeat(genesis) => { + if let Some(channel) = channels.read().await.get(&genesis) { + channel.send(msg).unwrap(); + } + } + P2pMessageKind::Block(genesis) => { + if let Some(channel) = channels.read().await.get(&genesis) { + channel.send(msg).unwrap(); + } + } + } } } -pub async fn publish_transaction( +async fn publish_signed_transaction( + db: &mut D, tributary: &Tributary, tx: Transaction, ) { log::debug!("publishing transaction {}", hex::encode(tx.hash())); - if let TransactionKind::Signed(signed) = tx.kind() { - if tributary - .next_nonce(signed.signer) - .await - .expect("we don't have a nonce, meaning we aren't a participant on this tributary") > - signed.nonce - { - log::warn!("we've already published this transaction. this should only appear on reboot"); - } else { - // We should've created a valid transaction - assert!(tributary.add_transaction(tx).await, "created an invalid transaction"); - } + + let mut txn = db.txn(); + let signer = if let TransactionKind::Signed(signed) = tx.kind() { + let signer = signed.signer; + + // Safe as we should deterministically create transactions, meaning if this is already on-disk, + // it's what we're saving now + MainDb::::save_signed_transaction(&mut txn, signed.nonce, tx); + + signer } else { - panic!("non-signed transaction passed to publish_transaction"); + panic!("non-signed transaction passed to publish_signed_transaction"); + }; + + // If we're trying to publish 5, when the last transaction published was 3, this will delay + // publication until the point in time we publish 4 + while let Some(tx) = MainDb::::take_signed_transaction( + &mut txn, + tributary + .next_nonce(signer) + .await + .expect("we don't have a nonce, meaning we aren't a participant on this tributary"), + ) { + // We should've created a valid transaction + // This does assume publish_signed_transaction hasn't been called twice with the same + // transaction, which risks a race condition on the validity of this assert + // Our use case only calls this function sequentially + assert!(tributary.add_transaction(tx).await, "created an invalid transaction"); } + txn.commit(); } -pub async fn handle_processors( +async fn handle_processor_messages( mut db: D, key: Zeroizing<::F>, serai: Arc, mut processors: Pro, - tributaries: Arc>>, + network: NetworkId, + mut new_tributary: mpsc::UnboundedReceiver>, ) { + let mut db_clone = db.clone(); // Enables cloning the DB while we have a txn let pub_key = Ristretto::generator() * key.deref(); + let mut tributaries = HashMap::new(); + loop { - // TODO: Dispatch this message to a task dedicated to handling this processor, preventing one - // processor from holding up all the others. This would require a peek method be added to the - // message-queue (to view multiple future messages at once) - // TODO: Do we handle having handled a message, by DB, yet having rebooted before `ack`ing it? - // Does the processor? - let msg = processors.recv().await; - - // TODO2: This is slow, and only works as long as a network only has a single Tributary - // (which means there's a lack of multisig rotation) - let spec = { - let mut spec = None; - for tributary in tributaries.read().await.values() { - if tributary.spec.set().network == msg.network { - spec = Some(tributary.spec.clone()); - break; - } + match new_tributary.try_recv() { + Ok(tributary) => { + tributaries.insert(tributary.spec.set().session, tributary); } - spec.expect("received message from processor we don't have a tributary for") - }; + Err(mpsc::error::TryRecvError::Empty) => {} + Err(mpsc::error::TryRecvError::Disconnected) => { + panic!("handle_processor_messages new_tributary sender closed") + } + } - let genesis = spec.genesis(); - // TODO: We probably want to NOP here, not panic? - let my_i = spec.i(pub_key).expect("processor message for network we aren't a validator in"); + // TODO: Check this ID is sane (last handled ID or expected next ID) + let msg = processors.recv(network).await; - let tx = match msg.msg.clone() { - ProcessorMessage::KeyGen(inner_msg) => match inner_msg { - key_gen::ProcessorMessage::Commitments { id, commitments } => { - Some(Transaction::DkgCommitments(id.attempt, commitments, Transaction::empty_signed())) - } - key_gen::ProcessorMessage::Shares { id, mut shares } => { - // Create a MuSig-based machine to inform Substrate of this key generation - let nonces = crate::tributary::dkg_confirmation_nonces(&key, &spec, id.attempt); - - let mut tx_shares = Vec::with_capacity(shares.len()); - for i in 1 ..= spec.n() { - let i = Participant::new(i).unwrap(); - if i == my_i { - continue; + // TODO: We need to verify the Batches published to Substrate + + if !MainDb::::handled_message(&db, msg.network, msg.id) { + let mut txn = db.txn(); + + let relevant_tributary = match &msg.msg { + // We'll only receive these if we fired GenerateKey, which we'll only do if if we're + // in-set, making the Tributary relevant + ProcessorMessage::KeyGen(inner_msg) => match inner_msg { + key_gen::ProcessorMessage::Commitments { id, .. } => Some(id.set.session), + key_gen::ProcessorMessage::Shares { id, .. } => Some(id.set.session), + key_gen::ProcessorMessage::GeneratedKeyPair { id, .. } => Some(id.set.session), + }, + // TODO: Review replacing key with Session in messages? + ProcessorMessage::Sign(inner_msg) => match inner_msg { + // We'll only receive Preprocess and Share if we're actively signing + sign::ProcessorMessage::Preprocess { id, .. } => { + Some(SubstrateDb::::session_for_key(&txn, &id.key).unwrap()) + } + sign::ProcessorMessage::Share { id, .. } => { + Some(SubstrateDb::::session_for_key(&txn, &id.key).unwrap()) + } + // While the Processor's Scanner will always emit Completed, that's routed through the + // Signer and only becomes a ProcessorMessage::Completed if the Signer is present and + // confirms it + sign::ProcessorMessage::Completed { key, .. } => { + Some(SubstrateDb::::session_for_key(&txn, key).unwrap()) + } + }, + ProcessorMessage::Coordinator(inner_msg) => match inner_msg { + // This is a special case as it's relevant to *all* Tributaries + // It doesn't return a Tributary to become `relevant_tributary` though + coordinator::ProcessorMessage::SubstrateBlockAck { network, block, plans } => { + assert_eq!( + *network, msg.network, + "processor claimed to be a different network than it was for SubstrateBlockAck", + ); + + // TODO: Find all Tributaries active at this Substrate block, and make sure we have + // them all + + for tributary in tributaries.values() { + // TODO: This needs to be scoped per multisig + TributaryDb::::set_plan_ids(&mut txn, tributary.spec.genesis(), *block, plans); + + let tx = Transaction::SubstrateBlock(*block); + log::trace!("processor message effected transaction {}", hex::encode(tx.hash())); + log::trace!("providing transaction {}", hex::encode(tx.hash())); + let res = tributary.tributary.provide_transaction(tx).await; + if !(res.is_ok() || (res == Err(ProvidedError::AlreadyProvided))) { + panic!("provided an invalid transaction: {res:?}"); + } } - tx_shares - .push(shares.remove(&i).expect("processor didn't send share for another validator")); + + None + } + // We'll only fire these if we are the Substrate signer, making the Tributary relevant + coordinator::ProcessorMessage::BatchPreprocess { id, .. } => { + Some(SubstrateDb::::session_for_key(&txn, &id.key).unwrap()) } + coordinator::ProcessorMessage::BatchShare { id, .. } => { + Some(SubstrateDb::::session_for_key(&txn, &id.key).unwrap()) + } + }, + // These don't return a relevant Tributary as there's no Tributary with action expected + ProcessorMessage::Substrate(inner_msg) => match inner_msg { + processor_messages::substrate::ProcessorMessage::Batch { batch } => { + assert_eq!( + batch.network, msg.network, + "processor sent us a batch for a different network than it was for", + ); + let this_batch_id = batch.id; + MainDb::::save_expected_batch(&mut txn, batch); + + // Re-define batch + // We can't drop it, yet it shouldn't be accidentally used in the following block + #[allow(clippy::let_unit_value, unused_variables)] + let batch = (); + + // Verify all `Batch`s which we've already indexed from Substrate + // This won't be complete, as it only runs when a `Batch` message is received, which + // will be before we get a `SignedBatch`. It is, however, incremental. We can use a + // complete version to finish the last section when we need a complete version. + let last = MainDb::::last_verified_batch(&txn, msg.network); + // This variable exists so Rust can verify Send/Sync properties + let mut faulty = None; + for id in last.map(|last| last + 1).unwrap_or(0) ..= this_batch_id { + if let Some(on_chain) = SubstrateDb::::batch_instructions_hash(&txn, network, id) { + let off_chain = MainDb::::expected_batch(&txn, network, id).unwrap(); + if on_chain != off_chain { + faulty = Some((id, off_chain, on_chain)); + break; + } + MainDb::::save_last_verified_batch(&mut txn, msg.network, id); + } + } - Some(Transaction::DkgShares { - attempt: id.attempt, - shares: tx_shares, - confirmation_nonces: nonces, - signed: Transaction::empty_signed(), - }) - } - key_gen::ProcessorMessage::GeneratedKeyPair { id, substrate_key, network_key } => { - assert_eq!( - id.set.network, msg.network, - "processor claimed to be a different network than it was for GeneratedKeyPair", - ); - // TODO: Also check the other KeyGenId fields - - // Tell the Tributary the key pair, get back the share for the MuSig signature - let mut txn = db.txn(); - let share = crate::tributary::generated_key_pair::( - &mut txn, - &key, - &spec, - &(Public(substrate_key), network_key.try_into().unwrap()), - id.attempt, - ); - txn.commit(); - - match share { - Ok(share) => { - Some(Transaction::DkgConfirmed(id.attempt, share, Transaction::empty_signed())) + if let Some((id, off_chain, on_chain)) = faulty { + // Halt operations on this network and spin, as this is a critical fault + loop { + log::error!( + "{}! network: {:?} id: {} off-chain: {} on-chain: {}", + "on-chain batch doesn't match off-chain", + network, + id, + hex::encode(off_chain), + hex::encode(on_chain), + ); + sleep(Duration::from_secs(60)).await; + } } - Err(p) => todo!("participant {p:?} sent invalid DKG confirmation preprocesses"), - } - } - }, - ProcessorMessage::Sign(msg) => match msg { - sign::ProcessorMessage::Preprocess { id, preprocess } => { - if id.attempt == 0 { - let mut txn = db.txn(); - MainDb::::save_first_preprocess(&mut txn, id.id, preprocess); - txn.commit(); None - } else { - Some(Transaction::SignPreprocess(SignData { - plan: id.id, - attempt: id.attempt, - data: preprocess, - signed: Transaction::empty_signed(), - })) } - } - sign::ProcessorMessage::Share { id, share } => Some(Transaction::SignShare(SignData { - plan: id.id, - attempt: id.attempt, - data: share, - signed: Transaction::empty_signed(), - })), - sign::ProcessorMessage::Completed { key: _, id, tx } => { - let r = Zeroizing::new(::F::random(&mut OsRng)); - #[allow(non_snake_case)] - let R = ::generator() * r.deref(); - let mut tx = Transaction::SignCompleted { - plan: id, - tx_hash: tx, - first_signer: pub_key, - signature: SchnorrSignature { R, s: ::F::ZERO }, - }; - let signed = SchnorrSignature::sign(&key, r, tx.sign_completed_challenge()); - match &mut tx { - Transaction::SignCompleted { signature, .. } => { - *signature = signed; + // If this is a new Batch, immediately publish it (if we can) + processor_messages::substrate::ProcessorMessage::SignedBatch { batch } => { + assert_eq!( + batch.batch.network, msg.network, + "processor sent us a signed batch for a different network than it was for", + ); + // TODO: Check this key's key pair's substrate key is authorized to publish batches + + log::debug!("received batch {:?} {}", batch.batch.network, batch.batch.id); + + // Save this batch to the disk + MainDb::::save_batch(&mut txn, batch.clone()); + + // Get the next-to-execute batch ID + async fn get_next(serai: &Serai, network: NetworkId) -> u32 { + let mut first = true; + loop { + if !first { + log::error!( + "{} {network:?}", + "couldn't connect to Serai node to get the next batch ID for", + ); + tokio::time::sleep(Duration::from_secs(5)).await; + } + first = false; + + let Ok(latest_block) = serai.get_latest_block().await else { + continue; + }; + let Ok(last) = serai.get_last_batch_for_network(latest_block.hash(), network).await + else { + continue; + }; + break if let Some(last) = last { last + 1 } else { 0 }; + } + } + let mut next = get_next(&serai, network).await; + + // Since we have a new batch, publish all batches yet to be published to Serai + // This handles the edge-case where batch n+1 is signed before batch n is + let mut batches = VecDeque::new(); + while let Some(batch) = MainDb::::batch(&txn, network, next) { + batches.push_back(batch); + next += 1; } - _ => unreachable!(), + + while let Some(batch) = batches.pop_front() { + // If this Batch should no longer be published, continue + if get_next(&serai, network).await > batch.batch.id { + continue; + } + + let tx = Serai::execute_batch(batch.clone()); + log::debug!( + "attempting to publish batch {:?} {}", + batch.batch.network, + batch.batch.id, + ); + // This publish may fail if this transactions already exists in the mempool, which is + // possible, or if this batch was already executed on-chain + // Either case will have eventual resolution and be handled by the above check on if + // this batch should execute + let res = serai.publish(&tx).await; + if res.is_ok() { + log::info!( + "published batch {network:?} {} (block {})", + batch.batch.id, + hex::encode(batch.batch.block), + ); + } else { + log::debug!( + "couldn't publish batch {:?} {}: {:?}", + batch.batch.network, + batch.batch.id, + res, + ); + // If we failed to publish it, restore it + batches.push_front(batch); + } + } + + None } - Some(tx) - } - }, - ProcessorMessage::Coordinator(inner_msg) => match inner_msg { - coordinator::ProcessorMessage::SubstrateBlockAck { network, block, plans } => { - assert_eq!( - network, msg.network, - "processor claimed to be a different network than it was for SubstrateBlockAck", - ); - - // Safe to use its own txn since this is static and just needs to be written before we - // provide SubstrateBlock - let mut txn = db.txn(); - TributaryDb::::set_plan_ids(&mut txn, genesis, block, &plans); - txn.commit(); - - Some(Transaction::SubstrateBlock(block)) - } - coordinator::ProcessorMessage::BatchPreprocess { id, block, preprocess } => { - log::info!( - "informed of batch (sign ID {}, attempt {}) for block {}", - hex::encode(id.id), - id.attempt, - hex::encode(block), - ); - // If this is the first attempt instance, wait until we synchronize around the batch - // first - if id.attempt == 0 { - // Save the preprocess to disk so we can publish it later - // This is fine to use its own TX since it's static and just needs to be written - // before this message finishes it handling (or with this message's finished handling) - let mut txn = db.txn(); - MainDb::::save_first_preprocess(&mut txn, id.id, preprocess); - txn.commit(); - - Some(Transaction::Batch(block.0, id.id)) - } else { - Some(Transaction::BatchPreprocess(SignData { + }, + }; + + // If there's a relevant Tributary... + if let Some(relevant_tributary) = relevant_tributary { + // Make sure we have it + // Per the reasoning above, we only return a Tributary as relevant if we're a participant + // Accordingly, we do *need* to have this Tributary now to handle it UNLESS the Tributary + // has already completed and this is simply an old message + // TODO: Check if the Tributary has already been completed + let Some(ActiveTributary { spec, tributary }) = tributaries.get(&relevant_tributary) else { + // Since we don't, sleep for a fraction of a second and move to the next loop iteration + // At the start of the loop, we'll check for new tributaries, making this eventually + // resolve + sleep(Duration::from_millis(100)).await; + continue; + }; + + let genesis = spec.genesis(); + + let tx = match msg.msg.clone() { + ProcessorMessage::KeyGen(inner_msg) => match inner_msg { + key_gen::ProcessorMessage::Commitments { id, commitments } => Some( + Transaction::DkgCommitments(id.attempt, commitments, Transaction::empty_signed()), + ), + key_gen::ProcessorMessage::Shares { id, mut shares } => { + // Create a MuSig-based machine to inform Substrate of this key generation + let nonces = crate::tributary::dkg_confirmation_nonces(&key, spec, id.attempt); + + let mut tx_shares = Vec::with_capacity(shares.len()); + for i in 1 ..= spec.n() { + let i = Participant::new(i).unwrap(); + if i == + spec + .i(pub_key) + .expect("processor message to DKG for a session we aren't a validator in") + { + continue; + } + tx_shares.push( + shares.remove(&i).expect("processor didn't send share for another validator"), + ); + } + + Some(Transaction::DkgShares { + attempt: id.attempt, + shares: tx_shares, + confirmation_nonces: nonces, + signed: Transaction::empty_signed(), + }) + } + key_gen::ProcessorMessage::GeneratedKeyPair { id, substrate_key, network_key } => { + assert_eq!( + id.set.network, msg.network, + "processor claimed to be a different network than it was for GeneratedKeyPair", + ); + // TODO2: Also check the other KeyGenId fields + + // Tell the Tributary the key pair, get back the share for the MuSig signature + let share = crate::tributary::generated_key_pair::( + &mut txn, + &key, + spec, + &(Public(substrate_key), network_key.try_into().unwrap()), + id.attempt, + ); + + match share { + Ok(share) => { + Some(Transaction::DkgConfirmed(id.attempt, share, Transaction::empty_signed())) + } + Err(p) => { + todo!("participant {p:?} sent invalid DKG confirmation preprocesses") + } + } + } + }, + ProcessorMessage::Sign(msg) => match msg { + sign::ProcessorMessage::Preprocess { id, preprocess } => { + if id.attempt == 0 { + MainDb::::save_first_preprocess(&mut txn, network, id.id, preprocess); + + None + } else { + Some(Transaction::SignPreprocess(SignData { + plan: id.id, + attempt: id.attempt, + data: preprocess, + signed: Transaction::empty_signed(), + })) + } + } + sign::ProcessorMessage::Share { id, share } => Some(Transaction::SignShare(SignData { plan: id.id, attempt: id.attempt, - data: preprocess, + data: share, signed: Transaction::empty_signed(), - })) - } - } - coordinator::ProcessorMessage::BatchShare { id, share } => { - Some(Transaction::BatchShare(SignData { - plan: id.id, - attempt: id.attempt, - data: share.to_vec(), - signed: Transaction::empty_signed(), - })) - } - }, - ProcessorMessage::Substrate(inner_msg) => match inner_msg { - processor_messages::substrate::ProcessorMessage::Update { batch } => { - assert_eq!( - batch.batch.network, msg.network, - "processor sent us a batch for a different network than it was for", - ); - // TODO: Check this key's key pair's substrate key is authorized to publish batches - - // Save this batch to the disk - MainDb::new(&mut db).save_batch(batch); - - /* - Use a dedicated task to publish batches due to the latency potentially incurred. - - This does not guarantee the batch has actually been published when the message is - `ack`ed to message-queue. Accordingly, if we reboot, these batches would be dropped - (as we wouldn't see the `Update` again, triggering our re-attempt to publish). - - The solution to this is to have the task try not to publish the batch which caused it - to be spawned, yet all saved batches which have yet to published. This does risk having - multiple tasks trying to publish all pending batches, yet these aren't notably complex. - */ - tokio::spawn({ - let mut db = db.clone(); - let serai = serai.clone(); - let network = msg.network; - async move { - // Since we have a new batch, publish all batches yet to be published to Serai - // This handles the edge-case where batch n+1 is signed before batch n is - while let Some(batch) = { - // Get the next-to-execute batch ID - let next = { - let mut first = true; - loop { - if !first { - log::error!( - "couldn't connect to Serai node to get the next batch ID for {network:?}", - ); - tokio::time::sleep(Duration::from_secs(5)).await; - } - first = false; - - let Ok(latest_block) = serai.get_latest_block().await else { continue }; - let Ok(last) = - serai.get_last_batch_for_network(latest_block.hash(), network).await - else { - continue; - }; - break if let Some(last) = last { last + 1 } else { 0 }; - } - }; - - // If we have this batch, attempt to publish it - MainDb::new(&mut db).batch(network, next) - } { - let id = batch.batch.id; - let block = batch.batch.block; - - let tx = Serai::execute_batch(batch); - // This publish may fail if this transactions already exists in the mempool, which - // is possible, or if this batch was already executed on-chain - // Either case will have eventual resolution and be handled by the above check on - // if this block should execute - if serai.publish(&tx).await.is_ok() { - log::info!("published batch {network:?} {id} (block {})", hex::encode(block)); + })), + sign::ProcessorMessage::Completed { key: _, id, tx } => { + let r = Zeroizing::new(::F::random(&mut OsRng)); + #[allow(non_snake_case)] + let R = ::generator() * r.deref(); + let mut tx = Transaction::SignCompleted { + plan: id, + tx_hash: tx, + first_signer: pub_key, + signature: SchnorrSignature { R, s: ::F::ZERO }, + }; + let signed = SchnorrSignature::sign(&key, r, tx.sign_completed_challenge()); + match &mut tx { + Transaction::SignCompleted { signature, .. } => { + *signature = signed; } + _ => unreachable!(), } + Some(tx) } - }); + }, + ProcessorMessage::Coordinator(inner_msg) => match inner_msg { + coordinator::ProcessorMessage::SubstrateBlockAck { .. } => unreachable!(), + coordinator::ProcessorMessage::BatchPreprocess { id, block, preprocess } => { + log::info!( + "informed of batch (sign ID {}, attempt {}) for block {}", + hex::encode(id.id), + id.attempt, + hex::encode(block), + ); + // If this is the first attempt instance, wait until we synchronize around + // the batch first + if id.attempt == 0 { + MainDb::::save_first_preprocess(&mut txn, spec.set().network, id.id, preprocess); + + Some(Transaction::Batch(block.0, id.id)) + } else { + Some(Transaction::BatchPreprocess(SignData { + plan: id.id, + attempt: id.attempt, + data: preprocess, + signed: Transaction::empty_signed(), + })) + } + } + coordinator::ProcessorMessage::BatchShare { id, share } => { + Some(Transaction::BatchShare(SignData { + plan: id.id, + attempt: id.attempt, + data: share.to_vec(), + signed: Transaction::empty_signed(), + })) + } + }, + ProcessorMessage::Substrate(inner_msg) => match inner_msg { + processor_messages::substrate::ProcessorMessage::Batch { .. } => unreachable!(), + processor_messages::substrate::ProcessorMessage::SignedBatch { .. } => unreachable!(), + }, + }; - None - } - }, - }; - - // If this created a transaction, publish it - if let Some(mut tx) = tx { - log::trace!("processor message effected transaction {}", hex::encode(tx.hash())); - let tributaries = tributaries.read().await; - log::trace!("read global tributaries"); - let Some(tributary) = tributaries.get(&genesis) else { - // TODO: This can happen since Substrate tells the Processor to generate commitments - // at the same time it tells the Tributary to be created - // There's no guarantee the Tributary will have been created though - panic!("processor is operating on tributary we don't have"); - }; - let tributary = tributary.tributary.read().await; - log::trace!("read specific tributary"); - - match tx.kind() { - TransactionKind::Provided(_) => { - log::trace!("providing transaction {}", hex::encode(tx.hash())); - let res = tributary.provide_transaction(tx).await; - if !(res.is_ok() || (res == Err(ProvidedError::AlreadyProvided))) { - panic!("provided an invalid transaction: {res:?}"); + // If this created a transaction, publish it + if let Some(mut tx) = tx { + log::trace!("processor message effected transaction {}", hex::encode(tx.hash())); + + match tx.kind() { + TransactionKind::Provided(_) => { + log::trace!("providing transaction {}", hex::encode(tx.hash())); + let res = tributary.provide_transaction(tx).await; + if !(res.is_ok() || (res == Err(ProvidedError::AlreadyProvided))) { + panic!("provided an invalid transaction: {res:?}"); + } + } + TransactionKind::Unsigned => { + log::trace!("publishing unsigned transaction {}", hex::encode(tx.hash())); + // Ignores the result since we can't differentiate already in-mempool from + // already on-chain from invalid + // TODO: Don't ignore the result + tributary.add_transaction(tx).await; + } + TransactionKind::Signed(_) => { + log::trace!("getting next nonce for Tributary TX in response to processor message"); + + let nonce = loop { + let Some(nonce) = NonceDecider::::nonce(&txn, genesis, &tx) + .expect("signed TX didn't have nonce") + else { + // This can be None if: + // 1) We scanned the relevant transaction(s) in a Tributary block + // 2) The processor was sent a message and responded + // 3) The Tributary TXN has yet to be committed + log::warn!("nonce has yet to be saved for processor-instigated transaction"); + sleep(Duration::from_millis(100)).await; + continue; + }; + break nonce; + }; + tx.sign(&mut OsRng, genesis, &key, nonce); + + publish_signed_transaction(&mut db_clone, tributary, tx).await; + } } } - TransactionKind::Unsigned => { - log::trace!("publishing unsigned transaction {}", hex::encode(tx.hash())); - // Ignores the result since we can't differentiate already in-mempool from already - // on-chain from invalid - // TODO: Don't ignore the result - tributary.add_transaction(tx).await; - } - TransactionKind::Signed(_) => { - // Get the next nonce - // TODO: This should be deterministic, not just DB-backed, to allow rebuilding validators - // without the prior instance's DB - // let mut txn = db.txn(); - // let nonce = MainDb::tx_nonce(&mut txn, msg.id, tributary); - - // TODO: This isn't deterministic, or at least DB-backed, and accordingly is unsafe - log::trace!("getting next nonce for Tributary TX in response to processor message"); - let nonce = tributary - .next_nonce(Ristretto::generator() * key.deref()) - .await - .expect("publishing a TX to a tributary we aren't in"); - tx.sign(&mut OsRng, genesis, &key, nonce); - - publish_transaction(&tributary, tx).await; - - // txn.commit(); - } } + + MainDb::::save_handled_message(&mut txn, msg.network, msg.id); + txn.commit(); } processors.ack(msg).await; } } +pub async fn handle_processors( + db: D, + key: Zeroizing<::F>, + serai: Arc, + processors: Pro, + mut new_tributary: broadcast::Receiver>, +) { + let mut channels = HashMap::new(); + for network in [NetworkId::Bitcoin, NetworkId::Ethereum, NetworkId::Monero] { + let (send, recv) = mpsc::unbounded_channel(); + tokio::spawn(handle_processor_messages( + db.clone(), + key.clone(), + serai.clone(), + processors.clone(), + network, + recv, + )); + channels.insert(network, send); + } + + // Listen to new tributary events + loop { + let tributary = new_tributary.recv().await.unwrap(); + channels[&tributary.spec.set().network].send(tributary).unwrap(); + } +} + pub async fn run( - mut raw_db: D, + raw_db: D, key: Zeroizing<::F>, p2p: P, processors: Pro, @@ -790,44 +990,88 @@ pub async fn run( ) { let serai = Arc::new(serai); + let (new_tributary_spec_send, mut new_tributary_spec_recv) = mpsc::unbounded_channel(); + // Reload active tributaries from the database + for spec in MainDb::::active_tributaries(&raw_db).1 { + new_tributary_spec_send.send(spec).unwrap(); + } + // Handle new Substrate blocks - tokio::spawn(scan_substrate(raw_db.clone(), key.clone(), processors.clone(), serai.clone())); + tokio::spawn(scan_substrate( + raw_db.clone(), + key.clone(), + processors.clone(), + serai.clone(), + new_tributary_spec_send, + )); // Handle the Tributaries - // Arc so this can be shared between the Tributary scanner task and the P2P task - // Write locks on this may take a while to acquire - let tributaries = Arc::new(RwLock::new(HashMap::<[u8; 32], ActiveTributary>::new())); + // This should be large enough for an entire rotation of all tributaries + // If it's too small, the coordinator fail to boot, which is a decent sanity check + let (new_tributary, mut new_tributary_listener_1) = broadcast::channel(32); + let new_tributary_listener_2 = new_tributary.subscribe(); + let new_tributary_listener_3 = new_tributary.subscribe(); + let new_tributary_listener_4 = new_tributary.subscribe(); + let new_tributary_listener_5 = new_tributary.subscribe(); - // Reload active tributaries from the database - for spec in MainDb::new(&mut raw_db).active_tributaries().1 { - let _ = add_tributary( - raw_db.clone(), - key.clone(), - p2p.clone(), - &mut *tributaries.write().await, - spec, - ) - .await; - } + // Spawn a task to further add Tributaries as needed + tokio::spawn({ + let raw_db = raw_db.clone(); + let key = key.clone(); + let processors = processors.clone(); + let p2p = p2p.clone(); + async move { + loop { + let spec = new_tributary_spec_recv.recv().await.unwrap(); + add_tributary( + raw_db.clone(), + key.clone(), + &processors, + p2p.clone(), + &new_tributary, + spec.clone(), + ) + .await; + } + } + }); // When we reach synchrony on an event requiring signing, send our preprocess for it let recognized_id = { let raw_db = raw_db.clone(); let key = key.clone(); - let tributaries = tributaries.clone(); - move |network, genesis, id_type, id| { - let raw_db = raw_db.clone(); + + let tributaries = Arc::new(RwLock::new(HashMap::new())); + tokio::spawn({ + let tributaries = tributaries.clone(); + async move { + loop { + match new_tributary_listener_1.recv().await { + Ok(tributary) => { + tributaries.write().await.insert(tributary.spec.genesis(), tributary.tributary); + } + Err(broadcast::error::RecvError::Lagged(_)) => { + panic!("recognized_id lagged to handle new_tributary") + } + Err(broadcast::error::RecvError::Closed) => panic!("new_tributary sender closed"), + } + } + } + }); + + move |network, genesis, id_type, id, nonce| { + let mut raw_db = raw_db.clone(); let key = key.clone(); let tributaries = tributaries.clone(); async move { - // SubstrateBlockAck is fired before Preprocess, creating a race between Tributary ack - // of the SubstrateBlock and the sending of all Preprocesses + // The transactions for these are fired before the preprocesses are actually + // received/saved, creating a race between Tributary ack and the availability of all + // Preprocesses // This waits until the necessary preprocess is available let get_preprocess = |raw_db, id| async move { loop { - let Some(preprocess) = MainDb::::first_preprocess(raw_db, id) else { - assert_eq!(id_type, RecognizedIdType::Plan); + let Some(preprocess) = MainDb::::first_preprocess(raw_db, network, id) else { sleep(Duration::from_millis(100)).await; continue; }; @@ -851,21 +1095,14 @@ pub async fn run( }), }; + tx.sign(&mut OsRng, genesis, &key, nonce); + let tributaries = tributaries.read().await; let Some(tributary) = tributaries.get(&genesis) else { + // TODO: This may happen if the task above is simply slow panic!("tributary we don't have came to consensus on an Batch"); }; - let tributary = tributary.tributary.read().await; - - // TODO: Same note as prior nonce acquisition - log::trace!("getting next nonce for Tributary TX containing Batch signing data"); - let nonce = tributary - .next_nonce(Ristretto::generator() * key.deref()) - .await - .expect("publishing a TX to a tributary we aren't in"); - tx.sign(&mut OsRng, genesis, &key, nonce); - - publish_transaction(&tributary, tx).await; + publish_signed_transaction(&mut raw_db, tributary, tx).await; } } }; @@ -877,22 +1114,21 @@ pub async fn run( raw_db, key.clone(), recognized_id, - p2p.clone(), processors.clone(), serai.clone(), - tributaries.clone(), + new_tributary_listener_2, )); } // Spawn the heartbeat task, which will trigger syncing if there hasn't been a Tributary block // in a while (presumably because we're behind) - tokio::spawn(heartbeat_tributaries(p2p.clone(), tributaries.clone())); + tokio::spawn(heartbeat_tributaries(p2p.clone(), new_tributary_listener_3)); // Handle P2P messages - tokio::spawn(handle_p2p(Ristretto::generator() * key.deref(), p2p, tributaries.clone())); + tokio::spawn(handle_p2p(Ristretto::generator() * key.deref(), p2p, new_tributary_listener_4)); // Handle all messages from processors - handle_processors(raw_db, key, serai, processors, tributaries).await; + handle_processors(raw_db, key, serai, processors, new_tributary_listener_5).await; } #[tokio::main] diff --git a/coordinator/src/p2p.rs b/coordinator/src/p2p.rs index 8d8cf68d1..bc252d506 100644 --- a/coordinator/src/p2p.rs +++ b/coordinator/src/p2p.rs @@ -149,10 +149,6 @@ struct Behavior { mdns: libp2p::mdns::tokio::Behaviour, } -lazy_static::lazy_static! { - static ref TIME_OF_LAST_P2P_MESSAGE: Mutex = Mutex::new(Instant::now()); -} - #[allow(clippy::type_complexity)] #[derive(Clone)] pub struct LibP2p( @@ -246,10 +242,16 @@ impl LibP2p { let (receive_send, receive_recv) = mpsc::unbounded_channel(); tokio::spawn({ + let mut time_of_last_p2p_message = Instant::now(); + #[allow(clippy::needless_pass_by_ref_mut)] // False positive - async fn broadcast_raw(p2p: &mut Swarm, msg: Vec) { + async fn broadcast_raw( + p2p: &mut Swarm, + time_of_last_p2p_message: &mut Instant, + msg: Vec, + ) { // Update the time of last message - *TIME_OF_LAST_P2P_MESSAGE.lock().await = Instant::now(); + *time_of_last_p2p_message = Instant::now(); match p2p.behaviour_mut().gossipsub.publish(IdentTopic::new(LIBP2P_TOPIC), msg.clone()) { Err(PublishError::SigningError(e)) => panic!("signing error when broadcasting: {e}"), @@ -267,8 +269,7 @@ impl LibP2p { async move { // Run this task ad-infinitum loop { - let time_since_last = - Instant::now().duration_since(*TIME_OF_LAST_P2P_MESSAGE.lock().await); + let time_since_last = Instant::now().duration_since(time_of_last_p2p_message); tokio::select! { biased; @@ -276,6 +277,7 @@ impl LibP2p { msg = broadcast_recv.recv() => { broadcast_raw( &mut swarm, + &mut time_of_last_p2p_message, msg.expect("broadcast_recv closed. are we shutting down?") ).await; } @@ -324,7 +326,11 @@ impl LibP2p { // (where a finalized block only occurs due to network activity), meaning this won't be // run _ = tokio::time::sleep(Duration::from_secs(80).saturating_sub(time_since_last)) => { - broadcast_raw(&mut swarm, P2pMessageKind::KeepAlive.serialize()).await; + broadcast_raw( + &mut swarm, + &mut time_of_last_p2p_message, + P2pMessageKind::KeepAlive.serialize() + ).await; } } } diff --git a/coordinator/src/processors.rs b/coordinator/src/processors.rs index 3dc0f0a6b..c147fad12 100644 --- a/coordinator/src/processors.rs +++ b/coordinator/src/processors.rs @@ -14,27 +14,24 @@ pub struct Message { #[async_trait::async_trait] pub trait Processors: 'static + Send + Sync + Clone { - async fn send(&self, network: NetworkId, msg: CoordinatorMessage); - async fn recv(&mut self) -> Message; + async fn send(&self, network: NetworkId, msg: impl Send + Into); + async fn recv(&mut self, network: NetworkId) -> Message; async fn ack(&mut self, msg: Message); } #[async_trait::async_trait] impl Processors for Arc { - async fn send(&self, network: NetworkId, msg: CoordinatorMessage) { + async fn send(&self, network: NetworkId, msg: impl Send + Into) { + let msg: CoordinatorMessage = msg.into(); let metadata = Metadata { from: self.service, to: Service::Processor(network), intent: msg.intent() }; let msg = serde_json::to_string(&msg).unwrap(); self.queue(metadata, msg.into_bytes()).await; } - async fn recv(&mut self) -> Message { - // TODO: Use a proper expected next ID - let msg = self.next(0).await; - - let network = match msg.from { - Service::Processor(network) => network, - Service::Coordinator => panic!("coordinator received coordinator message"), - }; + async fn recv(&mut self, network: NetworkId) -> Message { + let msg = self.next(Service::Processor(network)).await; + assert_eq!(msg.from, Service::Processor(network)); + let id = msg.id; // Deserialize it into a ProcessorMessage @@ -44,6 +41,6 @@ impl Processors for Arc { return Message { id, network, msg }; } async fn ack(&mut self, msg: Message) { - MessageQueue::ack(self, msg.id).await + MessageQueue::ack(self, Service::Processor(msg.network), msg.id).await } } diff --git a/coordinator/src/substrate/db.rs b/coordinator/src/substrate/db.rs index 6d9d06d92..2f702bd05 100644 --- a/coordinator/src/substrate/db.rs +++ b/coordinator/src/substrate/db.rs @@ -1,5 +1,12 @@ +use scale::{Encode, Decode}; + pub use serai_db::*; +use serai_client::{ + primitives::NetworkId, + validator_sets::primitives::{Session, KeyPair}, +}; + #[derive(Debug)] pub struct SubstrateDb(pub D); impl SubstrateDb { @@ -33,4 +40,41 @@ impl SubstrateDb { assert!(!Self::handled_event(txn, id, index)); txn.put(Self::event_key(&id, index), []); } + + fn session_key(key: &[u8]) -> Vec { + Self::substrate_key(b"session", key) + } + pub fn session_for_key(getter: &G, key: &[u8]) -> Option { + getter.get(Self::session_key(key)).map(|bytes| Session::decode(&mut bytes.as_ref()).unwrap()) + } + pub fn save_session_for_keys(txn: &mut D::Transaction<'_>, key_pair: &KeyPair, session: Session) { + let session = session.encode(); + let key_0 = Self::session_key(&key_pair.0); + let existing = txn.get(&key_0); + // This may trigger if 100% of a DKG are malicious, and they create a key equivalent to a prior + // key. Since it requires 100% maliciousness, not just 67% maliciousness, this will only assert + // in a modified-to-be-malicious stack, making it safe + assert!(existing.is_none() || (existing.as_ref() == Some(&session))); + txn.put(key_0, session.clone()); + txn.put(Self::session_key(&key_pair.1), session); + } + + fn batch_instructions_key(network: NetworkId, id: u32) -> Vec { + Self::substrate_key(b"batch", (network, id).encode()) + } + pub fn batch_instructions_hash( + getter: &G, + network: NetworkId, + id: u32, + ) -> Option<[u8; 32]> { + getter.get(Self::batch_instructions_key(network, id)).map(|bytes| bytes.try_into().unwrap()) + } + pub fn save_batch_instructions_hash( + txn: &mut D::Transaction<'_>, + network: NetworkId, + id: u32, + hash: [u8; 32], + ) { + txn.put(Self::batch_instructions_key(network, id), hash); + } } diff --git a/coordinator/src/substrate/mod.rs b/coordinator/src/substrate/mod.rs index 117b03ba9..0419bb085 100644 --- a/coordinator/src/substrate/mod.rs +++ b/coordinator/src/substrate/mod.rs @@ -1,4 +1,4 @@ -use core::{ops::Deref, time::Duration, future::Future}; +use core::{ops::Deref, time::Duration}; use std::collections::{HashSet, HashMap}; use zeroize::Zeroizing; @@ -9,7 +9,7 @@ use serai_client::{ SeraiError, Block, Serai, primitives::{BlockHash, NetworkId}, validator_sets::{ - primitives::{Session, ValidatorSet, KeyPair}, + primitives::{ValidatorSet, KeyPair}, ValidatorSetsEvent, }, in_instructions::InInstructionsEvent, @@ -18,7 +18,7 @@ use serai_client::{ use serai_db::DbTxn; -use processor_messages::{SubstrateContext, CoordinatorMessage}; +use processor_messages::SubstrateContext; use tokio::time::sleep; @@ -43,16 +43,10 @@ async fn in_set( Ok(Some(data.participants.iter().any(|(participant, _)| participant.0 == key))) } -async fn handle_new_set< - D: Db, - Fut: Future, - CNT: Clone + Fn(&mut D, TributarySpec) -> Fut, - Pro: Processors, ->( +async fn handle_new_set( db: &mut D, key: &Zeroizing<::F>, create_new_tributary: CNT, - processors: &Pro, serai: &Serai, block: &Block, set: ValidatorSet, @@ -84,7 +78,7 @@ async fn handle_new_set< let time = time + SUBSTRATE_TO_TRIBUTARY_TIME_DELAY; let spec = TributarySpec::new(block.hash(), time, set, set_data); - create_new_tributary(db, spec.clone()).await; + create_new_tributary(db, spec.clone()); } else { log::info!("not present in set {:?}", set); } @@ -92,41 +86,43 @@ async fn handle_new_set< Ok(()) } -async fn handle_key_gen( - key: &Zeroizing<::F>, +async fn handle_key_gen( + db: &mut D, processors: &Pro, serai: &Serai, block: &Block, set: ValidatorSet, key_pair: KeyPair, ) -> Result<(), SeraiError> { - if in_set(key, serai, set).await?.expect("KeyGen occurred for a set which doesn't exist") { - processors - .send( - set.network, - CoordinatorMessage::Substrate( - processor_messages::substrate::CoordinatorMessage::ConfirmKeyPair { - context: SubstrateContext { - serai_time: block.time().unwrap() / 1000, - network_latest_finalized_block: serai - .get_latest_block_for_network(block.hash(), set.network) - .await? - // The processor treats this as a magic value which will cause it to find a network - // block which has a time greater than or equal to the Serai time - .unwrap_or(BlockHash([0; 32])), - }, - set, - key_pair, - }, - ), - ) - .await; - } + // This has to be saved *before* we send ConfirmKeyPair + let mut txn = db.txn(); + SubstrateDb::::save_session_for_keys(&mut txn, &key_pair, set.session); + txn.commit(); + + processors + .send( + set.network, + processor_messages::substrate::CoordinatorMessage::ConfirmKeyPair { + context: SubstrateContext { + serai_time: block.time().unwrap() / 1000, + network_latest_finalized_block: serai + .get_latest_block_for_network(block.hash(), set.network) + .await? + // The processor treats this as a magic value which will cause it to find a network + // block which has a time greater than or equal to the Serai time + .unwrap_or(BlockHash([0; 32])), + }, + set, + key_pair, + }, + ) + .await; Ok(()) } -async fn handle_batch_and_burns( +async fn handle_batch_and_burns( + db: &mut D, processors: &Pro, serai: &Serai, block: &Block, @@ -152,16 +148,17 @@ async fn handle_batch_and_burns( let mut burns = HashMap::new(); for batch in serai.get_batch_events(hash).await? { - if let InInstructionsEvent::Batch { network, id, block: network_block } = batch { + if let InInstructionsEvent::Batch { network, id, block: network_block, instructions_hash } = + batch + { network_had_event(&mut burns, &mut batches, network); - // Track what Serai acknowledges as the latest block for this network - // If this Substrate block has multiple batches, the last batch's block will overwrite the - // prior batches - // Since batches within a block are guaranteed to be ordered, thanks to their incremental ID, - // the last batch will be the latest batch, so its block will be the latest block - // This is just a mild optimization to prevent needing an additional RPC call to grab this - batch_block.insert(network, network_block); + let mut txn = db.txn(); + SubstrateDb::::save_batch_instructions_hash(&mut txn, network, id, instructions_hash); + txn.commit(); + + // Make sure this is the only Batch event for this network in this Block + assert!(batch_block.insert(network, network_block).is_none()); // Add the batch included by this block batches.get_mut(&network).unwrap().push(id); @@ -198,23 +195,16 @@ async fn handle_batch_and_burns( processors .send( network, - CoordinatorMessage::Substrate( - processor_messages::substrate::CoordinatorMessage::SubstrateBlock { - context: SubstrateContext { - serai_time: block.time().unwrap() / 1000, - network_latest_finalized_block, - }, - network, - block: block.number(), - key: serai - .get_keys(ValidatorSet { network, session: Session(0) }) // TODO2 - .await? - .map(|keys| keys.1.into_inner()) - .expect("batch/burn for network which never set keys"), - burns: burns.remove(&network).unwrap(), - batches: batches.remove(&network).unwrap(), + processor_messages::substrate::CoordinatorMessage::SubstrateBlock { + context: SubstrateContext { + serai_time: block.time().unwrap() / 1000, + network_latest_finalized_block, }, - ), + network, + block: block.number(), + burns: burns.remove(&network).unwrap(), + batches: batches.remove(&network).unwrap(), + }, ) .await; } @@ -225,12 +215,7 @@ async fn handle_batch_and_burns( // Handle a specific Substrate block, returning an error when it fails to get data // (not blocking / holding) #[allow(clippy::needless_pass_by_ref_mut)] // False positive? -async fn handle_block< - D: Db, - Fut: Future, - CNT: Clone + Fn(&mut D, TributarySpec) -> Fut, - Pro: Processors, ->( +async fn handle_block( db: &mut SubstrateDb, key: &Zeroizing<::F>, create_new_tributary: CNT, @@ -259,8 +244,7 @@ async fn handle_block< if !SubstrateDb::::handled_event(&db.0, hash, event_id) { log::info!("found fresh new set event {:?}", new_set); - handle_new_set(&mut db.0, key, create_new_tributary.clone(), processors, serai, &block, set) - .await?; + handle_new_set(&mut db.0, key, create_new_tributary.clone(), serai, &block, set).await?; let mut txn = db.0.txn(); SubstrateDb::::handle_event(&mut txn, hash, event_id); txn.commit(); @@ -279,7 +263,7 @@ async fn handle_block< TributaryDb::::set_key_pair(&mut txn, set, &key_pair); txn.commit(); - handle_key_gen(key, processors, serai, &block, set, key_pair).await?; + handle_key_gen(&mut db.0, processors, serai, &block, set, key_pair).await?; } else { panic!("KeyGen event wasn't KeyGen: {key_gen:?}"); } @@ -296,7 +280,7 @@ async fn handle_block< // This does break the uniqueness of (hash, event_id) -> one event, yet // (network, (hash, event_id)) remains valid as a unique ID for an event if !SubstrateDb::::handled_event(&db.0, hash, event_id) { - handle_batch_and_burns(processors, serai, &block).await?; + handle_batch_and_burns(&mut db.0, processors, serai, &block).await?; } let mut txn = db.0.txn(); SubstrateDb::::handle_event(&mut txn, hash, event_id); @@ -305,12 +289,7 @@ async fn handle_block< Ok(()) } -pub async fn handle_new_blocks< - D: Db, - Fut: Future, - CNT: Clone + Fn(&mut D, TributarySpec) -> Fut, - Pro: Processors, ->( +pub async fn handle_new_blocks( db: &mut SubstrateDb, key: &Zeroizing<::F>, create_new_tributary: CNT, diff --git a/coordinator/src/tests/mod.rs b/coordinator/src/tests/mod.rs index 748ac4a67..6aaa907aa 100644 --- a/coordinator/src/tests/mod.rs +++ b/coordinator/src/tests/mod.rs @@ -30,12 +30,12 @@ impl MemProcessors { #[async_trait::async_trait] impl Processors for MemProcessors { - async fn send(&self, network: NetworkId, msg: CoordinatorMessage) { + async fn send(&self, network: NetworkId, msg: impl Send + Into) { let mut processors = self.0.write().await; let processor = processors.entry(network).or_insert_with(VecDeque::new); - processor.push_back(msg); + processor.push_back(msg.into()); } - async fn recv(&mut self) -> Message { + async fn recv(&mut self, _: NetworkId) -> Message { todo!() } async fn ack(&mut self, _: Message) { diff --git a/coordinator/src/tests/tributary/dkg.rs b/coordinator/src/tests/tributary/dkg.rs index 1da472fa8..aa8ea7c94 100644 --- a/coordinator/src/tests/tributary/dkg.rs +++ b/coordinator/src/tests/tributary/dkg.rs @@ -86,7 +86,7 @@ async fn dkg_test() { handle_new_blocks::<_, _, _, _, _, _, LocalP2p>( &mut scanner_db, key, - |_, _, _, _| async { + |_, _, _, _, _| async { panic!("provided TX caused recognized_id to be called in new_processors") }, &processors, @@ -112,7 +112,7 @@ async fn dkg_test() { handle_new_blocks::<_, _, _, _, _, _, LocalP2p>( &mut scanner_db, &keys[0], - |_, _, _, _| async { + |_, _, _, _, _| async { panic!("provided TX caused recognized_id to be called after Commitments") }, &processors, @@ -191,7 +191,7 @@ async fn dkg_test() { handle_new_blocks::<_, _, _, _, _, _, LocalP2p>( &mut scanner_db, &keys[0], - |_, _, _, _| async { + |_, _, _, _, _| async { panic!("provided TX caused recognized_id to be called after some shares") }, &processors, @@ -239,7 +239,7 @@ async fn dkg_test() { handle_new_blocks::<_, _, _, _, _, _, LocalP2p>( &mut scanner_db, &keys[0], - |_, _, _, _| async { panic!("provided TX caused recognized_id to be called after shares") }, + |_, _, _, _, _| async { panic!("provided TX caused recognized_id to be called after shares") }, &processors, |_, _| async { panic!("test tried to publish a new Serai TX") }, &spec, @@ -281,7 +281,7 @@ async fn dkg_test() { let key_pair = (serai_client::Public(substrate_key), network_key.try_into().unwrap()); let mut txs = vec![]; - for (k, key) in keys.iter().enumerate() { + for key in keys.iter() { let attempt = 0; // This is fine to re-use the one DB as such, due to exactly how this specific call is coded, // albeit poor @@ -306,7 +306,7 @@ async fn dkg_test() { handle_new_blocks::<_, _, _, _, _, _, LocalP2p>( &mut scanner_db, &keys[0], - |_, _, _, _| async { + |_, _, _, _, _| async { panic!("provided TX caused recognized_id to be called after DKG confirmation") }, &processors, diff --git a/coordinator/src/tests/tributary/handle_p2p.rs b/coordinator/src/tests/tributary/handle_p2p.rs index becf5059f..87576dd8f 100644 --- a/coordinator/src/tests/tributary/handle_p2p.rs +++ b/coordinator/src/tests/tributary/handle_p2p.rs @@ -1,11 +1,11 @@ use core::time::Duration; -use std::{sync::Arc, collections::HashMap}; +use std::sync::Arc; use rand_core::OsRng; use ciphersuite::{Ciphersuite, Ristretto}; -use tokio::{sync::RwLock, time::sleep}; +use tokio::{sync::broadcast, time::sleep}; use serai_db::MemDb; @@ -27,18 +27,18 @@ async fn handle_p2p_test() { let mut tributaries = new_tributaries(&keys, &spec).await; + let mut tributary_senders = vec![]; let mut tributary_arcs = vec![]; for (i, (p2p, tributary)) in tributaries.drain(..).enumerate() { - let tributary = Arc::new(RwLock::new(tributary)); + let tributary = Arc::new(tributary); tributary_arcs.push(tributary.clone()); - tokio::spawn(handle_p2p( - Ristretto::generator() * *keys[i], - p2p, - Arc::new(RwLock::new(HashMap::from([( - spec.genesis(), - ActiveTributary { spec: spec.clone(), tributary }, - )]))), - )); + let (new_tributary_send, new_tributary_recv) = broadcast::channel(5); + tokio::spawn(handle_p2p(Ristretto::generator() * *keys[i], p2p, new_tributary_recv)); + new_tributary_send + .send(ActiveTributary { spec: spec.clone(), tributary }) + .map_err(|_| "failed to send ActiveTributary") + .unwrap(); + tributary_senders.push(new_tributary_send); } let tributaries = tributary_arcs; @@ -46,22 +46,22 @@ async fn handle_p2p_test() { // We don't wait one block of time as we may have missed the chance for this block sleep(Duration::from_secs((2 * Tributary::::block_time()).into())) .await; - let tip = tributaries[0].read().await.tip().await; + let tip = tributaries[0].tip().await; assert!(tip != spec.genesis()); // Sleep one second to make sure this block propagates sleep(Duration::from_secs(1)).await; // Make sure every tributary has it for tributary in &tributaries { - assert!(tributary.read().await.reader().block(&tip).is_some()); + assert!(tributary.reader().block(&tip).is_some()); } // Then after another block of time, we should have yet another new block sleep(Duration::from_secs(Tributary::::block_time().into())).await; - let new_tip = tributaries[0].read().await.tip().await; + let new_tip = tributaries[0].tip().await; assert!(new_tip != tip); sleep(Duration::from_secs(1)).await; for tributary in tributaries { - assert!(tributary.read().await.reader().block(&new_tip).is_some()); + assert!(tributary.reader().block(&new_tip).is_some()); } } diff --git a/coordinator/src/tests/tributary/mod.rs b/coordinator/src/tests/tributary/mod.rs index fd198b8ed..be4a348b9 100644 --- a/coordinator/src/tests/tributary/mod.rs +++ b/coordinator/src/tests/tributary/mod.rs @@ -66,7 +66,7 @@ fn serialize_transaction() { // Create a valid vec of shares let mut shares = vec![]; // Create up to 512 participants - for i in 0 .. (OsRng.next_u64() % 512) { + for _ in 0 .. (OsRng.next_u64() % 512) { let mut share = vec![0; share_len]; OsRng.fill_bytes(&mut share); shares.push(share); diff --git a/coordinator/src/tests/tributary/sync.rs b/coordinator/src/tests/tributary/sync.rs index ced97bd6b..3dfc3757e 100644 --- a/coordinator/src/tests/tributary/sync.rs +++ b/coordinator/src/tests/tributary/sync.rs @@ -1,14 +1,11 @@ use core::time::Duration; -use std::{ - sync::Arc, - collections::{HashSet, HashMap}, -}; +use std::{sync::Arc, collections::HashSet}; use rand_core::OsRng; use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto}; -use tokio::{sync::RwLock, time::sleep}; +use tokio::{sync::broadcast, time::sleep}; use serai_db::MemDb; @@ -37,19 +34,20 @@ async fn sync_test() { let (syncer_p2p, syncer_tributary) = tributaries.pop().unwrap(); // Have the rest form a P2P net + let mut tributary_senders = vec![]; let mut tributary_arcs = vec![]; let mut p2p_threads = vec![]; for (i, (p2p, tributary)) in tributaries.drain(..).enumerate() { - let tributary = Arc::new(RwLock::new(tributary)); + let tributary = Arc::new(tributary); tributary_arcs.push(tributary.clone()); - let thread = tokio::spawn(handle_p2p( - Ristretto::generator() * *keys[i], - p2p, - Arc::new(RwLock::new(HashMap::from([( - spec.genesis(), - ActiveTributary { spec: spec.clone(), tributary }, - )]))), - )); + let (new_tributary_send, new_tributary_recv) = broadcast::channel(5); + let thread = + tokio::spawn(handle_p2p(Ristretto::generator() * *keys[i], p2p, new_tributary_recv)); + new_tributary_send + .send(ActiveTributary { spec: spec.clone(), tributary }) + .map_err(|_| "failed to send ActiveTributary") + .unwrap(); + tributary_senders.push(new_tributary_send); p2p_threads.push(thread); } let tributaries = tributary_arcs; @@ -60,14 +58,14 @@ async fn sync_test() { // propose by our 'offline' validator let block_time = u64::from(Tributary::::block_time()); sleep(Duration::from_secs(3 * block_time)).await; - let tip = tributaries[0].read().await.tip().await; + let tip = tributaries[0].tip().await; assert!(tip != spec.genesis()); // Sleep one second to make sure this block propagates sleep(Duration::from_secs(1)).await; // Make sure every tributary has it for tributary in &tributaries { - assert!(tributary.read().await.reader().block(&tip).is_some()); + assert!(tributary.reader().block(&tip).is_some()); } // Now that we've confirmed the other tributaries formed a net without issue, drop the syncer's @@ -76,31 +74,39 @@ async fn sync_test() { // Have it join the net let syncer_key = Ristretto::generator() * *syncer_key; - let syncer_tributary = Arc::new(RwLock::new(syncer_tributary)); - let syncer_tributaries = Arc::new(RwLock::new(HashMap::from([( - spec.genesis(), - ActiveTributary { spec: spec.clone(), tributary: syncer_tributary.clone() }, - )]))); - tokio::spawn(handle_p2p(syncer_key, syncer_p2p.clone(), syncer_tributaries.clone())); + let syncer_tributary = Arc::new(syncer_tributary); + let (syncer_tributary_send, syncer_tributary_recv) = broadcast::channel(5); + tokio::spawn(handle_p2p(syncer_key, syncer_p2p.clone(), syncer_tributary_recv)); + syncer_tributary_send + .send(ActiveTributary { spec: spec.clone(), tributary: syncer_tributary.clone() }) + .map_err(|_| "failed to send ActiveTributary to syncer") + .unwrap(); // It shouldn't automatically catch up. If it somehow was, our test would be broken // Sanity check this - let tip = tributaries[0].read().await.tip().await; - sleep(Duration::from_secs(2 * block_time)).await; - assert!(tributaries[0].read().await.tip().await != tip); - assert_eq!(syncer_tributary.read().await.tip().await, spec.genesis()); + let tip = tributaries[0].tip().await; + // Wait until a new block occurs + sleep(Duration::from_secs(3 * block_time)).await; + // Make sure a new block actually occurred + assert!(tributaries[0].tip().await != tip); + // Make sure the new block alone didn't trigger catching up + assert_eq!(syncer_tributary.tip().await, spec.genesis()); // Start the heartbeat protocol - tokio::spawn(heartbeat_tributaries(syncer_p2p, syncer_tributaries)); + let (syncer_heartbeat_tributary_send, syncer_heartbeat_tributary_recv) = broadcast::channel(5); + tokio::spawn(heartbeat_tributaries(syncer_p2p, syncer_heartbeat_tributary_recv)); + syncer_heartbeat_tributary_send + .send(ActiveTributary { spec: spec.clone(), tributary: syncer_tributary.clone() }) + .map_err(|_| "failed to send ActiveTributary to heartbeat") + .unwrap(); // The heartbeat is once every 10 blocks sleep(Duration::from_secs(10 * block_time)).await; - assert!(syncer_tributary.read().await.tip().await != spec.genesis()); + assert!(syncer_tributary.tip().await != spec.genesis()); // Verify it synced to the tip let syncer_tip = { - let tributary = tributaries[0].write().await; - let syncer_tributary = syncer_tributary.write().await; + let tributary = &tributaries[0]; let tip = tributary.tip().await; let syncer_tip = syncer_tributary.tip().await; @@ -114,7 +120,7 @@ async fn sync_test() { sleep(Duration::from_secs(block_time)).await; // Verify it's now keeping up - assert!(syncer_tributary.read().await.tip().await != syncer_tip); + assert!(syncer_tributary.tip().await != syncer_tip); // Verify it's now participating in consensus // Because only `t` validators are used in a commit, take n - t nodes offline @@ -128,7 +134,6 @@ async fn sync_test() { // wait for a block sleep(Duration::from_secs(block_time)).await; - let syncer_tributary = syncer_tributary.read().await; if syncer_tributary .reader() .parsed_commit(&syncer_tributary.tip().await) diff --git a/coordinator/src/tributary/db.rs b/coordinator/src/tributary/db.rs index acb6e842c..9a89139b6 100644 --- a/coordinator/src/tributary/db.rs +++ b/coordinator/src/tributary/db.rs @@ -17,11 +17,17 @@ pub enum Topic { impl Topic { fn as_key(&self, genesis: [u8; 32]) -> Vec { - match self { - Topic::Dkg => [genesis.as_slice(), b"dkg".as_ref()].concat(), - Topic::Batch(id) => [genesis.as_slice(), b"batch".as_ref(), id.as_ref()].concat(), - Topic::Sign(id) => [genesis.as_slice(), b"sign".as_ref(), id.as_ref()].concat(), - } + let mut res = genesis.to_vec(); + let (label, id) = match self { + Topic::Dkg => (b"dkg".as_slice(), [].as_slice()), + Topic::Batch(id) => (b"batch".as_slice(), id.as_slice()), + Topic::Sign(id) => (b"sign".as_slice(), id.as_slice()), + }; + res.push(u8::try_from(label.len()).unwrap()); + res.extend(label); + res.push(u8::try_from(id.len()).unwrap()); + res.extend(id); + res } } @@ -35,13 +41,12 @@ pub struct DataSpecification { impl DataSpecification { fn as_key(&self, genesis: [u8; 32]) -> Vec { - // TODO: Use a proper transcript here to avoid conflicts? - [ - self.topic.as_key(genesis).as_ref(), - self.label.as_bytes(), - self.attempt.to_le_bytes().as_ref(), - ] - .concat() + let mut res = self.topic.as_key(genesis); + let label_bytes = self.label.bytes(); + res.push(u8::try_from(label_bytes.len()).unwrap()); + res.extend(label_bytes); + res.extend(self.attempt.to_le_bytes()); + res } } diff --git a/coordinator/src/tributary/handle.rs b/coordinator/src/tributary/handle.rs index cfca4500d..1ec83a97d 100644 --- a/coordinator/src/tributary/handle.rs +++ b/coordinator/src/tributary/handle.rs @@ -17,7 +17,6 @@ use frost_schnorrkel::Schnorrkel; use serai_client::{ Signature, - primitives::NetworkId, validator_sets::primitives::{ValidatorSet, KeyPair, musig_context, set_keys_message}, subxt::utils::Encoded, Serai, @@ -26,8 +25,8 @@ use serai_client::{ use tributary::Signed; use processor_messages::{ - CoordinatorMessage, coordinator, key_gen::{self, KeyGenId}, + coordinator, sign::{self, SignId}, }; @@ -36,13 +35,24 @@ use serai_db::{Get, Db}; use crate::{ processors::Processors, tributary::{ - Transaction, TributarySpec, Topic, DataSpecification, TributaryDb, scanner::RecognizedIdType, + Transaction, TributarySpec, Topic, DataSpecification, TributaryDb, nonce_decider::NonceDecider, + scanner::RecognizedIdType, }, }; +const DKG_COMMITMENTS: &str = "commitments"; +const DKG_SHARES: &str = "shares"; const DKG_CONFIRMATION_NONCES: &str = "confirmation_nonces"; const DKG_CONFIRMATION_SHARES: &str = "confirmation_shares"; +// These s/b prefixes between Batch and Sign should be unnecessary, as Batch/Share entries +// themselves should already be domain separated +const BATCH_PREPROCESS: &str = "b_preprocess"; +const BATCH_SHARE: &str = "b_share"; + +const SIGN_PREPROCESS: &str = "s_preprocess"; +const SIGN_SHARE: &str = "s_share"; + // Instead of maintaing state, this simply re-creates the machine(s) in-full on every call (which // should only be once per tributary). // This simplifies data flow and prevents requiring multiple paths. @@ -224,13 +234,13 @@ pub fn generated_key_pair( DkgConfirmer::share(spec, key, attempt, preprocesses, key_pair) } -pub async fn handle_application_tx< +pub(crate) async fn handle_application_tx< D: Db, Pro: Processors, FPst: Future, PST: Clone + Fn(ValidatorSet, Encoded) -> FPst, FRid: Future, - RID: Clone + Fn(NetworkId, [u8; 32], RecognizedIdType, [u8; 32]) -> FRid, + RID: crate::RIDTrait, >( tx: Transaction, spec: &TributarySpec, @@ -290,7 +300,7 @@ pub async fn handle_application_tx< Transaction::DkgCommitments(attempt, bytes, signed) => { match handle( txn, - &DataSpecification { topic: Topic::Dkg, label: "commitments", attempt }, + &DataSpecification { topic: Topic::Dkg, label: DKG_COMMITMENTS, attempt }, bytes, &signed, ) { @@ -299,10 +309,10 @@ pub async fn handle_application_tx< processors .send( spec.set().network, - CoordinatorMessage::KeyGen(key_gen::CoordinatorMessage::Commitments { + key_gen::CoordinatorMessage::Commitments { id: KeyGenId { set: spec.set(), attempt }, commitments, - }), + }, ) .await; } @@ -345,7 +355,7 @@ pub async fn handle_application_tx< ); match handle( txn, - &DataSpecification { topic: Topic::Dkg, label: "shares", attempt }, + &DataSpecification { topic: Topic::Dkg, label: DKG_SHARES, attempt }, bytes, &signed, ) { @@ -355,10 +365,10 @@ pub async fn handle_application_tx< processors .send( spec.set().network, - CoordinatorMessage::KeyGen(key_gen::CoordinatorMessage::Shares { + key_gen::CoordinatorMessage::Shares { id: KeyGenId { set: spec.set(), attempt }, shares, - }), + }, ) .await; } @@ -414,7 +424,8 @@ pub async fn handle_application_tx< Transaction::Batch(_, batch) => { // Because this Batch has achieved synchrony, its batch ID should be authorized TributaryDb::::recognize_topic(txn, genesis, Topic::Batch(batch)); - recognized_id(spec.set().network, genesis, RecognizedIdType::Batch, batch).await; + let nonce = NonceDecider::::handle_batch(txn, genesis, batch); + recognized_id(spec.set().network, genesis, RecognizedIdType::Batch, batch, nonce).await; } Transaction::SubstrateBlock(block) => { @@ -423,9 +434,10 @@ pub async fn handle_application_tx< despite us not providing that transaction", ); - for id in plan_ids { + let nonces = NonceDecider::::handle_substrate_block(txn, genesis, &plan_ids); + for (nonce, id) in nonces.into_iter().zip(plan_ids.into_iter()) { TributaryDb::::recognize_topic(txn, genesis, Topic::Sign(id)); - recognized_id(spec.set().network, genesis, RecognizedIdType::Plan, id).await; + recognized_id(spec.set().network, genesis, RecognizedIdType::Plan, id, nonce).await; } } @@ -434,20 +446,22 @@ pub async fn handle_application_tx< txn, &DataSpecification { topic: Topic::Batch(data.plan), - label: "preprocess", + label: BATCH_PREPROCESS, attempt: data.attempt, }, data.data, &data.signed, ) { Some(Some(preprocesses)) => { + NonceDecider::::selected_for_signing_batch(txn, genesis, data.plan); + let key = TributaryDb::::key_pair(txn, spec.set()).unwrap().0 .0.to_vec(); processors .send( spec.set().network, - CoordinatorMessage::Coordinator(coordinator::CoordinatorMessage::BatchPreprocesses { - id: SignId { key: vec![], id: data.plan, attempt: data.attempt }, + coordinator::CoordinatorMessage::BatchPreprocesses { + id: SignId { key, id: data.plan, attempt: data.attempt }, preprocesses, - }), + }, ) .await; } @@ -460,23 +474,24 @@ pub async fn handle_application_tx< txn, &DataSpecification { topic: Topic::Batch(data.plan), - label: "share", + label: BATCH_SHARE, attempt: data.attempt, }, data.data, &data.signed, ) { Some(Some(shares)) => { + let key = TributaryDb::::key_pair(txn, spec.set()).unwrap().0 .0.to_vec(); processors .send( spec.set().network, - CoordinatorMessage::Coordinator(coordinator::CoordinatorMessage::BatchShares { - id: SignId { key: vec![], id: data.plan, attempt: data.attempt }, + coordinator::CoordinatorMessage::BatchShares { + id: SignId { key, id: data.plan, attempt: data.attempt }, shares: shares .into_iter() .map(|(validator, share)| (validator, share.try_into().unwrap())) .collect(), - }), + }, ) .await; } @@ -491,17 +506,18 @@ pub async fn handle_application_tx< txn, &DataSpecification { topic: Topic::Sign(data.plan), - label: "preprocess", + label: SIGN_PREPROCESS, attempt: data.attempt, }, data.data, &data.signed, ) { Some(Some(preprocesses)) => { + NonceDecider::::selected_for_signing_plan(txn, genesis, data.plan); processors .send( spec.set().network, - CoordinatorMessage::Sign(sign::CoordinatorMessage::Preprocesses { + sign::CoordinatorMessage::Preprocesses { id: SignId { key: key_pair .expect("completed SignPreprocess despite not setting the key pair") @@ -511,7 +527,7 @@ pub async fn handle_application_tx< attempt: data.attempt, }, preprocesses, - }), + }, ) .await; } @@ -523,7 +539,11 @@ pub async fn handle_application_tx< let key_pair = TributaryDb::::key_pair(txn, spec.set()); match handle( txn, - &DataSpecification { topic: Topic::Sign(data.plan), label: "share", attempt: data.attempt }, + &DataSpecification { + topic: Topic::Sign(data.plan), + label: SIGN_SHARE, + attempt: data.attempt, + }, data.data, &data.signed, ) { @@ -531,7 +551,7 @@ pub async fn handle_application_tx< processors .send( spec.set().network, - CoordinatorMessage::Sign(sign::CoordinatorMessage::Shares { + sign::CoordinatorMessage::Shares { id: SignId { key: key_pair .expect("completed SignShares despite not setting the key pair") @@ -541,7 +561,7 @@ pub async fn handle_application_tx< attempt: data.attempt, }, shares, - }), + }, ) .await; } @@ -561,11 +581,7 @@ pub async fn handle_application_tx< processors .send( spec.set().network, - CoordinatorMessage::Sign(sign::CoordinatorMessage::Completed { - key: key_pair.1.to_vec(), - id: plan, - tx: tx_hash, - }), + sign::CoordinatorMessage::Completed { key: key_pair.1.to_vec(), id: plan, tx: tx_hash }, ) .await; } diff --git a/coordinator/src/tributary/mod.rs b/coordinator/src/tributary/mod.rs index c97107db7..f151e18e1 100644 --- a/coordinator/src/tributary/mod.rs +++ b/coordinator/src/tributary/mod.rs @@ -30,6 +30,9 @@ use tributary::{ mod db; pub use db::*; +mod nonce_decider; +pub use nonce_decider::*; + mod handle; pub use handle::*; @@ -53,6 +56,7 @@ impl TributarySpec { let mut validators = vec![]; for (participant, amount) in set_data.participants { // TODO: Ban invalid keys from being validators on the Serai side + // (make coordinator key a session key?) let participant = ::read_G::<&[u8]>(&mut participant.0.as_ref()) .expect("invalid key registered as participant"); // Give one weight on Tributary per bond instance @@ -294,7 +298,7 @@ impl ReadWrite for Transaction { let share_len = usize::from(u16::from_le_bytes(share_len)); let mut shares = vec![]; - for i in 0 .. u16::from_le_bytes(share_quantity) { + for _ in 0 .. u16::from_le_bytes(share_quantity) { let mut share = vec![0; share_len]; reader.read_exact(&mut share)?; shares.push(share); @@ -487,7 +491,7 @@ impl TransactionTrait for Transaction { } } - if let Transaction::SignCompleted { plan, tx_hash, first_signer, signature } = self { + if let Transaction::SignCompleted { first_signer, signature, .. } = self { if !signature.verify(*first_signer, self.sign_completed_challenge()) { Err(TransactionError::InvalidContent)?; } diff --git a/coordinator/src/tributary/nonce_decider.rs b/coordinator/src/tributary/nonce_decider.rs new file mode 100644 index 000000000..eb95c5395 --- /dev/null +++ b/coordinator/src/tributary/nonce_decider.rs @@ -0,0 +1,127 @@ +use core::marker::PhantomData; + +use serai_db::{Get, DbTxn, Db}; + +use crate::tributary::Transaction; + +/// Decides the nonce which should be used for a transaction on a Tributary. +/// +/// Deterministically builds a list of nonces to use based on the on-chain events and expected +/// transactions in response. Enables rebooting/rebuilding validators with full safety. +pub struct NonceDecider(PhantomData); + +const BATCH_CODE: u8 = 0; +const BATCH_SIGNING_CODE: u8 = 1; +const PLAN_CODE: u8 = 2; +const PLAN_SIGNING_CODE: u8 = 3; + +impl NonceDecider { + fn next_nonce_key(genesis: [u8; 32]) -> Vec { + D::key(b"coordinator_tributary_nonce", b"next", genesis) + } + fn allocate_nonce(txn: &mut D::Transaction<'_>, genesis: [u8; 32]) -> u32 { + let key = Self::next_nonce_key(genesis); + let next = + txn.get(&key).map(|bytes| u32::from_le_bytes(bytes.try_into().unwrap())).unwrap_or(3); + txn.put(key, (next + 1).to_le_bytes()); + next + } + + fn item_nonce_key(genesis: [u8; 32], code: u8, id: [u8; 32]) -> Vec { + D::key( + b"coordinator_tributary_nonce", + b"item", + [genesis.as_slice(), [code].as_ref(), id.as_ref()].concat(), + ) + } + fn set_nonce( + txn: &mut D::Transaction<'_>, + genesis: [u8; 32], + code: u8, + id: [u8; 32], + nonce: u32, + ) { + txn.put(Self::item_nonce_key(genesis, code, id), nonce.to_le_bytes()) + } + fn db_nonce(getter: &G, genesis: [u8; 32], code: u8, id: [u8; 32]) -> Option { + getter + .get(Self::item_nonce_key(genesis, code, id)) + .map(|bytes| u32::from_le_bytes(bytes.try_into().unwrap())) + } + + pub fn handle_batch(txn: &mut D::Transaction<'_>, genesis: [u8; 32], batch: [u8; 32]) -> u32 { + let nonce_for = Self::allocate_nonce(txn, genesis); + Self::set_nonce(txn, genesis, BATCH_CODE, batch, nonce_for); + nonce_for + } + pub fn selected_for_signing_batch( + txn: &mut D::Transaction<'_>, + genesis: [u8; 32], + batch: [u8; 32], + ) { + let nonce_for = Self::allocate_nonce(txn, genesis); + Self::set_nonce(txn, genesis, BATCH_SIGNING_CODE, batch, nonce_for); + } + + pub fn handle_substrate_block( + txn: &mut D::Transaction<'_>, + genesis: [u8; 32], + plans: &[[u8; 32]], + ) -> Vec { + let mut res = Vec::with_capacity(plans.len()); + for plan in plans { + let nonce_for = Self::allocate_nonce(txn, genesis); + Self::set_nonce(txn, genesis, PLAN_CODE, *plan, nonce_for); + res.push(nonce_for); + } + res + } + pub fn selected_for_signing_plan( + txn: &mut D::Transaction<'_>, + genesis: [u8; 32], + plan: [u8; 32], + ) { + let nonce_for = Self::allocate_nonce(txn, genesis); + Self::set_nonce(txn, genesis, PLAN_SIGNING_CODE, plan, nonce_for); + } + + pub fn nonce(getter: &G, genesis: [u8; 32], tx: &Transaction) -> Option> { + match tx { + Transaction::DkgCommitments(attempt, _, _) => { + assert_eq!(*attempt, 0); + Some(Some(0)) + } + Transaction::DkgShares { attempt, .. } => { + assert_eq!(*attempt, 0); + Some(Some(1)) + } + Transaction::DkgConfirmed(attempt, _, _) => { + assert_eq!(*attempt, 0); + Some(Some(2)) + } + + Transaction::Batch(_, _) => None, + Transaction::SubstrateBlock(_) => None, + + Transaction::BatchPreprocess(data) => { + assert_eq!(data.attempt, 0); + Some(Self::db_nonce(getter, genesis, BATCH_CODE, data.plan)) + } + Transaction::BatchShare(data) => { + assert_eq!(data.attempt, 0); + Some(Self::db_nonce(getter, genesis, BATCH_SIGNING_CODE, data.plan)) + } + + Transaction::SignPreprocess(data) => { + assert_eq!(data.attempt, 0); + Some(Self::db_nonce(getter, genesis, PLAN_CODE, data.plan)) + } + Transaction::SignShare(data) => { + assert_eq!(data.attempt, 0); + Some(Self::db_nonce(getter, genesis, PLAN_SIGNING_CODE, data.plan)) + } + + Transaction::SignCompleted { .. } => None, + } + } +} diff --git a/coordinator/src/tributary/scanner.rs b/coordinator/src/tributary/scanner.rs index fe8a18d26..5d8f00168 100644 --- a/coordinator/src/tributary/scanner.rs +++ b/coordinator/src/tributary/scanner.rs @@ -4,9 +4,7 @@ use zeroize::Zeroizing; use ciphersuite::{Ciphersuite, Ristretto}; -use serai_client::{ - primitives::NetworkId, validator_sets::primitives::ValidatorSet, subxt::utils::Encoded, -}; +use serai_client::{validator_sets::primitives::ValidatorSet, subxt::utils::Encoded}; use tributary::{ Transaction as TributaryTransaction, Block, TributaryReader, @@ -40,7 +38,7 @@ async fn handle_block< FPst: Future, PST: Clone + Fn(ValidatorSet, Encoded) -> FPst, FRid: Future, - RID: Clone + Fn(NetworkId, [u8; 32], RecognizedIdType, [u8; 32]) -> FRid, + RID: crate::RIDTrait, P: P2p, >( db: &mut TributaryDb, @@ -101,13 +99,13 @@ async fn handle_block< // TODO2: Trigger any necessary re-attempts } -pub async fn handle_new_blocks< +pub(crate) async fn handle_new_blocks< D: Db, Pro: Processors, FPst: Future, PST: Clone + Fn(ValidatorSet, Encoded) -> FPst, FRid: Future, - RID: Clone + Fn(NetworkId, [u8; 32], RecognizedIdType, [u8; 32]) -> FRid, + RID: crate::RIDTrait, P: P2p, >( db: &mut TributaryDb, diff --git a/coordinator/tributary/src/blockchain.rs b/coordinator/tributary/src/blockchain.rs index 78c2ca2bf..d21928ec0 100644 --- a/coordinator/tributary/src/blockchain.rs +++ b/coordinator/tributary/src/blockchain.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::collections::{VecDeque, HashMap}; use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto}; @@ -13,7 +13,7 @@ use crate::{ transaction::{Signed, TransactionKind, Transaction as TransactionTrait}, }; -#[derive(Clone, PartialEq, Eq, Debug)] +#[derive(Debug)] pub(crate) struct Blockchain { db: Option, genesis: [u8; 32], @@ -24,11 +24,13 @@ pub(crate) struct Blockchain { provided: ProvidedTransactions, mempool: Mempool, + + pub(crate) next_block_notifications: VecDeque>, } impl Blockchain { - fn tip_key(&self) -> Vec { - D::key(b"tributary_blockchain", b"tip", self.genesis) + fn tip_key(genesis: [u8; 32]) -> Vec { + D::key(b"tributary_blockchain", b"tip", genesis) } fn block_number_key(&self) -> Vec { D::key(b"tributary_blockchain", b"block_number", self.genesis) @@ -76,11 +78,13 @@ impl Blockchain { provided: ProvidedTransactions::new(db.clone(), genesis), mempool: Mempool::new(db, genesis), + + next_block_notifications: VecDeque::new(), }; if let Some((block_number, tip)) = { let db = res.db.as_ref().unwrap(); - db.get(res.block_number_key()).map(|number| (number, db.get(res.tip_key()).unwrap())) + db.get(res.block_number_key()).map(|number| (number, db.get(Self::tip_key(genesis)).unwrap())) } { res.block_number = u32::from_le_bytes(block_number.try_into().unwrap()); res.tip.copy_from_slice(&tip); @@ -132,6 +136,10 @@ impl Blockchain { db.get(Self::block_after_key(&genesis, block)).map(|bytes| bytes.try_into().unwrap()) } + pub(crate) fn tip_from_db(db: &D, genesis: [u8; 32]) -> [u8; 32] { + db.get(Self::tip_key(genesis)).map(|bytes| bytes.try_into().unwrap()).unwrap_or(genesis) + } + pub(crate) fn add_transaction( &mut self, internal: bool, @@ -226,7 +234,7 @@ impl Blockchain { let mut txn = db.txn(); self.tip = block.hash(); - txn.put(self.tip_key(), self.tip); + txn.put(Self::tip_key(self.genesis), self.tip); self.block_number += 1; txn.put(self.block_number_key(), self.block_number.to_le_bytes()); @@ -270,6 +278,10 @@ impl Blockchain { txn.commit(); self.db = Some(db); + for tx in self.next_block_notifications.drain(..) { + let _ = tx.send(()); + } + Ok(()) } } diff --git a/coordinator/tributary/src/lib.rs b/coordinator/tributary/src/lib.rs index ebb7b165c..3c5227b0c 100644 --- a/coordinator/tributary/src/lib.rs +++ b/coordinator/tributary/src/lib.rs @@ -336,6 +336,15 @@ impl Tributary { _ => false, } } + + /// Get a Future which will resolve once the next block has been added. + pub async fn next_block_notification( + &self, + ) -> impl Send + Sync + core::future::Future> { + let (tx, rx) = tokio::sync::oneshot::channel(); + self.network.blockchain.write().await.next_block_notifications.push_back(tx); + rx + } } #[derive(Clone)] @@ -344,7 +353,8 @@ impl TributaryReader { pub fn genesis(&self) -> [u8; 32] { self.1 } - // Since these values are static, they can be safely read from the database without lock + + // Since these values are static once set, they can be safely read from the database without lock // acquisition pub fn block(&self, hash: &[u8; 32]) -> Option> { Blockchain::::block_from_db(&self.0, self.1, hash) @@ -363,4 +373,9 @@ impl TributaryReader { .commit(hash) .map(|commit| Commit::::decode(&mut commit.as_ref()).unwrap().end_time) } + + // This isn't static, yet can be read with only minor discrepancy risks + pub fn tip(&self) -> [u8; 32] { + Blockchain::::tip_from_db(&self.0, self.1) + } } diff --git a/coordinator/tributary/src/tests/blockchain.rs b/coordinator/tributary/src/tests/blockchain.rs index 0f58b5995..21051095c 100644 --- a/coordinator/tributary/src/tests/blockchain.rs +++ b/coordinator/tributary/src/tests/blockchain.rs @@ -104,7 +104,7 @@ fn invalid_block() { { // Add a valid transaction - let mut blockchain = blockchain.clone(); + let (_, mut blockchain) = new_blockchain(genesis, &[tx.1.signer]); assert!(blockchain.add_transaction::( true, Transaction::Application(tx.clone()), @@ -129,7 +129,7 @@ fn invalid_block() { { // Invalid signature - let mut blockchain = blockchain.clone(); + let (_, mut blockchain) = new_blockchain(genesis, &[tx.1.signer]); assert!(blockchain.add_transaction::( true, Transaction::Application(tx), diff --git a/deny.toml b/deny.toml index bade778c3..980f78c48 100644 --- a/deny.toml +++ b/deny.toml @@ -67,6 +67,8 @@ exceptions = [ { allow = ["AGPL-3.0"], name = "serai-client" }, + { allow = ["AGPL-3.0"], name = "mini-serai" }, + { allow = ["AGPL-3.0"], name = "serai-docker-tests" }, { allow = ["AGPL-3.0"], name = "serai-message-queue-tests" }, { allow = ["AGPL-3.0"], name = "serai-processor-tests" }, diff --git a/docs/Getting Started.md b/docs/Getting Started.md index f88dfeafc..287d12ee2 100644 --- a/docs/Getting Started.md +++ b/docs/Getting Started.md @@ -66,6 +66,7 @@ cargo build --release --all-features Running tests requires: +- [A rootless Docker setup](https://docs.docker.com/engine/security/rootless/) - A properly configured Bitcoin regtest node (available via Docker) - A properly configured Monero regtest node (available via Docker) - A properly configured monero-wallet-rpc instance diff --git a/docs/coordinator/Coordinator.md b/docs/coordinator/Coordinator.md index 10c16318a..93f352255 100644 --- a/docs/coordinator/Coordinator.md +++ b/docs/coordinator/Coordinator.md @@ -7,23 +7,28 @@ node. This document primarily details its flow with regards to the Serai node and processor. -## New Set Event +### New Set Event On `validator_sets::pallet::Event::NewSet`, the coordinator spawns a tributary for the new set. It additionally sends the processor `key_gen::CoordinatorMessage::GenerateKey`. -## Key Generation Event +### Key Generation Event On `validator_sets::pallet::Event::KeyGen`, the coordinator sends `substrate::CoordinatorMessage::ConfirmKeyPair` to the processor. -# Update +### Batch -On `key_gen::ProcessorMessage::Update`, the coordinator publishes an unsigned -transaction containing the signed batch to the Serai blockchain. +On `substrate::ProcessorMessage::Batch`, the coordinator notes what the on-chain +`Batch` should be, for verification once published. -# Sign Completed +### SignedBatch + +On `substrate::ProcessorMessage::SignedBatch`, the coordinator publishes an +unsigned transaction containing the signed batch to the Serai blockchain. + +### Sign Completed On `sign::ProcessorMessage::Completed`, the coordinator makes a tributary transaction containing the transaction hash the signing process was supposedly diff --git a/docs/policy/Canonical Chain.md b/docs/policy/Canonical Chain.md new file mode 100644 index 000000000..9eb1ae62e --- /dev/null +++ b/docs/policy/Canonical Chain.md @@ -0,0 +1,72 @@ +# Canonical Chain + +As Serai is a network connected to many external networks, at some point we will +likely have to ask ourselves what the canonical chain for a network is. This +document intends to establish soft, non-binding policy, in the hopes it'll guide +most discussions on the matter. + +The canonical chain is the chain Serai follows and honors transactions on. Serai +does not guarantee operations availability nor integrity on any chains other +than the canonical chain. Which chain is considered canonical is dependent on +several factors. + +### Finalization + +Serai finalizes blocks from external networks onto itself. Once a block is +finalized, it is considered irreversible. Accordingly, the primary tenet +regarding what chain Serai will honor is the chain Serai has finalized. We can +only assume the integrity of our coins on that chain. + +### Node Software + +Only node software which passes a quality threshold and actively identifies as +belonging to an external network's protocol should be run. Never should a +transformative node (a node trying to create a new network from an existing one) +be run in place of a node actually for the external network. Beyond active +identification, it must have community recognition as belonging. + +If the majority of a community actively identifying as the network stands behind +a hard fork, it should not be considered as a new network yet the next step of +the existing one. If a hard fork breaks Serai's integrity, it should not be +supported. + +Multiple independent nodes should be run in order to reduce the likelihood of +vulnerabilities to any specific node's faults. + +### Rollbacks + +Over time, various networks have rolled back in response to exploits. A rollback +should undergo the same scrutiny as a hard fork. If the rollback breaks Serai's +integrity, yet someone identifying as from the project offers to restore +integrity out-of-band, integrity is considered kept so long as the offer is +followed through on. + +Since a rollback would break Serai's finalization policy, a technical note on +how it could be implemented is provided. + +Assume a blockchain from `0 .. 100` exists, with `100a ..= 500a` being rolled +back blocks. The new chain extends from `99` with `100b ..= 200b`. Serai would +define the canonical chain as `0 .. 100`, `100a ..= 500a`, `100b ..= 200b`, with +`100b` building off `500a`. Serai would have to perform data-availability for +`100a ..= 500a` (such as via a JSON file in-tree), and would have to modify the +processor to edit its `Eventuality`s/UTXOs at `500a` back to the state at `99`. +Any `Burn`s handled after `99` should be handled once again, if the transactions +from `100a ..= 500a` cannot simply be carried over. + +### On Fault + +If the canonical chain does put Serai's coins into an invalid state, +irreversibly and without amends, then the discrepancy should be amortized to all +users as feasible, yet affected operations should otherwise halt if under +permanent duress. + +For example, if Serai lists a token which has a by-governance blacklist +function, and is blacklisted without appeal, Serai should destroy all associated +sriXYZ and cease operations. + +If a bug, either in the chain or in Serai's own code, causes a loss of 10% of +coins (without amends), operations should halt until all outputs in system can +have their virtual amount reduced by a total amount of the loss, +proportionalized to each output. Alternatively, Serai could decrease all token +balances by 10%. All liquidity/swap operations should be halted until users are +given proper time to withdraw, if they so choose, before operations resume. diff --git a/docs/processor/Multisig Rotation.md b/docs/processor/Multisig Rotation.md new file mode 100644 index 000000000..822eeccaf --- /dev/null +++ b/docs/processor/Multisig Rotation.md @@ -0,0 +1,176 @@ +# Multisig Rotation + +Substrate is expected to determine when a new validator set instance will be +created, and with it, a new multisig. Upon the successful creation of a new +multisig, as determined by the new multisig setting their key pair on Substrate, +rotation begins. + +### Timeline + +The following timeline is established: + +1) The new multisig is created, and has its keys set on Serai. Once the next + `Batch` with a new external network block is published, its block becomes the + "queue block". The new multisig is set to activate at the "queue block", plus + `CONFIRMATIONS` blocks (the "activation block"). + + We don't use the last `Batch`'s external network block, as that `Batch` may + be older than `CONFIRMATIONS` blocks. Any yet-to-be-included-and-finalized + `Batch` will be within `CONFIRMATIONS` blocks of what any processor has + scanned however, as it'll wait for inclusion and finalization before + continuing scanning. + +2) Once the "activation block" itself has been finalized on Serai, UIs should + start exclusively using the new multisig. If the "activation block" isn't + finalized within `2 * CONFIRMATIONS` blocks, UIs should stop making + transactions to any multisig on that network. + + Waiting for Serai's finalization prevents a UI from using an unfinalized + "activation block" before a re-organization to a shorter chain. If a + transaction to Serai was carried from the unfinalized "activation block" + to the shorter chain, it'd no longer be after the "activation block" and + accordingly would be ignored. + + We could not wait for Serai to finalize the block, yet instead wait for the + block to have `CONFIRMATIONS` confirmations. This would prevent needing to + wait for an indeterminate amount of time for Serai to finalize the + "activation block", with the knowledge it should be finalized. Doing so would + open UIs to eclipse attacks, where they live on an alternate chain where a + possible "activation block" is finalized, yet Serai finalizes a distinct + "activation block". If the alternate chain was longer than the finalized + chain, the above issue would be reopened. + + The reason for UIs stopping under abnormal behavior is as follows. Given a + sufficiently delayed `Batch` for the "activation block", UIs will use the old + multisig past the point it will be deprecated. Accordingly, UIs must realize + when `Batch`s are so delayed and continued transactions are a risk. While + `2 * CONFIRMATIONS` is presumably well within the 6 hour period (defined + below), that period exists for low-fee transactions at time of congestion. It + does not exist for UIs with old state, though it can be used to compensate + for them (reducing the tolerance for inclusion delays). `2 * CONFIRMATIONS` + is before the 6 hour period is enacted, preserving the tolerance for + inclusion delays, yet still should only happen under highly abnormal + circumstances. + + In order to minimize the time it takes for "activation block" to be + finalized, a `Batch` will always be created for it, regardless of it would + otherwise have a `Batch` created. + +3) The prior multisig continues handling `Batch`s and `Burn`s for + `CONFIRMATIONS` blocks, plus 10 minutes, after the "activation block". + + The first `CONFIRMATIONS` blocks is due to the fact the new multisig + shouldn't actually be sent coins during this period, making it irrelevant. + If coins are prematurely sent to the new multisig, they're artificially + delayed until the end of the `CONFIRMATIONS` blocks plus 10 minutes period. + This prevents an adversary from minting Serai tokens using coins in the new + multisig, yet then burning them to drain the prior multisig, creating a lack + of liquidity for several blocks. + + The reason for the 10 minutes is to provide grace to honest UIs. Since UIs + will wait until Serai confirms the "activation block" for keys before sending + to them, which will take `CONFIRMATIONS` blocks plus some latency, UIs would + make transactions to the prior multisig past the end of this period if it was + `CONFIRMATIONS` alone. Since the next period is `CONFIRMATIONS` blocks, which + is how long transactions take to confirm, transactions made past the end of + this period would only received after the next period. After the next period, + the prior multisig adds fees and a delay to all received funds (as it + forwards the funds from itself to the new multisig). The 10 minutes provides + grace for latency. + + The 10 minutes is a delay on anyone who immediately transitions to the new + multisig, in a no latency environment, yet the delay is preferable to fees + from forwarding. It also should be less than 10 minutes thanks to various + latencies. + +4) The prior multisig continues handling `Batch`s and `Burn`s for another + `CONFIRMATIONS` blocks. + + This is for two reasons: + + 1) Coins sent to the new multisig still need time to gain sufficient + confirmations. + 2) All outputs belonging to the prior multisig should become available within + `CONFIRMATIONS` blocks. + + All `Burn`s handled during this period should use the new multisig for the + change address. This should effect a transfer of most outputs. + + With the expected transfer of most outputs, and the new multisig receiving + new external transactions, the new multisig takes the responsibility of + signing all unhandled and newly emitted `Burn`s. + +5) For the next 6 hours, all non-`Branch` outputs received are immediately + forwarded to the new multisig. Only external transactions to the new multisig + are included in `Batch`s. + + The new multisig infers the `InInstruction`, and refund address, for + forwarded `External` outputs via reading what they were for the original + `External` output. + + Alternatively, the `InInstruction`, with refund address explicitly included, + could be included in the forwarding transaction. This may fail if the + `InInstruction` omitted the refund address and is too large to fit in a + transaction with one explicitly included. On such failure, the refund would + be immediately issued instead. + +6) Once the 6 hour period has expired, the prior multisig stops handling outputs + it didn't itself create. Any remaining `Eventuality`s are completed, and any + available/freshly available outputs are forwarded (creating new + `Eventuality`s which also need to successfully resolve). + + Once all the 6 hour period has expired, no `Eventuality`s remain, and all + outputs are forwarded, the multisig publishes a final `Batch` of the first + block, plus `CONFIRMATIONS`, which met these conditions, regardless of if it + would've otherwise had a `Batch`. Then, it reports to Substrate has closed. + No further actions by it, nor its validators, are expected (unless those + validators remain present in the new multisig). + +7) The new multisig confirms all transactions from all prior multisigs were made + as expected, including the reported `Batch`s. + + Unfortunately, we cannot solely check the immediately prior multisig due to + the ability for two sequential malicious multisigs to steal. If multisig + `n - 2` only transfers a fraction of its coins to multisig `n - 1`, multisig + `n - 1` can 'honestly' operate on the dishonest state it was given, + laundering it. This would let multisig `n - 1` forward the results of its + as-expected operations from a dishonest starting point to the new multisig, + and multisig `n` would attest to multisig `n - 1`'s expected (and therefore + presumed honest) operations, assuming liability. This would cause an honest + multisig to face full liability for the invalid state, causing it to be fully + slashed (as needed to reacquire any lost coins). + + This would appear short-circuitable if multisig `n - 1` transfers coins + exceeding the relevant Serai tokens' supply. Serai never expects to operate + in an over-solvent state, yet balance should trend upwards due to a flat fee + applied to each received output (preventing a griefing attack). Any balance + greater than the tokens' supply may have had funds skimmed off the top, yet + they'd still guarantee the solvency of Serai without any additional fees + passed to users. Unfortunately, due to the requirement to verify the `Batch`s + published (as else the Serai tokens' supply may be manipulated), this cannot + actually be achieved (at least, not without a ZK proof the published `Batch`s + were correct). + +8) The new multisig reports a successful close of the prior multisig, and + becomes the sole multisig with full responsibilities. + +### Latency and Fees + +Slightly before the end of step 3, the new multisig should start receiving new +external outputs. These won't be confirmed for another `CONFIRMATIONS` blocks, +and the new multisig won't start handling `Burn`s for another `CONFIRMATIONS` +blocks plus 10 minutes. Accordingly, the new multisig should only become +responsible for `Burn`s shortly after it has taken ownership of the stream of +newly received coins. + +Before it takes responsibility, it also should've been transferred all internal +outputs under the standard scheduling flow. Any delayed outputs will be +immediately forwarded, and external stragglers are only reported to Serai once +sufficiently confirmed in the new multisig. Accordingly, liquidity should avoid +fragmentation during rotation. The only latency should be on the 10 minutes +present, and on delayed outputs, which should've been immediately usable, having +to wait another `CONFIRMATIONS` blocks to be confirmed once forwarded. + +Immediate forwarding does unfortunately prevent batching inputs to reduce fees. +Given immediate forwarding only applies to latent outputs, considered +exceptional, and the protocol's fee handling ensures solvency, this is accepted. diff --git a/docs/processor/Processor.md b/docs/processor/Processor.md index 293bcb3ee..ca8cf4282 100644 --- a/docs/processor/Processor.md +++ b/docs/processor/Processor.md @@ -6,7 +6,7 @@ transactions with payments. This document primarily discusses its flow with regards to the coordinator. -## Generate Key +### Generate Key On `key_gen::CoordinatorMessage::GenerateKey`, the processor begins a pair of instances of the distributed key generation protocol specified in the FROST @@ -20,26 +20,26 @@ Serai's overall key generation protocol. The commitments for both protocols are sent to the coordinator in a single `key_gen::ProcessorMessage::Commitments`. -## Key Gen Commitments +### Key Gen Commitments On `key_gen::CoordinatorMessage::Commitments`, the processor continues the specified key generation instance. The secret shares for each fellow participant are sent to the coordinator in a `key_gen::ProcessorMessage::Shares`. -### Key Gen Shares +#### Key Gen Shares On `key_gen::CoordinatorMessage::Shares`, the processor completes the specified key generation instance. The generated key pair is sent to the coordinator in a `key_gen::ProcessorMessage::GeneratedKeyPair`. -## Confirm Key Pair +### Confirm Key Pair On `substrate::CoordinatorMessage::ConfirmKeyPair`, the processor starts using the newly confirmed key, scanning blocks on the external network for transfers to it. -## Connected Network Block +### External Network Block When the external network has a new block, which is considered finalized (either due to being literally finalized or due to having a sufficient amount @@ -48,8 +48,14 @@ of confirmations), it's scanned. Outputs to the key of Serai's multisig are saved to the database. Outputs which newly transfer into Serai are used to build `Batch`s for the block. The processor then begins a threshold signature protocol with its key pair's -Ristretto key to sign the `Batch`s. The protocol's preprocess is sent to the -coordinator in a `coordinator::ProcessorMessage::BatchPreprocess`. +Ristretto key to sign the `Batch`s. + +The `Batch`s are each sent to the coordinator in a +`substrate::ProcessorMessage::Batch`, enabling the coordinator to know what +`Batch`s *should* be published to Serai. After each +`substrate::ProcessorMessage::Batch`, the preprocess for the first instance of +its signing protocol is sent to the coordinator in a +`coordinator::ProcessorMessage::BatchPreprocess`. As a design comment, we *may* be able to sign now possible, already scheduled, branch/leaf transactions at this point. Doing so would be giving a mutable @@ -57,26 +63,26 @@ borrow over the scheduler to both the external network and the Serai network, and would accordingly be unsafe. We may want to look at splitting the scheduler in two, in order to reduce latency (TODO). -## Batch Preprocesses +### Batch Preprocesses On `coordinator::CoordinatorMessage::BatchPreprocesses`, the processor continues the specified batch signing protocol, sending `coordinator::ProcessorMessage::BatchShare` to the coordinator. -## Batch Shares +### Batch Shares On `coordinator::CoordinatorMessage::BatchShares`, the processor completes the specified batch signing protocol. If successful, the processor -stops signing for this batch and sends `substrate::ProcessorMessage::Update` to -the coordinator. +stops signing for this batch and sends +`substrate::ProcessorMessage::SignedBatch` to the coordinator. -## Batch Re-attempt +### Batch Re-attempt On `coordinator::CoordinatorMessage::BatchReattempt`, the processor will create a new instance of the batch signing protocol. The new protocol's preprocess is sent to the coordinator in a `coordinator::ProcessorMessage::BatchPreprocess`. -## Substrate Block +### Substrate Block On `substrate::CoordinatorMessage::SubstrateBlock`, the processor: @@ -89,13 +95,13 @@ On `substrate::CoordinatorMessage::SubstrateBlock`, the processor: 4) Sends `sign::ProcessorMessage::Preprocess` for each plan now being signed for. -## Sign Preprocesses +### Sign Preprocesses On `sign::CoordinatorMessage::Preprocesses`, the processor continues the specified transaction signing protocol, sending `sign::ProcessorMessage::Share` to the coordinator. -## Sign Shares +### Sign Shares On `sign::CoordinatorMessage::Shares`, the processor completes the specified transaction signing protocol. If successful, the processor stops signing for @@ -104,7 +110,7 @@ this transaction and publishes the signed transaction. Then, broadcasted to all validators so everyone can observe the attempt completed, producing a signed and published transaction. -## Sign Re-attempt +### Sign Re-attempt On `sign::CoordinatorMessage::Reattempt`, the processor will create a new a new instance of the transaction signing protocol if it hasn't already @@ -112,7 +118,7 @@ completed/observed completion of an instance of the signing protocol. The new protocol's preprocess is sent to the coordinator in a `sign::ProcessorMessage::Preprocess`. -## Sign Completed +### Sign Completed On `sign::CoordinatorMessage::Completed`, the processor verifies the included transaction hash actually refers to an accepted transaction which completes the diff --git a/docs/processor/Scanning.md b/docs/processor/Scanning.md new file mode 100644 index 000000000..f03e36058 --- /dev/null +++ b/docs/processor/Scanning.md @@ -0,0 +1,31 @@ +# Scanning + +Only blocks with finality, either actual or sufficiently probabilistic, are +operated upon. This is referred to as a block with `CONFIRMATIONS` +confirmations, the block itself being the first confirmation. + +For chains which promise finality on a known schedule, `CONFIRMATIONS` is set to +`1` and each group of finalized blocks is treated as a single block, with the +tail block's hash representing the entire group. + +For chains which offer finality, on an unknown schedule, `CONFIRMATIONS` is +still set to `1` yet blocks aren't aggregated into a group. They're handled +individually, yet only once finalized. This allows networks which form +finalization erratically to not have to agree on when finalizations were formed, +solely that the blocks contained have a finalized descendant. + +### Notability, causing a `Batch` + +`Batch`s are only created for blocks which it benefits to achieve ordering on. +These are: + +- Blocks which contain transactions relevant to Serai +- Blocks which in which a new multisig activates +- Blocks in which a prior multisig retires + +### Waiting for `Batch` inclusion + +Once a `Batch` is created, it is expected to eventually be included on Serai. +If the `Batch` isn't included within `CONFIRMATIONS` blocks of its creation, the +scanner will wait until its inclusion before scanning +`batch_block + CONFIRMATIONS`. diff --git a/message-queue/src/client.rs b/message-queue/src/client.rs index 928c2735c..f1bf29d0b 100644 --- a/message-queue/src/client.rs +++ b/message-queue/src/client.rs @@ -140,9 +140,9 @@ impl MessageQueue { } } - pub async fn next(&self, expected: u64) -> QueuedMessage { + pub async fn next(&self, from: Service) -> QueuedMessage { loop { - let json = self.json_call("next", serde_json::json!([self.service, expected])).await; + let json = self.json_call("next", serde_json::json!([from, self.service])).await; // Convert from a Value to a type via reserialization let msg: Option = serde_json::from_str( @@ -174,24 +174,23 @@ impl MessageQueue { ); } // TODO: Verify the sender's signature - // TODO: Check the ID is sane return msg; } } - pub async fn ack(&self, id: u64) { + pub async fn ack(&self, from: Service, id: u64) { // TODO: Should this use OsRng? Deterministic or deterministic + random may be better. let nonce = Zeroizing::new(::F::random(&mut OsRng)); let nonce_pub = Ristretto::generator() * nonce.deref(); let sig = SchnorrSignature::::sign( &self.priv_key, nonce, - ack_challenge(self.service, self.pub_key, id, nonce_pub), + ack_challenge(self.service, self.pub_key, from, id, nonce_pub), ) .serialize(); - let json = self.json_call("ack", serde_json::json!([self.service, id, sig])).await; + let json = self.json_call("ack", serde_json::json!([from, self.service, id, sig])).await; if json.get("result") != Some(&serde_json::Value::Bool(true)) { panic!("failed to ack message {id}: {json}"); } diff --git a/message-queue/src/main.rs b/message-queue/src/main.rs index c59abe5b6..f2ae69c4b 100644 --- a/message-queue/src/main.rs +++ b/message-queue/src/main.rs @@ -25,12 +25,17 @@ mod binaries { pub(crate) type Db = serai_db::RocksDB; - lazy_static::lazy_static! { - pub(crate) static ref KEYS: Arc::G>>> = - Arc::new(RwLock::new(HashMap::new())); - pub(crate) static ref QUEUES: Arc>>>> = - Arc::new(RwLock::new(HashMap::new())); + #[allow(clippy::type_complexity)] + mod clippy { + use super::*; + lazy_static::lazy_static! { + pub(crate) static ref KEYS: Arc::G>>> = + Arc::new(RwLock::new(HashMap::new())); + pub(crate) static ref QUEUES: Arc>>>> = + Arc::new(RwLock::new(HashMap::new())); + } } + pub(crate) use self::clippy::*; // queue RPC method /* @@ -71,16 +76,17 @@ mod binaries { fn key(domain: &'static [u8], key: impl AsRef<[u8]>) -> Vec { [&[u8::try_from(domain.len()).unwrap()], domain, key.as_ref()].concat() } - fn intent_key(from: Service, intent: &[u8]) -> Vec { - key(b"intent_seen", bincode::serialize(&(from, intent)).unwrap()) + fn intent_key(from: Service, to: Service, intent: &[u8]) -> Vec { + key(b"intent_seen", bincode::serialize(&(from, to, intent)).unwrap()) } let mut db = db.write().unwrap(); let mut txn = db.txn(); - let intent_key = intent_key(meta.from, &meta.intent); + let intent_key = intent_key(meta.from, meta.to, &meta.intent); if Get::get(&txn, &intent_key).is_some() { log::warn!( - "Prior queued message attempted to be queued again. From: {:?} Intent: {}", + "Prior queued message attempted to be queued again. From: {:?} To: {:?} Intent: {}", meta.from, + meta.to, hex::encode(&meta.intent) ); return; @@ -88,7 +94,7 @@ mod binaries { DbTxn::put(&mut txn, intent_key, []); // Queue it - let id = (*QUEUES).read().unwrap()[&meta.to].write().unwrap().queue_message( + let id = (*QUEUES).read().unwrap()[&(meta.from, meta.to)].write().unwrap().queue_message( &mut txn, QueuedMessage { from: meta.from, @@ -99,26 +105,21 @@ mod binaries { }, ); - log::info!("Queued message from {:?}. It is {:?} {id}", meta.from, meta.to); + log::info!("Queued message. From: {:?} To: {:?} ID: {id}", meta.from, meta.to); DbTxn::commit(txn); } // next RPC method /* - Gets the next message in queue for this service. - - This is not authenticated due to the fact every nonce would have to be saved to prevent replays, - or a challenge-response protocol implemented. Neither are worth doing when there should be no - sensitive data on this server. + Gets the next message in queue for the named services. - The expected index is used to ensure a service didn't fall out of sync with this service. It - should always be either the next message's ID or *TODO*. + This is not authenticated due to the fact every nonce would have to be saved to prevent + replays, or a challenge-response protocol implemented. Neither are worth doing when there + should be no sensitive data on this server. */ - pub(crate) fn get_next_message(service: Service, _expected: u64) -> Option { - // TODO: Verify the expected next message ID matches - + pub(crate) fn get_next_message(from: Service, to: Service) -> Option { let queue_outer = (*QUEUES).read().unwrap(); - let queue = queue_outer[&service].read().unwrap(); + let queue = queue_outer[&(from, to)].read().unwrap(); let next = queue.last_acknowledged().map(|i| i + 1).unwrap_or(0); queue.get_message(next) } @@ -128,10 +129,10 @@ mod binaries { Acknowledges a message as received and handled, meaning it'll no longer be returned as the next message. */ - pub(crate) fn ack_message(service: Service, id: u64, sig: SchnorrSignature) { + pub(crate) fn ack_message(from: Service, to: Service, id: u64, sig: SchnorrSignature) { { - let from = (*KEYS).read().unwrap()[&service]; - assert!(sig.verify(from, ack_challenge(service, from, id, sig.R))); + let to_key = (*KEYS).read().unwrap()[&to]; + assert!(sig.verify(to_key, ack_challenge(to, to_key, from, id, sig.R))); } // Is it: @@ -141,9 +142,9 @@ mod binaries { // It's the second if we acknowledge messages before saving them as acknowledged // TODO: Check only a proper message is being acked - log::info!("{:?} is acknowledging {}", service, id); + log::info!("Acknowledging From: {:?} To: {:?} ID: {}", from, to, id); - (*QUEUES).read().unwrap()[&service].write().unwrap().ack_message(id) + (*QUEUES).read().unwrap()[&(from, to)].write().unwrap().ack_message(id) } } @@ -182,13 +183,29 @@ async fn main() { Some(::G::from_bytes(&repr).unwrap()) }; + const ALL_EXT_NETWORKS: [NetworkId; 3] = + [NetworkId::Bitcoin, NetworkId::Ethereum, NetworkId::Monero]; + let register_service = |service, key| { (*KEYS).write().unwrap().insert(service, key); - (*QUEUES).write().unwrap().insert(service, RwLock::new(Queue(db.clone(), service))); + let mut queues = (*QUEUES).write().unwrap(); + if service == Service::Coordinator { + for network in ALL_EXT_NETWORKS { + queues.insert( + (service, Service::Processor(network)), + RwLock::new(Queue(db.clone(), service, Service::Processor(network))), + ); + } + } else { + queues.insert( + (service, Service::Coordinator), + RwLock::new(Queue(db.clone(), service, Service::Coordinator)), + ); + } }; // Make queues for each NetworkId, other than Serai - for network in [NetworkId::Bitcoin, NetworkId::Ethereum, NetworkId::Monero] { + for network in ALL_EXT_NETWORKS { // Use a match so we error if the list of NetworkIds changes let Some(key) = read_key(match network { NetworkId::Serai => unreachable!(), @@ -229,17 +246,18 @@ async fn main() { .unwrap(); module .register_method("next", |args, _| { - let args = args.parse::<(Service, u64)>().unwrap(); - Ok(get_next_message(args.0, args.1)) + let (from, to) = args.parse::<(Service, Service)>().unwrap(); + Ok(get_next_message(from, to)) }) .unwrap(); module .register_method("ack", |args, _| { - let args = args.parse::<(Service, u64, Vec)>().unwrap(); + let args = args.parse::<(Service, Service, u64, Vec)>().unwrap(); ack_message( args.0, args.1, - SchnorrSignature::::read(&mut args.2.as_slice()).unwrap(), + args.2, + SchnorrSignature::::read(&mut args.3.as_slice()).unwrap(), ); Ok(true) }) diff --git a/message-queue/src/messages.rs b/message-queue/src/messages.rs index 9ad393dea..89557ad7f 100644 --- a/message-queue/src/messages.rs +++ b/message-queue/src/messages.rs @@ -48,15 +48,17 @@ pub fn message_challenge( } pub fn ack_challenge( + to: Service, + to_key: ::G, from: Service, - from_key: ::G, id: u64, nonce: ::G, ) -> ::F { let mut transcript = RecommendedTranscript::new(b"Serai Message Queue v0.1 Ackowledgement"); transcript.domain_separate(b"metadata"); + transcript.append_message(b"to", bincode::serialize(&to).unwrap()); + transcript.append_message(b"to_key", to_key.to_bytes()); transcript.append_message(b"from", bincode::serialize(&from).unwrap()); - transcript.append_message(b"from_key", from_key.to_bytes()); transcript.domain_separate(b"message"); transcript.append_message(b"id", id.to_le_bytes()); transcript.domain_separate(b"signature"); diff --git a/message-queue/src/queue.rs b/message-queue/src/queue.rs index 282734103..aa9e99cd6 100644 --- a/message-queue/src/queue.rs +++ b/message-queue/src/queue.rs @@ -3,14 +3,14 @@ use serai_db::{DbTxn, Db}; use crate::messages::*; #[derive(Clone, Debug)] -pub(crate) struct Queue(pub(crate) D, pub(crate) Service); +pub(crate) struct Queue(pub(crate) D, pub(crate) Service, pub(crate) Service); impl Queue { fn key(domain: &'static [u8], key: impl AsRef<[u8]>) -> Vec { [&[u8::try_from(domain.len()).unwrap()], domain, key.as_ref()].concat() } fn message_count_key(&self) -> Vec { - Self::key(b"message_count", serde_json::to_vec(&self.1).unwrap()) + Self::key(b"message_count", bincode::serialize(&(self.1, self.2)).unwrap()) } pub(crate) fn message_count(&self) -> u64 { self @@ -21,7 +21,7 @@ impl Queue { } fn last_acknowledged_key(&self) -> Vec { - Self::key(b"last_acknowledged", serde_json::to_vec(&self.1).unwrap()) + Self::key(b"last_acknowledged", bincode::serialize(&(self.1, self.2)).unwrap()) } pub(crate) fn last_acknowledged(&self) -> Option { self @@ -31,7 +31,7 @@ impl Queue { } fn message_key(&self, id: u64) -> Vec { - Self::key(b"message", serde_json::to_vec(&(self.1, id)).unwrap()) + Self::key(b"message", bincode::serialize(&(self.1, self.2, id)).unwrap()) } // TODO: This is fine as-used, yet gets from the DB while having a txn. It should get from the // txn diff --git a/mini/Cargo.toml b/mini/Cargo.toml new file mode 100644 index 000000000..fc0abf8a0 --- /dev/null +++ b/mini/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "mini-serai" +version = "0.1.0" +description = "A miniature version of Serai used to test for race conditions" +license = "AGPL-3.0-only" +repository = "https://github.com/serai-dex/serai/tree/develop/mini" +authors = ["Luke Parker "] +keywords = [] +edition = "2021" +publish = false + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[dependencies] +loom = "0.7" diff --git a/mini/LICENSE b/mini/LICENSE new file mode 100644 index 000000000..f684d0271 --- /dev/null +++ b/mini/LICENSE @@ -0,0 +1,15 @@ +AGPL-3.0-only license + +Copyright (c) 2023 Luke Parker + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License Version 3 as +published by the Free Software Foundation. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . diff --git a/mini/README.md b/mini/README.md new file mode 100644 index 000000000..8e8daf71a --- /dev/null +++ b/mini/README.md @@ -0,0 +1,27 @@ +# Mini Serai + +A miniature version of the Serai stack, intended to demonstrate a lack of +system-wide race conditions in the officially stated flows. + +### Why + +When working on multiple multisigs, a race condition was noted. Originally, the +documentation stated that the activation block of the new multisig would be the +block after the next `Batch`'s block. This introduced a race condition, where +since multiple `Batch`s can be signed at the same time, multiple `Batch`s can +exist in the mempool at the same time. This could cause `Batch`s [1, 2] to +exist in the mempool, 1 to be published (causing 2 to be the activation block of +the new multisig), yet then the already signed 2 to be published (despite +no longer being accurate as it only had events for a subset of keys). + +This effort initially modeled and tested this single race condition, yet aims to +grow to the entire system. Then we just have to prove the actual Serai stack's +flow reduces to the miniature flow modeled here. While further efforts are +needed to prove Serai's implementation of the flow is itself free of race +conditions, this is a layer of defense over the theory. + +### How + +[loom](https://docs.rs/loom) is a library which will execute a block of code +with every possible combination of orders in order to test results aren't +invalidated by order of execution. diff --git a/mini/src/lib.rs b/mini/src/lib.rs new file mode 100644 index 000000000..57cd8693e --- /dev/null +++ b/mini/src/lib.rs @@ -0,0 +1,150 @@ +use std::sync::{Arc as StdArc, RwLock as StdRwLock}; + +use loom::{ + thread::{self, JoinHandle}, + sync::{Arc, RwLock, mpsc}, +}; + +#[cfg(test)] +mod tests; + +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub struct Batch { + block: u64, + keys: Vec, +} + +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub enum Event { + IncludedBatch(Batch), + // Allows if let else on this without clippy believing it's redundant + __Ignore, +} + +// The amount of blocks to scan after we publish a batch, before confirming the batch was +// included. +// Prevents race conditions on rotation regarding when the new keys activate. +const BATCH_FTL: u64 = 3; + +#[derive(Debug)] +pub struct Serai { + handle: JoinHandle<()>, + remaining_ticks: Arc>, + // Activation block, ID + pub active_keys: Arc>>, + pub mempool_batches: Arc>>, + pub events: mpsc::Receiver, + all_events_unsafe: StdArc>>, +} + +impl Serai { + #[allow(clippy::new_without_default)] + pub fn new(ticks: usize, mut queued_key: bool) -> Serai { + let remaining_ticks = Arc::new(RwLock::new(ticks)); + + let active_keys = Arc::new(RwLock::new(vec![(0, 0)])); + let mempool_batches = Arc::new(RwLock::new(vec![])); + let (events_sender, events_receiver) = mpsc::channel(); + let all_events_unsafe = StdArc::new(StdRwLock::new(vec![])); + + let handle = thread::spawn({ + let remaining_ticks = remaining_ticks.clone(); + + let active_keys = active_keys.clone(); + let mempool_batches = mempool_batches.clone(); + let all_events_unsafe = all_events_unsafe.clone(); + + move || { + while { + let mut remaining_ticks = remaining_ticks.write().unwrap(); + let ticking = *remaining_ticks != 0; + *remaining_ticks = remaining_ticks.saturating_sub(1); + ticking + } { + let mut batches = mempool_batches.write().unwrap(); + if !batches.is_empty() { + let batch: Batch = batches.remove(0); + + // Activate keys after the FTL + if queued_key { + let mut active_keys = active_keys.write().unwrap(); + let len = active_keys.len().try_into().unwrap(); + // TODO: active_keys is under Serai, yet the processor is the one actually with the + // context on when it activates + // This should be re-modeled as an event + active_keys.push((batch.block + BATCH_FTL, len)); + } + queued_key = false; + + let event = Event::IncludedBatch(batch); + events_sender.send(event.clone()).unwrap(); + all_events_unsafe.write().unwrap().push(event); + } + } + } + }); + + Serai { + handle, + remaining_ticks, + mempool_batches, + active_keys, + events: events_receiver, + all_events_unsafe, + } + } + + pub fn exhausted(&self) -> bool { + *self.remaining_ticks.read().unwrap() == 0 + } + + pub fn join(self) -> Vec { + self.handle.join().unwrap(); + + self.all_events_unsafe.read().unwrap().clone() + } +} + +#[derive(Debug)] +pub struct Processor { + handle: JoinHandle, +} + +impl Processor { + pub fn new(serai: Serai, blocks: u64) -> Processor { + let handle = thread::spawn(move || { + let mut last_finalized_block = 0; + for b in 0 .. blocks { + // If this block is too far ahead of Serai's last block, wait for Serai to process + // Note this wait only has to occur if we have a Batch which has yet to be included + // mini just publishes a Batch for every Block at this point in time, meaning it always has + // to wait + while b >= (last_finalized_block + BATCH_FTL) { + if serai.exhausted() { + return serai; + } + let Ok(event) = serai.events.recv() else { return serai }; + if let Event::IncludedBatch(Batch { block, .. }) = event { + last_finalized_block = block; + } + } + serai.mempool_batches.write().unwrap().push(Batch { + block: b, + keys: serai + .active_keys + .read() + .unwrap() + .iter() + .filter_map(|(activation_block, id)| Some(*id).filter(|_| b >= *activation_block)) + .collect(), + }); + } + serai + }); + Processor { handle } + } + + pub fn join(self) -> Serai { + self.handle.join().unwrap() + } +} diff --git a/mini/src/tests/activation_race/mod.rs b/mini/src/tests/activation_race/mod.rs new file mode 100644 index 000000000..846664125 --- /dev/null +++ b/mini/src/tests/activation_race/mod.rs @@ -0,0 +1,174 @@ +use std::{ + collections::HashSet, + sync::{Arc as StdArc, RwLock as StdRwLock}, +}; + +use crate::*; + +#[test] +fn activation_race() { + #[derive(Debug)] + struct EagerProcessor { + handle: JoinHandle, + } + + impl EagerProcessor { + fn new(serai: Serai, batches: u64) -> EagerProcessor { + let handle = thread::spawn(move || { + for b in 0 .. batches { + serai.mempool_batches.write().unwrap().push(Batch { + block: b, + keys: serai + .active_keys + .read() + .unwrap() + .iter() + .filter_map(|(activation_block, id)| Some(*id).filter(|_| b >= *activation_block)) + .collect(), + }); + } + serai + }); + EagerProcessor { handle } + } + + fn join(self) -> Serai { + self.handle.join().unwrap() + } + } + + let results = StdArc::new(StdRwLock::new(HashSet::new())); + + loom::model({ + let results = results.clone(); + move || { + let serai = Serai::new(4, true); + let processor = EagerProcessor::new(serai, 4); + let serai = processor.join(); + let events = serai.join(); + + results.write().unwrap().insert(events); + } + }); + + let results: HashSet<_> = results.read().unwrap().clone(); + assert_eq!(results.len(), 6); + for result in results { + for (b, batch) in result.into_iter().enumerate() { + if b < 3 { + assert_eq!( + batch, + Event::IncludedBatch(Batch { block: b.try_into().unwrap(), keys: vec![0] }) + ); + } else { + let Event::IncludedBatch(batch) = batch else { panic!("unexpected event") }; + assert_eq!(batch.block, b.try_into().unwrap()); + assert!((batch.keys == vec![0]) || (batch.keys == vec![0, 1])); + } + } + } +} + +#[test] +fn sequential_solves_activation_race() { + #[derive(Debug)] + struct DelayedProcessor { + handle: JoinHandle, + } + + impl DelayedProcessor { + fn new(serai: Serai, batches: u64) -> DelayedProcessor { + let handle = thread::spawn(move || { + for b in 0 .. batches { + let batch = { + let mut batches = serai.mempool_batches.write().unwrap(); + let batch = Batch { + block: b, + keys: serai + .active_keys + .read() + .unwrap() + .iter() + .filter_map(|(activation_block, id)| Some(*id).filter(|_| b >= *activation_block)) + .collect(), + }; + batches.push(batch.clone()); + batch + }; + + while (!serai.exhausted()) && + (serai.events.recv().unwrap() != Event::IncludedBatch(batch.clone())) + { + loom::thread::yield_now(); + } + } + serai + }); + DelayedProcessor { handle } + } + + fn join(self) -> Serai { + self.handle.join().unwrap() + } + } + + let results = StdArc::new(StdRwLock::new(HashSet::new())); + + loom::model({ + let results = results.clone(); + move || { + let serai = Serai::new(4, true); + let processor = DelayedProcessor::new(serai, 4); + let serai = processor.join(); + let events = serai.join(); + + results.write().unwrap().insert(events); + } + }); + + let results: HashSet<_> = results.read().unwrap().clone(); + assert_eq!(results.len(), 5); + for result in results { + for (b, batch) in result.into_iter().enumerate() { + assert_eq!( + batch, + Event::IncludedBatch(Batch { + block: b.try_into().unwrap(), + keys: if b < 3 { vec![0] } else { vec![0, 1] } + }), + ); + } + } +} + +#[test] +fn ftl_solves_activation_race() { + let results = StdArc::new(StdRwLock::new(HashSet::new())); + + loom::model({ + let results = results.clone(); + move || { + let serai = Serai::new(4, true); + // Uses Processor since this Processor has this algorithm implemented + let processor = Processor::new(serai, 4); + let serai = processor.join(); + let events = serai.join(); + + results.write().unwrap().insert(events); + } + }); + + let results: HashSet<_> = results.read().unwrap().clone(); + assert_eq!(results.len(), 5); + for result in results { + for (b, batch) in result.into_iter().enumerate() { + assert_eq!( + batch, + Event::IncludedBatch(Batch { + block: b.try_into().unwrap(), + keys: if b < 3 { vec![0] } else { vec![0, 1] } + }), + ); + } + } +} diff --git a/mini/src/tests/mod.rs b/mini/src/tests/mod.rs new file mode 100644 index 000000000..76fce26c3 --- /dev/null +++ b/mini/src/tests/mod.rs @@ -0,0 +1 @@ +mod activation_race; diff --git a/orchestration/coins/bitcoin/Dockerfile b/orchestration/coins/bitcoin/Dockerfile index cff65eb98..cf06d864f 100644 --- a/orchestration/coins/bitcoin/Dockerfile +++ b/orchestration/coins/bitcoin/Dockerfile @@ -7,8 +7,7 @@ ENV BITCOIN_DATA=/home/bitcoin/.bitcoin WORKDIR /home/bitcoin -RUN apk update && \ - apk --no-cache add ca-certificates bash su-exec git gnupg +RUN apk --no-cache add git gnupg # Download Bitcoin RUN wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz \ @@ -27,8 +26,24 @@ RUN grep bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz SHA256SUMS | sha256s RUN tar xzvf bitcoin-${BITCOIN_VERSION}-x86_64-linux-gnu.tar.gz RUN mv bitcoin-${BITCOIN_VERSION}/bin/bitcoind . +# Also build mimalloc +FROM debian:bookworm-slim as mimalloc + +RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git +RUN git clone https://github.com/microsoft/mimalloc && \ + cd mimalloc && \ + mkdir -p out/secure && \ + cd out/secure && \ + cmake -DMI_SECURE=ON ../.. && \ + make && \ + cp ./libmimalloc-secure.so ../../../libmimalloc.so + +# Build the actual image FROM debian:bookworm-slim as image +COPY --from=mimalloc libmimalloc.so /usr/lib +RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload + # Upgrade packages RUN apt update && apt upgrade -y && apt autoremove -y && apt clean diff --git a/orchestration/coins/monero/Dockerfile b/orchestration/coins/monero/Dockerfile index 930a936e1..991228bce 100644 --- a/orchestration/coins/monero/Dockerfile +++ b/orchestration/coins/monero/Dockerfile @@ -11,8 +11,16 @@ ENV GLIBC_VERSION=2.28-r0 WORKDIR /home/monero -RUN apk update \ - && apk --no-cache add ca-certificates gnupg bash su-exec +RUN apk update && apk --no-cache add gcc g++ libc-dev make cmake git +RUN git clone https://github.com/microsoft/mimalloc && \ + cd mimalloc && \ + mkdir -p out/secure && \ + cd out/secure && \ + cmake -DMI_SECURE=ON ../.. && \ + make && \ + cp ./libmimalloc-secure.so ../../../libmimalloc.so + +RUN apk --no-cache add gnupg # Download Monero RUN wget https://downloads.getmonero.org/cli/monero-linux-x64-v${MONERO_VERSION}.tar.bz2 @@ -29,8 +37,11 @@ RUN tar -xvjf monero-linux-x64-v${MONERO_VERSION}.tar.bz2 --strip-components=1 # Build the actual image FROM alpine:latest as image +COPY --from=builder /home/monero/libmimalloc.so /usr/lib +ENV LD_PRELOAD=libmimalloc.so + # Upgrade packages -RUN apk update && apk upgrade && apk add gcompat +RUN apk update && apk upgrade && apk --no-cache add gcompat # Switch to a non-root user # System user (not a human), shell of nologin, no password assigned diff --git a/orchestration/coordinator/Dockerfile b/orchestration/coordinator/Dockerfile index 039a5819e..92dc1115b 100644 --- a/orchestration/coordinator/Dockerfile +++ b/orchestration/coordinator/Dockerfile @@ -15,6 +15,7 @@ ADD message-queue /serai/message-queue ADD processor /serai/processor ADD coordinator /serai/coordinator ADD substrate /serai/substrate +ADD mini /serai/mini ADD tests /serai/tests ADD Cargo.toml /serai ADD Cargo.lock /serai @@ -32,9 +33,23 @@ RUN --mount=type=cache,target=/root/.cargo \ mkdir /serai/bin && \ mv /serai/target/release/serai-coordinator /serai/bin -# Prepare Image +# Also build mimalloc +FROM debian:bookworm-slim as mimalloc + +RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git +RUN git clone https://github.com/microsoft/mimalloc && \ + cd mimalloc && \ + mkdir -p out/secure && \ + cd out/secure && \ + cmake -DMI_SECURE=ON ../.. && \ + make && \ + cp ./libmimalloc-secure.so ../../../libmimalloc.so + +# Build the actual image FROM debian:bookworm-slim as image -LABEL description="STAGE 2: Copy and Run" + +COPY --from=mimalloc libmimalloc.so /usr/lib +RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload # Upgrade packages and install openssl RUN apt update && apt upgrade -y && apt install -y libssl-dev && apt autoremove && apt clean diff --git a/orchestration/message-queue/Dockerfile b/orchestration/message-queue/Dockerfile index c4a17fce1..1571f4a36 100644 --- a/orchestration/message-queue/Dockerfile +++ b/orchestration/message-queue/Dockerfile @@ -12,6 +12,7 @@ ADD message-queue /serai/message-queue ADD processor /serai/processor ADD coordinator /serai/coordinator ADD substrate /serai/substrate +ADD mini /serai/mini ADD tests /serai/tests ADD Cargo.toml /serai ADD Cargo.lock /serai @@ -29,9 +30,23 @@ RUN --mount=type=cache,target=/root/.cargo \ mkdir /serai/bin && \ mv /serai/target/release/serai-message-queue /serai/bin -# Prepare Image +# Also build mimalloc +FROM debian:bookworm-slim as mimalloc + +RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git +RUN git clone https://github.com/microsoft/mimalloc && \ + cd mimalloc && \ + mkdir -p out/secure && \ + cd out/secure && \ + cmake -DMI_SECURE=ON ../.. && \ + make && \ + cp ./libmimalloc-secure.so ../../../libmimalloc.so + +# Build the actual image FROM debian:bookworm-slim as image -LABEL description="STAGE 2: Copy and Run" + +COPY --from=mimalloc libmimalloc.so /usr/lib +RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload # Upgrade packages RUN apt update && apt upgrade -y diff --git a/orchestration/processor/Dockerfile b/orchestration/processor/Dockerfile index 13c7559f6..a6c2a2666 100644 --- a/orchestration/processor/Dockerfile +++ b/orchestration/processor/Dockerfile @@ -12,6 +12,7 @@ ADD message-queue /serai/message-queue ADD processor /serai/processor ADD coordinator /serai/coordinator ADD substrate /serai/substrate +ADD mini /serai/mini ADD tests /serai/tests ADD Cargo.toml /serai ADD Cargo.lock /serai @@ -32,9 +33,23 @@ RUN --mount=type=cache,target=/root/.cargo \ mkdir /serai/bin && \ mv /serai/target/release/serai-processor /serai/bin -# Prepare Image +# Also build mimalloc +FROM debian:bookworm-slim as mimalloc + +RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git +RUN git clone https://github.com/microsoft/mimalloc && \ + cd mimalloc && \ + mkdir -p out/secure && \ + cd out/secure && \ + cmake -DMI_SECURE=ON ../.. && \ + make && \ + cp ./libmimalloc-secure.so ../../../libmimalloc.so + +# Build the actual image FROM debian:bookworm-slim as image -LABEL description="STAGE 2: Copy and Run" + +COPY --from=mimalloc libmimalloc.so /usr/lib +RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload # Upgrade packages and install openssl RUN apt update && apt upgrade -y && apt install -y libssl-dev diff --git a/orchestration/runtime/Dockerfile b/orchestration/runtime/Dockerfile index 4d10530b7..b8b13ac63 100644 --- a/orchestration/runtime/Dockerfile +++ b/orchestration/runtime/Dockerfile @@ -20,6 +20,7 @@ ADD message-queue /serai/message-queue ADD processor /serai/processor ADD coordinator /serai/coordinator ADD substrate /serai/substrate +ADD mini /serai/mini ADD tests /serai/tests ADD Cargo.toml /serai ADD Cargo.lock /serai diff --git a/orchestration/serai/Dockerfile b/orchestration/serai/Dockerfile index 8c5d8347f..21d0f8a41 100644 --- a/orchestration/serai/Dockerfile +++ b/orchestration/serai/Dockerfile @@ -12,6 +12,7 @@ ADD message-queue /serai/message-queue ADD processor /serai/processor ADD coordinator /serai/coordinator ADD substrate /serai/substrate +ADD mini /serai/mini ADD tests /serai/tests ADD Cargo.toml /serai ADD Cargo.lock /serai @@ -32,9 +33,23 @@ RUN --mount=type=cache,target=/root/.cargo \ mkdir /serai/bin && \ mv /serai/target/release/serai-node /serai/bin -# Prepare Image +# Also build mimalloc +FROM debian:bookworm-slim as mimalloc + +RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git +RUN git clone https://github.com/microsoft/mimalloc && \ + cd mimalloc && \ + mkdir -p out/secure && \ + cd out/secure && \ + cmake -DMI_SECURE=ON ../.. && \ + make && \ + cp ./libmimalloc-secure.so ../../../libmimalloc.so + +# Build the actual image FROM debian:bookworm-slim as image -LABEL description="STAGE 2: Copy and Run" + +COPY --from=mimalloc libmimalloc.so /usr/lib +RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload # Upgrade packages RUN apt update && apt upgrade -y diff --git a/processor/README.md b/processor/README.md index 78eeb092a..37d11e0d4 100644 --- a/processor/README.md +++ b/processor/README.md @@ -1,56 +1,5 @@ # Processor -The Serai processor scans a specified chain, communicating with the coordinator. - -### Key Generation - -The coordinator will tell the processor if it's been included in managing a -coin. If so, the processor is to begin the key generation protocol, relying on -the coordinator to provided authenticated communication with the remote parties. - -When the key generation protocol successfully completes, the processor is -expected to inform the coordinator so it may vote on it on the Substrate chain. -Once the key is voted in, it'll become active. - -### Scanning - -Sufficiently confirmed block become finalized in the eyes of the procesor. -Finalized blocks are scanned and have their outputs emitted, though not acted -on. - -### Reporting - -The processor reports finalized blocks to the coordinator. Once the group -acknowledges the block as finalized, they begin a threshold signing protocol -to sign the block's outputs as a `Batch`. - -Once the `Batch` is signed, the processor emits an `Update` with the signed -batch. Serai includes it, definitively ordering its outputs within the context -of Serai. - -### Confirmed Outputs - -With the outputs' ordering, validators are able to act on them. - -Actions are triggered by passing the outputs to the scheduler. The scheduler -will do one of two things: - -1) Use the output -2) Accumulate it for later usage - -### Burn Events - -When the Serai chain issues a `Burn` event, the processor should send coins -accordingly. This is done by scheduling the payments out. - -# TODO - -- Items marked TODO -- Items marked TODO2, yet those only need to be done after protonet -- Test the implementors of Coin against the trait API -- Test the databases -- Test eventuality handling - -- Coordinator communication - -Kafka? RPC ping to them, which we don't count as 'sent' until we get a pong? +The Serai processor scans a specified external network, communicating with the +coordinator. For details on its exact messaging flow, and overall policies, +please view `docs/processor`. diff --git a/processor/messages/README.md b/processor/messages/README.md deleted file mode 100644 index 815eecb46..000000000 --- a/processor/messages/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Processor - -The Serai processor scans a specified chain, communicating with the coordinator. - -### Key Generation - -The coordinator will tell the processor if it's been included in managing a -coin. If so, the processor is to begin the key generation protocol, relying on -the coordinator to provided authenticated communication with the remote parties. - -When the key generation protocol successfully completes, the processor is -expected to inform the coordinator so it may vote on it on the Substrate chain. -Once the key is voted in, it'll become active. - -### Scanning - -The processor is expected to scan all sufficiently confirmed blocks from a given -coin. This will create a list of outputs, considered pending. - -### Reporting - -These outputs are to be placed in a `Batch`, identified by the block containing -them. Batches are provided in an `Update` to Serai, paired by an agreed upon -block number. - -The processor will also produce an `Update` if there have been no batches within -the confirmation window. - -### Confirmed Outputs - -Once outputs have been acknowledged by Serai, they are considered confirmed. -With their confirmation, the validators are ready to create actions based on -them. - -Actions are triggered by passing the outputs to the scheduler. The scheduler -will do one of two things: - -1) Use the output -2) Accumulate it for later usage - -### Burn Events - -When the Serai chain issues a `Burn` event, the processor should send coins -accordingly. This is done by scheduling the payments out. diff --git a/processor/messages/src/lib.rs b/processor/messages/src/lib.rs index 5ca16544f..01ab54cf8 100644 --- a/processor/messages/src/lib.rs +++ b/processor/messages/src/lib.rs @@ -8,7 +8,7 @@ use serde::{Serialize, Deserialize}; use dkg::{Participant, ThresholdParams}; use serai_primitives::{BlockHash, NetworkId}; -use in_instructions_primitives::SignedBatch; +use in_instructions_primitives::{Batch, SignedBatch}; use tokens_primitives::OutInstructionWithBalance; use validator_sets_primitives::{ValidatorSet, KeyPair}; @@ -161,7 +161,6 @@ pub mod substrate { context: SubstrateContext, network: NetworkId, block: u64, - key: Vec, burns: Vec, batches: Vec, }, @@ -179,7 +178,8 @@ pub mod substrate { #[derive(Clone, PartialEq, Eq, Debug, Zeroize, Encode, Decode, Serialize, Deserialize)] pub enum ProcessorMessage { - Update { batch: SignedBatch }, + Batch { batch: Batch }, + SignedBatch { batch: SignedBatch }, } } @@ -365,8 +365,9 @@ impl ProcessorMessage { ProcessorMessage::Substrate(msg) => { let (sub, id) = match msg { // Unique since network and ID binding - substrate::ProcessorMessage::Update { batch, .. } => { - (0, (batch.batch.network, batch.batch.id).encode()) + substrate::ProcessorMessage::Batch { batch } => (0, (batch.network, batch.id).encode()), + substrate::ProcessorMessage::SignedBatch { batch, .. } => { + (1, (batch.batch.network, batch.batch.id).encode()) } }; diff --git a/processor/src/coordinator.rs b/processor/src/coordinator.rs index 056cf6467..7f4e39fb0 100644 --- a/processor/src/coordinator.rs +++ b/processor/src/coordinator.rs @@ -10,14 +10,15 @@ pub struct Message { #[async_trait::async_trait] pub trait Coordinator { - async fn send(&mut self, msg: ProcessorMessage); + async fn send(&mut self, msg: impl Send + Into); async fn recv(&mut self) -> Message; async fn ack(&mut self, msg: Message); } #[async_trait::async_trait] impl Coordinator for MessageQueue { - async fn send(&mut self, msg: ProcessorMessage) { + async fn send(&mut self, msg: impl Send + Into) { + let msg: ProcessorMessage = msg.into(); let metadata = Metadata { from: self.service, to: Service::Coordinator, intent: msg.intent() }; let msg = serde_json::to_string(&msg).unwrap(); @@ -25,8 +26,7 @@ impl Coordinator for MessageQueue { } async fn recv(&mut self) -> Message { - // TODO: Use a proper expected next ID - let msg = self.next(0).await; + let msg = self.next(Service::Coordinator).await; let id = msg.id; @@ -38,6 +38,6 @@ impl Coordinator for MessageQueue { } async fn ack(&mut self, msg: Message) { - MessageQueue::ack(self, msg.id).await + MessageQueue::ack(self, Service::Coordinator, msg.id).await } } diff --git a/processor/src/db.rs b/processor/src/db.rs index 916c35dda..212341d0c 100644 --- a/processor/src/db.rs +++ b/processor/src/db.rs @@ -1,8 +1,12 @@ use core::marker::PhantomData; +use std::io::Read; + +use scale::{Encode, Decode}; +use serai_client::validator_sets::primitives::{ValidatorSet, KeyPair}; pub use serai_db::*; -use crate::{Plan, networks::Network}; +use crate::networks::{Block, Network}; #[derive(Debug)] pub struct MainDb(D, PhantomData); @@ -25,74 +29,35 @@ impl MainDb { txn.put(Self::handled_key(id), []) } - fn plan_key(id: &[u8]) -> Vec { - Self::main_key(b"plan", id) - } - fn signing_key(key: &[u8]) -> Vec { - Self::main_key(b"signing", key) - } - pub fn save_signing(txn: &mut D::Transaction<'_>, key: &[u8], block_number: u64, plan: &Plan) { - let id = plan.id(); - - { - let mut signing = txn.get(Self::signing_key(key)).unwrap_or(vec![]); - - // If we've already noted we're signing this, return - assert_eq!(signing.len() % 32, 0); - for i in 0 .. (signing.len() / 32) { - if signing[(i * 32) .. ((i + 1) * 32)] == id { - return; - } + fn pending_activation_key() -> Vec { + Self::main_key(b"pending_activation", []) + } + pub fn set_pending_activation( + txn: &mut D::Transaction<'_>, + block_before_queue_block: >::Id, + set: ValidatorSet, + key_pair: KeyPair, + ) { + let mut buf = (set, key_pair).encode(); + buf.extend(block_before_queue_block.as_ref()); + txn.put(Self::pending_activation_key(), buf); + } + pub fn pending_activation( + getter: &G, + ) -> Option<(>::Id, ValidatorSet, KeyPair)> { + if let Some(bytes) = getter.get(Self::pending_activation_key()) { + if !bytes.is_empty() { + let mut slice = bytes.as_slice(); + let (set, key_pair) = <(ValidatorSet, KeyPair)>::decode(&mut slice).unwrap(); + let mut block_before_queue_block = >::Id::default(); + slice.read_exact(block_before_queue_block.as_mut()).unwrap(); + assert!(slice.is_empty()); + return Some((block_before_queue_block, set, key_pair)); } - - signing.extend(&id); - txn.put(Self::signing_key(key), id); - } - - { - let mut buf = block_number.to_le_bytes().to_vec(); - plan.write(&mut buf).unwrap(); - txn.put(Self::plan_key(&id), &buf); } + None } - - pub fn signing(&self, key: &[u8]) -> Vec<(u64, Plan)> { - let signing = self.0.get(Self::signing_key(key)).unwrap_or(vec![]); - let mut res = vec![]; - - assert_eq!(signing.len() % 32, 0); - for i in 0 .. (signing.len() / 32) { - let id = &signing[(i * 32) .. ((i + 1) * 32)]; - let buf = self.0.get(Self::plan_key(id)).unwrap(); - - let block_number = u64::from_le_bytes(buf[.. 8].try_into().unwrap()); - let plan = Plan::::read::<&[u8]>(&mut &buf[16 ..]).unwrap(); - assert_eq!(id, &plan.id()); - res.push((block_number, plan)); - } - - res - } - - pub fn finish_signing(&mut self, txn: &mut D::Transaction<'_>, key: &[u8], id: [u8; 32]) { - let mut signing = self.0.get(Self::signing_key(key)).unwrap_or(vec![]); - assert_eq!(signing.len() % 32, 0); - - let mut found = false; - for i in 0 .. (signing.len() / 32) { - let start = i * 32; - let end = i + 32; - if signing[start .. end] == id { - found = true; - signing = [&signing[.. start], &signing[end ..]].concat().to_vec(); - break; - } - } - - if !found { - log::warn!("told to finish signing {} yet wasn't actively signing it", hex::encode(id)); - } - - txn.put(Self::signing_key(key), signing); + pub fn clear_pending_activation(txn: &mut D::Transaction<'_>) { + txn.put(Self::pending_activation_key(), []); } } diff --git a/processor/src/key_gen.rs b/processor/src/key_gen.rs index 15be33db9..fe6905da1 100644 --- a/processor/src/key_gen.rs +++ b/processor/src/key_gen.rs @@ -40,9 +40,8 @@ impl KeyGenDb { fn save_params(txn: &mut D::Transaction<'_>, set: &ValidatorSet, params: &ThresholdParams) { txn.put(Self::params_key(set), bincode::serialize(params).unwrap()); } - fn params(getter: &G, set: &ValidatorSet) -> ThresholdParams { - // Directly unwraps the .get() as this will only be called after being set - bincode::deserialize(&getter.get(Self::params_key(set)).unwrap()).unwrap() + fn params(getter: &G, set: &ValidatorSet) -> Option { + getter.get(Self::params_key(set)).map(|bytes| bincode::deserialize(&bytes).unwrap()) } // Not scoped to the set since that'd have latter attempts overwrite former @@ -92,13 +91,13 @@ impl KeyGenDb { fn read_keys( getter: &G, key: &[u8], - ) -> (Vec, (ThresholdKeys, ThresholdKeys)) { - let keys_vec = getter.get(key).unwrap(); + ) -> Option<(Vec, (ThresholdKeys, ThresholdKeys))> { + let keys_vec = getter.get(key)?; let mut keys_ref: &[u8] = keys_vec.as_ref(); let substrate_keys = ThresholdKeys::new(ThresholdCore::read(&mut keys_ref).unwrap()); let mut network_keys = ThresholdKeys::new(ThresholdCore::read(&mut keys_ref).unwrap()); N::tweak_keys(&mut network_keys); - (keys_vec, (substrate_keys, network_keys)) + Some((keys_vec, (substrate_keys, network_keys))) } fn confirm_keys( txn: &mut D::Transaction<'_>, @@ -106,7 +105,8 @@ impl KeyGenDb { key_pair: KeyPair, ) -> (ThresholdKeys, ThresholdKeys) { let (keys_vec, keys) = - Self::read_keys(txn, &Self::generated_keys_key(set, (&key_pair.0 .0, key_pair.1.as_ref()))); + Self::read_keys(txn, &Self::generated_keys_key(set, (&key_pair.0 .0, key_pair.1.as_ref()))) + .unwrap(); assert_eq!(key_pair.0 .0, keys.0.group_key().to_bytes()); assert_eq!( { @@ -121,10 +121,10 @@ impl KeyGenDb { fn keys( getter: &G, key: &::G, - ) -> (ThresholdKeys, ThresholdKeys) { - let res = Self::read_keys(getter, &Self::keys_key(key)).1; + ) -> Option<(ThresholdKeys, ThresholdKeys)> { + let res = Self::read_keys(getter, &Self::keys_key(key))?.1; assert_eq!(&res.1.group_key(), key); - res + Some(res) } } @@ -147,13 +147,21 @@ impl KeyGen { KeyGen { db, entropy, active_commit: HashMap::new(), active_share: HashMap::new() } } + pub fn in_set(&self, set: &ValidatorSet) -> bool { + // We determine if we're in set using if we have the parameters for a set's key generation + KeyGenDb::::params(&self.db, set).is_some() + } + pub fn keys( &self, key: &::G, - ) -> (ThresholdKeys, ThresholdKeys) { + ) -> Option<(ThresholdKeys, ThresholdKeys)> { // This is safe, despite not having a txn, since it's a static value // The only concern is it may not be set when expected, or it may be set unexpectedly - // Since this unwraps, it being unset when expected to be set will cause a panic + // + // They're only expected to be set on boot, if confirmed. If they were confirmed yet the + // transaction wasn't committed, their confirmation will be re-handled + // // The only other concern is if it's set when it's not safe to use // The keys are only written on confirmation, and the transaction writing them is atomic to // every associated operation @@ -220,7 +228,7 @@ impl KeyGen { panic!("commitments when already handled commitments"); } - let params = KeyGenDb::::params(txn, &id.set); + let params = KeyGenDb::::params(txn, &id.set).unwrap(); // Unwrap the machines, rebuilding them if we didn't have them in our cache // We won't if the processor rebooted @@ -288,7 +296,7 @@ impl KeyGen { CoordinatorMessage::Shares { id, shares } => { info!("Received shares for {:?}", id); - let params = KeyGenDb::::params(txn, &id.set); + let params = KeyGenDb::::params(txn, &id.set).unwrap(); // Same commentary on inconsistency as above exists let machines = self.active_share.remove(&id.set).unwrap_or_else(|| { diff --git a/processor/src/main.rs b/processor/src/main.rs index f364b2aa7..471d93286 100644 --- a/processor/src/main.rs +++ b/processor/src/main.rs @@ -1,28 +1,19 @@ -use std::{ - time::Duration, - collections::{VecDeque, HashMap}, -}; +use std::{sync::RwLock, time::Duration, collections::HashMap}; use zeroize::{Zeroize, Zeroizing}; use transcript::{Transcript, RecommendedTranscript}; -use ciphersuite::group::GroupEncoding; -use frost::{curve::Ciphersuite, ThresholdKeys}; +use ciphersuite::{group::GroupEncoding, Ciphersuite}; -use log::{info, warn, error}; +use log::{info, warn}; use tokio::time::sleep; -use scale::{Encode, Decode}; - use serai_client::{ - primitives::{MAX_DATA_LEN, BlockHash, NetworkId}, - tokens::primitives::{OutInstruction, OutInstructionWithBalance}, - in_instructions::primitives::{ - Shorthand, RefundableInInstruction, InInstructionWithBalance, Batch, MAX_BATCH_SIZE, - }, + primitives::{BlockHash, NetworkId}, + validator_sets::primitives::{ValidatorSet, KeyPair}, }; -use messages::{SubstrateContext, CoordinatorMessage, ProcessorMessage}; +use messages::CoordinatorMessage; use serai_env as env; @@ -32,7 +23,7 @@ mod plan; pub use plan::*; mod networks; -use networks::{OutputType, Output, PostFeeBranch, Block, Network}; +use networks::{PostFeeBranch, Block, Network, get_latest_block_number, get_block}; #[cfg(feature = "bitcoin")] use networks::Bitcoin; #[cfg(feature = "monero")] @@ -56,76 +47,12 @@ use signer::{SignerEvent, Signer}; mod substrate_signer; use substrate_signer::{SubstrateSignerEvent, SubstrateSigner}; -mod scanner; -use scanner::{ScannerEvent, Scanner, ScannerHandle}; - -mod scheduler; -use scheduler::Scheduler; +mod multisigs; +use multisigs::{MultisigEvent, MultisigManager}; #[cfg(test)] mod tests; -async fn get_latest_block_number(network: &N) -> usize { - loop { - match network.get_latest_block_number().await { - Ok(number) => { - return number; - } - Err(e) => { - error!( - "couldn't get the latest block number in main's error-free get_block. {} {}", - "this should only happen if the node is offline. error: ", e - ); - sleep(Duration::from_secs(10)).await; - } - } - } -} - -async fn get_block(network: &N, block_number: usize) -> N::Block { - loop { - match network.get_block(block_number).await { - Ok(block) => { - return block; - } - Err(e) => { - error!("couldn't get block {block_number} in main's error-free get_block. error: {}", e); - sleep(Duration::from_secs(10)).await; - } - } - } -} - -async fn get_fee(network: &N, block_number: usize) -> N::Fee { - // TODO2: Use an fee representative of several blocks - get_block(network, block_number).await.median_fee() -} - -async fn prepare_send( - network: &N, - keys: ThresholdKeys, - block_number: usize, - fee: N::Fee, - plan: Plan, -) -> (Option<(N::SignableTransaction, N::Eventuality)>, Vec) { - loop { - match network.prepare_send(keys.clone(), block_number, plan.clone(), fee).await { - Ok(prepared) => { - return prepared; - } - Err(e) => { - error!("couldn't prepare a send for plan {}: {e}", hex::encode(plan.id())); - // The processor is either trying to create an invalid TX (fatal) or the node went - // offline - // The former requires a patch, the latter is a connection issue - // If the latter, this is an appropriate sleep. If the former, we should panic, yet - // this won't flood the console ad infinitum - sleep(Duration::from_secs(60)).await; - } - } - } -} - // Items which are mutably borrowed by Tributary. // Any exceptions to this have to be carefully monitored in order to ensure consistency isn't // violated. @@ -164,71 +91,29 @@ struct TributaryMutable { // Items which are mutably borrowed by Substrate. // Any exceptions to this have to be carefully monitored in order to ensure consistency isn't // violated. -struct SubstrateMutable { - // The scanner is expected to autonomously operate, scanning blocks as they appear. - // When a block is sufficiently confirmed, the scanner mutates the signer to try and get a Batch - // signed. - // The scanner itself only mutates its list of finalized blocks and in-memory state though. - // Disk mutations to the scan-state only happen when Substrate says to. - - // This can't be mutated as soon as a Batch is signed since the mutation which occurs then is - // paired with the mutations caused by Burn events. Substrate's ordering determines if such a - // pairing exists. - scanner: ScannerHandle, - - // Schedulers take in new outputs, from the scanner, and payments, from Burn events on Substrate. - // These are paired when possible, in the name of efficiency. Accordingly, both mutations must - // happen by Substrate. - schedulers: HashMap, Scheduler>, -} -async fn sign_plans( - txn: &mut D::Transaction<'_>, - network: &N, - substrate_mutable: &mut SubstrateMutable, - signers: &mut HashMap, Signer>, - context: SubstrateContext, - plans: Vec>, -) { - let mut plans = VecDeque::from(plans); - - let mut block_hash = >::Id::default(); - block_hash.as_mut().copy_from_slice(&context.network_latest_finalized_block.0); - // block_number call is safe since it unwraps - let block_number = substrate_mutable - .scanner - .block_number(&block_hash) - .await - .expect("told to sign_plans on a context we're not synced to"); - - let fee = get_fee(network, block_number).await; - - while let Some(plan) = plans.pop_front() { - let id = plan.id(); - info!("preparing plan {}: {:?}", hex::encode(id), plan); - - let key = plan.key.to_bytes(); - MainDb::::save_signing(txn, key.as_ref(), block_number.try_into().unwrap(), &plan); - let (tx, branches) = - prepare_send(network, signers.get_mut(key.as_ref()).unwrap().keys(), block_number, fee, plan) - .await; +/* + The MultisigManager contains the Scanner and Schedulers. - for branch in branches { - substrate_mutable - .schedulers - .get_mut(key.as_ref()) - .expect("didn't have a scheduler for a key we have a plan for") - .created_output::(txn, branch.expected, branch.actual); - } + The scanner is expected to autonomously operate, scanning blocks as they appear. When a block is + sufficiently confirmed, the scanner causes the Substrate signer to sign a batch. It itself only + mutates its list of finalized blocks, to protect against re-orgs, and its in-memory state though. - if let Some((tx, eventuality)) = tx { - substrate_mutable.scanner.register_eventuality(block_number, id, eventuality.clone()).await; - signers.get_mut(key.as_ref()).unwrap().sign_transaction(txn, id, tx, eventuality).await; - } + Disk mutations to the scan-state only happens once the relevant `Batch` is included on Substrate. + It can't be mutated as soon as the `Batch` is signed as we need to know the order of `Batch`s + relevant to `Burn`s. - // TODO: If the TX is None, should we restore its inputs to the scheduler? - } -} + Schedulers take in new outputs, confirmed in `Batch`s, and outbound payments, triggered by + `Burn`s. + + Substrate also decides when to move to a new multisig, hence why this entire object is + Substate-mutable. + + Since MultisigManager should always be verifiable, and the Tributary is temporal, MultisigManager + being entirely SubstrateMutable shows proper data pipe-lining. +*/ + +type SubstrateMutable = MultisigManager; async fn handle_coordinator_msg( txn: &mut D::Transaction<'_>, @@ -239,15 +124,18 @@ async fn handle_coordinator_msg( msg: &Message, ) { // If this message expects a higher block number than we have, halt until synced - async fn wait(scanner: &ScannerHandle, block_hash: &BlockHash) { + async fn wait( + txn: &D::Transaction<'_>, + substrate_mutable: &SubstrateMutable, + block_hash: &BlockHash, + ) { let mut needed_hash = >::Id::default(); needed_hash.as_mut().copy_from_slice(&block_hash.0); - let block_number = loop { + loop { // Ensure our scanner has scanned this block, which means our daemon has this block at // a sufficient depth - // The block_number may be set even if scanning isn't complete - let Some(block_number) = scanner.block_number(&needed_hash).await else { + if substrate_mutable.block_number(txn, &needed_hash).await.is_none() { warn!( "node is desynced. we haven't scanned {} which should happen after {} confirms", hex::encode(&needed_hash), @@ -256,19 +144,10 @@ async fn handle_coordinator_msg( sleep(Duration::from_secs(10)).await; continue; }; - break block_number; - }; - - // While the scanner has cemented this block, that doesn't mean it's been scanned for all - // keys - // ram_scanned will return the lowest scanned block number out of all keys - // This is a safe call which fulfills the unfulfilled safety requirements from the prior call - while scanner.ram_scanned().await < block_number { - sleep(Duration::from_secs(1)).await; + break; } - // TODO: Sanity check we got an AckBlock (or this is the AckBlock) for the block in - // question + // TODO2: Sanity check we got an AckBlock (or this is the AckBlock) for the block in question /* let synced = |context: &SubstrateContext, key| -> Result<(), ()> { @@ -288,54 +167,86 @@ async fn handle_coordinator_msg( } if let Some(required) = msg.msg.required_block() { - // wait only reads from, it doesn't mutate, the scanner - wait(&substrate_mutable.scanner, &required).await; + // wait only reads from, it doesn't mutate, substrate_mutable + wait(txn, substrate_mutable, &required).await; } - // TODO: Shouldn't we create a txn here and pass it around as needed? - // The txn would ack this message ID. If we detect this message ID as handled in the DB, - // we'd move on here. Only after committing the TX would we report it as acked. + async fn activate_key( + network: &N, + substrate_mutable: &mut SubstrateMutable, + tributary_mutable: &mut TributaryMutable, + txn: &mut D::Transaction<'_>, + set: ValidatorSet, + key_pair: KeyPair, + activation_number: usize, + ) { + info!("activating {set:?}'s keys at {activation_number}"); + + let network_key = ::Curve::read_G::<&[u8]>(&mut key_pair.1.as_ref()) + .expect("Substrate finalized invalid point as a network's key"); + + if tributary_mutable.key_gen.in_set(&set) { + // See TributaryMutable's struct definition for why this block is safe + let KeyConfirmed { substrate_keys, network_keys } = + tributary_mutable.key_gen.confirm(txn, set, key_pair.clone()).await; + if set.session.0 == 0 { + tributary_mutable.substrate_signer = Some(SubstrateSigner::new(N::NETWORK, substrate_keys)); + } + tributary_mutable + .signers + .insert(key_pair.1.into(), Signer::new(network.clone(), network_keys)); + } + + substrate_mutable.add_key(txn, activation_number, network_key).await; + } match msg.msg.clone() { CoordinatorMessage::KeyGen(msg) => { - coordinator - .send(ProcessorMessage::KeyGen(tributary_mutable.key_gen.handle(txn, msg).await)) - .await; + coordinator.send(tributary_mutable.key_gen.handle(txn, msg).await).await; } CoordinatorMessage::Sign(msg) => { - tributary_mutable.signers.get_mut(msg.key()).unwrap().handle(txn, msg).await; + tributary_mutable + .signers + .get_mut(msg.key()) + .expect("coordinator told us to sign with a signer we don't have") + .handle(txn, msg) + .await; } CoordinatorMessage::Coordinator(msg) => { - if let Some(substrate_signer) = tributary_mutable.substrate_signer.as_mut() { - substrate_signer.handle(txn, msg).await; - } + tributary_mutable + .substrate_signer + .as_mut() + .expect( + "coordinator told us to sign a batch when we don't have a Substrate signer at this time", + ) + .handle(txn, msg) + .await; } CoordinatorMessage::Substrate(msg) => { match msg { messages::substrate::CoordinatorMessage::ConfirmKeyPair { context, set, key_pair } => { // This is the first key pair for this network so no block has been finalized yet - let activation_number = if context.network_latest_finalized_block.0 == [0; 32] { + // TODO: Write documentation for this in docs/ + // TODO: Use an Option instead of a magic? + if context.network_latest_finalized_block.0 == [0; 32] { assert!(tributary_mutable.signers.is_empty()); assert!(tributary_mutable.substrate_signer.is_none()); - assert!(substrate_mutable.schedulers.is_empty()); + // We can't check this as existing is no longer pub + // assert!(substrate_mutable.existing.as_ref().is_none()); // Wait until a network's block's time exceeds Serai's time - // TODO: This assumes the network has a monotonic clock for its blocks' times, which - // isn't a viable assumption // If the latest block number is 10, then the block indexed by 1 has 10 confirms // 10 + 1 - 10 = 1 - while get_block( - network, - (get_latest_block_number(network).await + 1).saturating_sub(N::CONFIRMATIONS), - ) - .await - .time() < - context.serai_time - { + let mut block_i; + while { + block_i = + (get_latest_block_number(network).await + 1).saturating_sub(N::CONFIRMATIONS); + get_block(network, block_i).await.time() < context.serai_time + } { info!( "serai confirmed the first key pair for a set. {} {}", "we're waiting for a network's finalized block's time to exceed unix time ", @@ -345,9 +256,7 @@ async fn handle_coordinator_msg( } // Find the first block to do so - let mut earliest = - (get_latest_block_number(network).await + 1).saturating_sub(N::CONFIRMATIONS); - assert!(get_block(network, earliest).await.time() >= context.serai_time); + let mut earliest = block_i; // earliest > 0 prevents a panic if Serai creates keys before the genesis block // which... should be impossible // Yet a prevented panic is a prevented panic @@ -358,107 +267,101 @@ async fn handle_coordinator_msg( } // Use this as the activation block - earliest + let activation_number = earliest; + + activate_key( + network, + substrate_mutable, + tributary_mutable, + txn, + set, + key_pair, + activation_number, + ) + .await; } else { - let mut activation_block = >::Id::default(); - activation_block.as_mut().copy_from_slice(&context.network_latest_finalized_block.0); - // This block_number call is safe since it unwraps - substrate_mutable - .scanner - .block_number(&activation_block) - .await - .expect("KeyConfirmed from context we haven't synced") - }; - - info!("activating {set:?}'s keys at {activation_number}"); - - // See TributaryMutable's struct definition for why this block is safe - let KeyConfirmed { substrate_keys, network_keys } = - tributary_mutable.key_gen.confirm(txn, set, key_pair).await; - // TODO2: Don't immediately set this, set it once it's active - tributary_mutable.substrate_signer = - Some(SubstrateSigner::new(N::NETWORK, substrate_keys)); - - let key = network_keys.group_key(); - - substrate_mutable.scanner.rotate_key(txn, activation_number, key).await; - substrate_mutable - .schedulers - .insert(key.to_bytes().as_ref().to_vec(), Scheduler::::new::(txn, key)); - - tributary_mutable - .signers - .insert(key.to_bytes().as_ref().to_vec(), Signer::new(network.clone(), network_keys)); + let mut block_before_queue_block = >::Id::default(); + block_before_queue_block + .as_mut() + .copy_from_slice(&context.network_latest_finalized_block.0); + // We can't set these keys for activation until we know their queue block, which we + // won't until the next Batch is confirmed + // Set this variable so when we get the next Batch event, we can handle it + MainDb::::set_pending_activation(txn, block_before_queue_block, set, key_pair); + } } messages::substrate::CoordinatorMessage::SubstrateBlock { context, network: network_id, - block, - key: key_vec, + block: substrate_block, burns, batches, } => { assert_eq!(network_id, N::NETWORK, "coordinator sent us data for another network"); - let mut block_id = >::Id::default(); - block_id.as_mut().copy_from_slice(&context.network_latest_finalized_block.0); + if let Some((block, set, key_pair)) = MainDb::::pending_activation(txn) { + // Only run if this is a Batch belonging to a distinct block + if context.network_latest_finalized_block.as_ref() != block.as_ref() { + let mut queue_block = >::Id::default(); + queue_block.as_mut().copy_from_slice(context.network_latest_finalized_block.as_ref()); + + let activation_number = substrate_mutable + .block_number(txn, &queue_block) + .await + .expect("KeyConfirmed from context we haven't synced") + + N::CONFIRMATIONS; + + activate_key( + network, + substrate_mutable, + tributary_mutable, + txn, + set, + key_pair, + activation_number, + ) + .await; - let key = ::read_G::<&[u8]>(&mut key_vec.as_ref()).unwrap(); + MainDb::::clear_pending_activation(txn); + } + } - // We now have to acknowledge every block for this key up to the acknowledged block - let outputs = substrate_mutable.scanner.ack_up_to_block(txn, key, block_id).await; - // Since this block was acknowledged, we no longer have to sign the batch for it + // Since this block was acknowledged, we no longer have to sign the batches for it if let Some(substrate_signer) = tributary_mutable.substrate_signer.as_mut() { for batch_id in batches { substrate_signer.batch_signed(txn, batch_id); } } - let mut payments = vec![]; - for out in burns { - let OutInstructionWithBalance { - instruction: OutInstruction { address, data }, - balance, - } = out; - assert_eq!(balance.coin.network(), N::NETWORK); - - if let Ok(address) = N::Address::try_from(address.consume()) { - // TODO: Add coin to payment - payments.push(Payment { - address, - data: data.map(|data| data.consume()), - amount: balance.amount.0, - }); - } - } + let (acquired_lock, to_sign) = + substrate_mutable.substrate_block(txn, network, context, burns).await; - let plans = substrate_mutable - .schedulers - .get_mut(&key_vec) - .expect("key we don't have a scheduler for acknowledged a block") - .schedule::(txn, outputs, payments); - - coordinator - .send(ProcessorMessage::Coordinator( - messages::coordinator::ProcessorMessage::SubstrateBlockAck { + // Send SubstrateBlockAck, with relevant plan IDs, before we trigger the signing of these + // plans + if !tributary_mutable.signers.is_empty() { + coordinator + .send(messages::coordinator::ProcessorMessage::SubstrateBlockAck { network: N::NETWORK, - block, - plans: plans.iter().map(|plan| plan.id()).collect(), - }, - )) - .await; + block: substrate_block, + plans: to_sign.iter().map(|signable| signable.1).collect(), + }) + .await; + } + + // See commentary in TributaryMutable for why this is safe + let signers = &mut tributary_mutable.signers; + for (key, id, tx, eventuality) in to_sign { + if let Some(signer) = signers.get_mut(key.to_bytes().as_ref()) { + signer.sign_transaction(txn, id, tx, eventuality).await; + } + } - sign_plans( - txn, - network, - substrate_mutable, - // See commentary in TributaryMutable for why this is safe - &mut tributary_mutable.signers, - context, - plans, - ) - .await; + // This is not premature, even if this block had multiple `Batch`s created, as the first + // `Batch` alone will trigger all Plans/Eventualities/Signs + if acquired_lock { + substrate_mutable.release_scanner_lock().await; + } } } } @@ -502,63 +405,58 @@ async fn boot( // We don't need to re-issue GenerateKey orders because the coordinator is expected to // schedule/notify us of new attempts + // TODO: Is this above comment still true? Not at all due to the planned lack of DKG timeouts? let key_gen = KeyGen::::new(raw_db.clone(), entropy(b"key-gen_entropy")); - // The scanner has no long-standing orders to re-issue - let (mut scanner, active_keys) = Scanner::new(network.clone(), raw_db.clone()); - let mut schedulers = HashMap::, Scheduler>::new(); + let (multisig_manager, current_keys, actively_signing) = + MultisigManager::new(raw_db, network).await; + let mut substrate_signer = None; let mut signers = HashMap::new(); - let main_db = MainDb::new(raw_db.clone()); + let main_db = MainDb::::new(raw_db.clone()); - for key in &active_keys { - schedulers.insert(key.to_bytes().as_ref().to_vec(), Scheduler::from_db(raw_db, *key).unwrap()); - - let (substrate_keys, network_keys) = key_gen.keys(key); + for (i, key) in current_keys.iter().enumerate() { + let Some((substrate_keys, network_keys)) = key_gen.keys(key) else { continue }; + let network_key = network_keys.group_key(); + // If this is the oldest key, load the SubstrateSigner for it as the active SubstrateSigner + // The new key only takes responsibility once the old key is fully deprecated + // // We don't have to load any state for this since the Scanner will re-fire any events - // necessary - // TODO2: This uses most recent as signer, use the active one - substrate_signer = Some(SubstrateSigner::new(N::NETWORK, substrate_keys)); + // necessary, only no longer scanning old blocks once Substrate acks them + if i == 0 { + substrate_signer = Some(SubstrateSigner::new(N::NETWORK, substrate_keys)); + } + // The Scanner re-fires events as needed for substrate_signer yet not signer + // This is due to the transactions which we start signing from due to a block not being + // guaranteed to be signed before we stop scanning the block on reboot + // We could simplify the Signer flow by delaying when it acks a block, yet that'd: + // 1) Increase the startup time + // 2) Cause re-emission of Batch events, which we'd need to check the safety of + // (TODO: Do anyways?) + // 3) Violate the attempt counter (TODO: Is this already being violated?) let mut signer = Signer::new(network.clone(), network_keys); - // Load any TXs being actively signed + // Sign any TXs being actively signed let key = key.to_bytes(); - for (block_number, plan) in main_db.signing(key.as_ref()) { - let block_number = block_number.try_into().unwrap(); - - let fee = get_fee(network, block_number).await; - - let id = plan.id(); - info!("reloading plan {}: {:?}", hex::encode(id), plan); - - let (Some((tx, eventuality)), _) = - prepare_send(network, signer.keys(), block_number, fee, plan).await - else { - panic!("previously created transaction is no longer being created") - }; - - scanner.register_eventuality(block_number, id, eventuality.clone()).await; - // TODO: Reconsider if the Signer should have the eventuality, or if just the network/scanner - // should - let mut txn = raw_db.txn(); - signer.sign_transaction(&mut txn, id, tx, eventuality).await; - // This should only have re-writes of existing data - drop(txn); + for (plan, tx, eventuality) in &actively_signing { + if plan.key == network_key { + let mut txn = raw_db.txn(); + signer.sign_transaction(&mut txn, plan.id(), tx.clone(), eventuality.clone()).await; + // This should only have re-writes of existing data + drop(txn); + } } signers.insert(key.as_ref().to_vec(), signer); } - ( - main_db, - TributaryMutable { key_gen, substrate_signer, signers }, - SubstrateMutable { scanner, schedulers }, - ) + (main_db, TributaryMutable { key_gen, substrate_signer, signers }, multisig_manager) } +#[allow(clippy::await_holding_lock)] // Needed for txn, unfortunately can't be down-scoped async fn run(mut raw_db: D, network: N, mut coordinator: Co) { // We currently expect a contextless bidirectional mapping between these two values // (which is that any value of A can be interpreted as B and vice versa) @@ -566,59 +464,18 @@ async fn run(mut raw_db: D, network: N, mut // This check ensures no network which doesn't have a bidirectional mapping is defined assert_eq!(>::Id::default().as_ref().len(), BlockHash([0u8; 32]).0.len()); - let (mut main_db, mut tributary_mutable, mut substrate_mutable) = - boot(&mut raw_db, &network).await; + let (main_db, mut tributary_mutable, mut substrate_mutable) = boot(&mut raw_db, &network).await; // We can't load this from the DB as we can't guarantee atomic increments with the ack function + // TODO: Load with a slight tolerance let mut last_coordinator_msg = None; loop { - // Check if the signers have events - // The signers will only have events after the following select executes, which will then - // trigger the loop again, hence why having the code here with no timer is fine - for (key, signer) in tributary_mutable.signers.iter_mut() { - while let Some(msg) = signer.events.pop_front() { - match msg { - SignerEvent::ProcessorMessage(msg) => { - coordinator.send(ProcessorMessage::Sign(msg)).await; - } + // The following select uses this txn in both branches, hence why needing a RwLock to pass it + // around is needed + let txn = RwLock::new(raw_db.txn()); - SignerEvent::SignedTransaction { id, tx } => { - coordinator - .send(ProcessorMessage::Sign(messages::sign::ProcessorMessage::Completed { - key: key.clone(), - id, - tx: tx.as_ref().to_vec(), - })) - .await; - - let mut txn = raw_db.txn(); - // This does mutate the Scanner, yet the eventuality protocol is only run to mutate - // the signer, which is Tributary mutable (and what's currently being mutated) - substrate_mutable.scanner.drop_eventuality(id).await; - main_db.finish_signing(&mut txn, key, id); - txn.commit(); - } - } - } - } - - if let Some(signer) = tributary_mutable.substrate_signer.as_mut() { - while let Some(msg) = signer.events.pop_front() { - match msg { - SubstrateSignerEvent::ProcessorMessage(msg) => { - coordinator.send(ProcessorMessage::Coordinator(msg)).await; - } - SubstrateSignerEvent::SignedBatch(batch) => { - coordinator - .send(ProcessorMessage::Substrate(messages::substrate::ProcessorMessage::Update { - batch, - })) - .await; - } - } - } - } + let mut outer_msg = None; tokio::select! { // This blocks the entire processor until it finishes handling this message @@ -627,13 +484,15 @@ async fn run(mut raw_db: D, network: N, mut // the other messages in the queue, it may be beneficial to parallelize these // They could likely be parallelized by type (KeyGen, Sign, Substrate) without issue msg = coordinator.recv() => { + let mut txn = txn.write().unwrap(); + let txn = &mut txn; + assert_eq!(msg.id, (last_coordinator_msg.unwrap_or(msg.id - 1) + 1)); last_coordinator_msg = Some(msg.id); // Only handle this if we haven't already if !main_db.handled_message(msg.id) { - let mut txn = raw_db.txn(); - MainDb::::handle_message(&mut txn, msg.id); + MainDb::::handle_message(txn, msg.id); // This is isolated to better think about how its ordered, or rather, about how the other // cases aren't ordered @@ -646,111 +505,98 @@ async fn run(mut raw_db: D, network: N, mut // This is safe so long as Tributary and Substrate messages don't both expect mutable // references over the same data handle_coordinator_msg( - &mut txn, + &mut **txn, &network, &mut coordinator, &mut tributary_mutable, &mut substrate_mutable, &msg, ).await; - - txn.commit(); } - coordinator.ack(msg).await; + outer_msg = Some(msg); }, - msg = substrate_mutable.scanner.events.recv() => { - let mut txn = raw_db.txn(); - - match msg.unwrap() { - ScannerEvent::Block { block, outputs } => { - let mut block_hash = [0; 32]; - block_hash.copy_from_slice(block.as_ref()); - // TODO: Move this out from Scanner now that the Scanner no longer handles batches - let mut batch_id = substrate_mutable.scanner.next_batch_id(&txn); - - // start with empty batch - let mut batches = vec![Batch { - network: N::NETWORK, - id: batch_id, - block: BlockHash(block_hash), - instructions: vec![], - }]; - for output in outputs { - // If these aren't externally received funds, don't handle it as an instruction - if output.kind() != OutputType::External { - continue; - } - - let mut data = output.data(); - let max_data_len = usize::try_from(MAX_DATA_LEN).unwrap(); - // TODO: Refund if we hit one of the following continues - if data.len() > max_data_len { - error!( - "data in output {} exceeded MAX_DATA_LEN ({MAX_DATA_LEN}): {}. skipping", - hex::encode(output.id()), - data.len(), - ); - continue; - } - - let Ok(shorthand) = Shorthand::decode(&mut data) else { continue }; - let Ok(instruction) = RefundableInInstruction::try_from(shorthand) else { continue }; - - // TODO2: Set instruction.origin if not set (and handle refunds in general) - let instruction = InInstructionWithBalance { - instruction: instruction.instruction, - balance: output.balance(), - }; - - let batch = batches.last_mut().unwrap(); - batch.instructions.push(instruction); - - // check if batch is over-size - if batch.encode().len() > MAX_BATCH_SIZE { - // pop the last instruction so it's back in size - let instruction = batch.instructions.pop().unwrap(); - - // bump the id for the new batch - batch_id += 1; - - // make a new batch with this instruction included - batches.push(Batch { - network: N::NETWORK, - id: batch_id, - block: BlockHash(block_hash), - instructions: vec![instruction], - }); - } - } - - // Save the next batch ID - substrate_mutable.scanner.set_next_batch_id(&mut txn, batch_id + 1); - + msg = substrate_mutable.next_event(&txn) => { + let mut txn = txn.write().unwrap(); + let txn = &mut txn; + match msg { + MultisigEvent::Batches(retired_key_new_key, batches) => { // Start signing this batch for batch in batches { info!("created batch {} ({} instructions)", batch.id, batch.instructions.len()); + coordinator.send( + messages::substrate::ProcessorMessage::Batch { batch: batch.clone() } + ).await; + if let Some(substrate_signer) = tributary_mutable.substrate_signer.as_mut() { - substrate_signer - .sign(&mut txn, batch) - .await; + substrate_signer.sign(txn, batch).await; } } - }, - ScannerEvent::Completed(id, tx) => { - // We don't know which signer had this plan, so inform all of them - for (_, signer) in tributary_mutable.signers.iter_mut() { - signer.eventuality_completion(&mut txn, id, &tx).await; + if let Some((retired_key, new_key)) = retired_key_new_key { + // Safe to mutate since all signing operations are done and no more will be added + tributary_mutable.signers.remove(retired_key.to_bytes().as_ref()); + tributary_mutable.substrate_signer.take(); + if let Some((substrate_keys, _)) = tributary_mutable.key_gen.keys(&new_key) { + tributary_mutable.substrate_signer = + Some(SubstrateSigner::new(N::NETWORK, substrate_keys)); + } } }, + MultisigEvent::Completed(key, id, tx) => { + if let Some(signer) = tributary_mutable.signers.get_mut(&key) { + signer.completed(txn, id, tx); + } + } } - - txn.commit(); }, } + + // Check if the signers have events + // The signers will only have events after the above select executes, so having no timeout on + // the above is fine + // TODO: Have the Signers return these events, allowing removing these channels? + for (key, signer) in tributary_mutable.signers.iter_mut() { + while let Some(msg) = signer.events.pop_front() { + match msg { + SignerEvent::ProcessorMessage(msg) => { + coordinator.send(msg).await; + } + + SignerEvent::SignedTransaction { id, tx } => { + // It is important ProcessorMessage::Completed is only emitted if a Signer we had + // created the TX completed (which having it only emitted after a SignerEvent ensures) + coordinator + .send(messages::sign::ProcessorMessage::Completed { + key: key.clone(), + id, + tx: tx.as_ref().to_vec(), + }) + .await; + } + } + } + } + + if let Some(signer) = tributary_mutable.substrate_signer.as_mut() { + while let Some(msg) = signer.events.pop_front() { + match msg { + SubstrateSignerEvent::ProcessorMessage(msg) => { + coordinator.send(msg).await; + } + SubstrateSignerEvent::SignedBatch(batch) => { + coordinator.send(messages::substrate::ProcessorMessage::SignedBatch { batch }).await; + } + } + } + } + + txn.into_inner().unwrap().commit(); + if let Some(msg) = outer_msg { + coordinator.ack(msg).await; + } } } diff --git a/processor/src/multisigs/db.rs b/processor/src/multisigs/db.rs new file mode 100644 index 000000000..353bc4f81 --- /dev/null +++ b/processor/src/multisigs/db.rs @@ -0,0 +1,189 @@ +use core::marker::PhantomData; + +use ciphersuite::Ciphersuite; + +pub use serai_db::*; + +use scale::{Encode, Decode}; +use serai_client::in_instructions::primitives::InInstructionWithBalance; + +use crate::{ + Get, Db, Plan, + networks::{Transaction, Network}, +}; + +#[derive(Debug)] +pub struct MultisigsDb(PhantomData, PhantomData); +impl MultisigsDb { + fn multisigs_key(dst: &'static [u8], key: impl AsRef<[u8]>) -> Vec { + D::key(b"MULTISIGS", dst, key) + } + + fn next_batch_key() -> Vec { + Self::multisigs_key(b"next_batch", []) + } + // Set the next batch ID to use + pub fn set_next_batch_id(txn: &mut D::Transaction<'_>, batch: u32) { + txn.put(Self::next_batch_key(), batch.to_le_bytes()); + } + // Get the next batch ID + pub fn next_batch_id(getter: &G) -> u32 { + getter.get(Self::next_batch_key()).map_or(0, |v| u32::from_le_bytes(v.try_into().unwrap())) + } + + fn plan_key(id: &[u8]) -> Vec { + Self::multisigs_key(b"plan", id) + } + fn resolved_key(tx: &[u8]) -> Vec { + Self::multisigs_key(b"resolved", tx) + } + fn signing_key(key: &[u8]) -> Vec { + Self::multisigs_key(b"signing", key) + } + pub fn save_active_plan( + txn: &mut D::Transaction<'_>, + key: &[u8], + block_number: u64, + plan: &Plan, + ) { + let id = plan.id(); + + { + let mut signing = txn.get(Self::signing_key(key)).unwrap_or(vec![]); + + // If we've already noted we're signing this, return + assert_eq!(signing.len() % 32, 0); + for i in 0 .. (signing.len() / 32) { + if signing[(i * 32) .. ((i + 1) * 32)] == id { + return; + } + } + + signing.extend(&id); + txn.put(Self::signing_key(key), id); + } + + { + let mut buf = block_number.to_le_bytes().to_vec(); + plan.write(&mut buf).unwrap(); + txn.put(Self::plan_key(&id), &buf); + } + } + + pub fn active_plans(getter: &G, key: &[u8]) -> Vec<(u64, Plan)> { + let signing = getter.get(Self::signing_key(key)).unwrap_or(vec![]); + let mut res = vec![]; + + assert_eq!(signing.len() % 32, 0); + for i in 0 .. (signing.len() / 32) { + let id = &signing[(i * 32) .. ((i + 1) * 32)]; + let buf = getter.get(Self::plan_key(id)).unwrap(); + + let block_number = u64::from_le_bytes(buf[.. 8].try_into().unwrap()); + let plan = Plan::::read::<&[u8]>(&mut &buf[8 ..]).unwrap(); + assert_eq!(id, &plan.id()); + res.push((block_number, plan)); + } + + res + } + + pub fn resolved_plan( + getter: &G, + tx: >::Id, + ) -> Option<[u8; 32]> { + getter.get(tx.as_ref()).map(|id| id.try_into().unwrap()) + } + pub fn plan_by_key_with_self_change( + getter: &G, + key: ::G, + id: [u8; 32], + ) -> bool { + let plan = + Plan::::read::<&[u8]>(&mut &getter.get(Self::plan_key(&id)).unwrap()[8 ..]).unwrap(); + assert_eq!(plan.id(), id); + (key == plan.key) && (Some(N::change_address(plan.key)) == plan.change) + } + pub fn resolve_plan( + txn: &mut D::Transaction<'_>, + key: &[u8], + plan: [u8; 32], + resolution: >::Id, + ) { + let mut signing = txn.get(Self::signing_key(key)).unwrap_or(vec![]); + assert_eq!(signing.len() % 32, 0); + + let mut found = false; + for i in 0 .. (signing.len() / 32) { + let start = i * 32; + let end = i + 32; + if signing[start .. end] == plan { + found = true; + signing = [&signing[.. start], &signing[end ..]].concat().to_vec(); + break; + } + } + + if !found { + log::warn!("told to finish signing {} yet wasn't actively signing it", hex::encode(plan)); + } + + txn.put(Self::signing_key(key), signing); + + txn.put(Self::resolved_key(resolution.as_ref()), plan); + } + + fn forwarded_output_key(amount: u64) -> Vec { + Self::multisigs_key(b"forwarded_output", amount.to_le_bytes()) + } + pub fn save_forwarded_output( + txn: &mut D::Transaction<'_>, + instruction: InInstructionWithBalance, + ) { + let key = Self::forwarded_output_key(instruction.balance.amount.0); + let mut existing = txn.get(&key).unwrap_or(vec![]); + existing.extend(instruction.encode()); + txn.put(key, existing); + } + pub fn take_forwarded_output( + txn: &mut D::Transaction<'_>, + amount: u64, + ) -> Option { + let key = Self::forwarded_output_key(amount); + + let outputs = txn.get(&key)?; + let mut outputs_ref = outputs.as_slice(); + + let res = InInstructionWithBalance::decode(&mut outputs_ref).unwrap(); + assert!(outputs_ref.len() < outputs.len()); + if outputs_ref.is_empty() { + txn.del(&key); + } else { + txn.put(&key, outputs_ref); + } + Some(res) + } + + fn delayed_output_keys() -> Vec { + Self::multisigs_key(b"delayed_outputs", []) + } + pub fn save_delayed_output(txn: &mut D::Transaction<'_>, instruction: InInstructionWithBalance) { + let key = Self::delayed_output_keys(); + let mut existing = txn.get(&key).unwrap_or(vec![]); + existing.extend(instruction.encode()); + txn.put(key, existing); + } + pub fn take_delayed_outputs(txn: &mut D::Transaction<'_>) -> Vec { + let key = Self::delayed_output_keys(); + + let Some(outputs) = txn.get(&key) else { return vec![] }; + txn.del(key); + + let mut outputs_ref = outputs.as_slice(); + let mut res = vec![]; + while !outputs_ref.is_empty() { + res.push(InInstructionWithBalance::decode(&mut outputs_ref).unwrap()); + } + res + } +} diff --git a/processor/src/multisigs/mod.rs b/processor/src/multisigs/mod.rs new file mode 100644 index 000000000..968e094c9 --- /dev/null +++ b/processor/src/multisigs/mod.rs @@ -0,0 +1,927 @@ +use core::time::Duration; +use std::{sync::RwLock, collections::HashMap}; + +use ciphersuite::{group::GroupEncoding, Ciphersuite}; + +use scale::{Encode, Decode}; +use messages::SubstrateContext; + +use serai_client::{ + primitives::{BlockHash, MAX_DATA_LEN}, + in_instructions::primitives::{ + InInstructionWithBalance, Batch, RefundableInInstruction, Shorthand, MAX_BATCH_SIZE, + }, + tokens::primitives::{OutInstruction, OutInstructionWithBalance}, +}; + +use log::{info, error}; + +use tokio::time::sleep; + +#[cfg(not(test))] +mod scanner; +#[cfg(test)] +pub mod scanner; + +use scanner::{ScannerEvent, ScannerHandle, Scanner}; + +mod db; +use db::MultisigsDb; + +#[cfg(not(test))] +mod scheduler; +#[cfg(test)] +pub mod scheduler; +use scheduler::Scheduler; + +use crate::{ + Get, Db, Payment, PostFeeBranch, Plan, + networks::{OutputType, Output, Transaction, SignableTransaction, Block, Network, get_block}, +}; + +// InInstructionWithBalance from an external output +fn instruction_from_output(output: &N::Output) -> Option { + assert_eq!(output.kind(), OutputType::External); + + let mut data = output.data(); + let max_data_len = usize::try_from(MAX_DATA_LEN).unwrap(); + if data.len() > max_data_len { + error!( + "data in output {} exceeded MAX_DATA_LEN ({MAX_DATA_LEN}): {}. skipping", + hex::encode(output.id()), + data.len(), + ); + None?; + } + + let Ok(shorthand) = Shorthand::decode(&mut data) else { None? }; + let Ok(instruction) = RefundableInInstruction::try_from(shorthand) else { None? }; + + // TODO2: Set instruction.origin if not set (and handle refunds in general) + Some(InInstructionWithBalance { instruction: instruction.instruction, balance: output.balance() }) +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum RotationStep { + // Use the existing multisig for all actions (steps 1-3) + UseExisting, + // Use the new multisig as change (step 4) + NewAsChange, + // The existing multisig is expected to solely forward transactions at this point (step 5) + ForwardFromExisting, + // The existing multisig is expected to finish its own transactions and do nothing more + // (step 6) + ClosingExisting, +} + +async fn get_fee(network: &N, block_number: usize) -> N::Fee { + // TODO2: Use an fee representative of several blocks + get_block(network, block_number).await.median_fee() +} + +async fn prepare_send( + network: &N, + block_number: usize, + fee: N::Fee, + plan: Plan, +) -> (Option<(N::SignableTransaction, N::Eventuality)>, Vec) { + loop { + match network.prepare_send(block_number, plan.clone(), fee).await { + Ok(prepared) => { + return prepared; + } + Err(e) => { + error!("couldn't prepare a send for plan {}: {e}", hex::encode(plan.id())); + // The processor is either trying to create an invalid TX (fatal) or the node went + // offline + // The former requires a patch, the latter is a connection issue + // If the latter, this is an appropriate sleep. If the former, we should panic, yet + // this won't flood the console ad infinitum + sleep(Duration::from_secs(60)).await; + } + } + } +} + +pub struct MultisigViewer { + activation_block: usize, + key: ::G, + scheduler: Scheduler, +} + +#[allow(clippy::type_complexity)] +#[derive(Clone, Debug)] +pub enum MultisigEvent { + // Batches to publish + Batches(Option<(::G, ::G)>, Vec), + // Eventuality completion found on-chain + Completed(Vec, [u8; 32], N::Transaction), +} + +pub struct MultisigManager { + scanner: ScannerHandle, + existing: Option>, + new: Option>, +} + +impl MultisigManager { + pub async fn new( + raw_db: &D, + network: &N, + ) -> ( + Self, + Vec<::G>, + Vec<(Plan, N::SignableTransaction, N::Eventuality)>, + ) { + // The scanner has no long-standing orders to re-issue + let (mut scanner, current_keys) = Scanner::new(network.clone(), raw_db.clone()); + + let mut schedulers = vec![]; + + assert!(current_keys.len() <= 2); + let mut actively_signing = vec![]; + for (_, key) in ¤t_keys { + schedulers.push(Scheduler::from_db(raw_db, *key).unwrap()); + + // Load any TXs being actively signed + let key = key.to_bytes(); + for (block_number, plan) in MultisigsDb::::active_plans(raw_db, key.as_ref()) { + let block_number = block_number.try_into().unwrap(); + + let fee = get_fee(network, block_number).await; + + let id = plan.id(); + info!("reloading plan {}: {:?}", hex::encode(id), plan); + + let key_bytes = plan.key.to_bytes(); + + let (Some((tx, eventuality)), _) = + prepare_send(network, block_number, fee, plan.clone()).await + else { + panic!("previously created transaction is no longer being created") + }; + + scanner + .register_eventuality(key_bytes.as_ref(), block_number, id, eventuality.clone()) + .await; + actively_signing.push((plan, tx, eventuality)); + } + } + + ( + MultisigManager { + scanner, + existing: current_keys.get(0).cloned().map(|(activation_block, key)| MultisigViewer { + activation_block, + key, + scheduler: schedulers.remove(0), + }), + new: current_keys.get(1).cloned().map(|(activation_block, key)| MultisigViewer { + activation_block, + key, + scheduler: schedulers.remove(0), + }), + }, + current_keys.into_iter().map(|(_, key)| key).collect(), + actively_signing, + ) + } + + /// Returns the block number for a block hash, if it's known and all keys have scanned the block. + // This is guaranteed to atomically increment so long as no new keys are added to the scanner + // which activate at a block before the currently highest scanned block. This is prevented by + // the processor waiting for `Batch` inclusion before scanning too far ahead, and activation only + // happening after the "too far ahead" window. + pub async fn block_number( + &self, + getter: &G, + hash: &>::Id, + ) -> Option { + let latest = ScannerHandle::::block_number(getter, hash)?; + + // While the scanner has cemented this block, that doesn't mean it's been scanned for all + // keys + // ram_scanned will return the lowest scanned block number out of all keys + if latest > self.scanner.ram_scanned().await { + return None; + } + Some(latest) + } + + pub async fn add_key( + &mut self, + txn: &mut D::Transaction<'_>, + activation_block: usize, + external_key: ::G, + ) { + self.scanner.register_key(txn, activation_block, external_key).await; + let viewer = Some(MultisigViewer { + activation_block, + key: external_key, + scheduler: Scheduler::::new::(txn, external_key), + }); + + if self.existing.is_none() { + self.existing = viewer; + return; + } + self.new = viewer; + } + + fn current_rotation_step(&self, block_number: usize) -> RotationStep { + fn ceil_div(num: usize, denom: usize) -> usize { + let res = num / denom; + if (res * denom) == num { + return res; + } + res + 1 + } + + let Some(new) = self.new.as_ref() else { return RotationStep::UseExisting }; + + // Period numbering here has no meaning other than these the time values useful here, and the + // order they're built in. They have no reference/shared marker with anything else + + // ESTIMATED_BLOCK_TIME_IN_SECONDS is fine to use here. While inaccurate, it shouldn't be + // drastically off, and even if it is, it's a hiccup to latency handling only possible when + // rotating. The error rate wouldn't be acceptable if it was allowed to accumulate over time, + // yet rotation occurs on Serai's clock, disconnecting any errors here from any prior. + + // N::CONFIRMATIONS + 10 minutes + let period_1_start = new.activation_block + + N::CONFIRMATIONS + + ceil_div(10 * 60, N::ESTIMATED_BLOCK_TIME_IN_SECONDS); + + // N::CONFIRMATIONS + let period_2_start = period_1_start + N::CONFIRMATIONS; + + // 6 hours after period 2 + // Also ensure 6 hours is greater than the amount of CONFIRMATIONS, for sanity purposes + let period_3_start = + period_2_start + ((6 * 60 * 60) / N::ESTIMATED_BLOCK_TIME_IN_SECONDS).max(N::CONFIRMATIONS); + + if block_number < period_1_start { + RotationStep::UseExisting + } else if block_number < period_2_start { + RotationStep::NewAsChange + } else if block_number < period_3_start { + RotationStep::ForwardFromExisting + } else { + RotationStep::ClosingExisting + } + } + + // Convert new Burns to Payments. + // + // Also moves payments from the old Scheduler to the new multisig if the step calls for it. + fn burns_to_payments( + &mut self, + txn: &mut D::Transaction<'_>, + step: RotationStep, + burns: Vec, + ) -> (Vec>, Vec>) { + let mut payments = vec![]; + for out in burns { + let OutInstructionWithBalance { instruction: OutInstruction { address, data }, balance } = + out; + assert_eq!(balance.coin.network(), N::NETWORK); + + if let Ok(address) = N::Address::try_from(address.consume()) { + // TODO: Add coin to payment + payments.push(Payment { + address, + data: data.map(|data| data.consume()), + amount: balance.amount.0, + }); + } + } + + let payments = payments; + match step { + RotationStep::UseExisting | RotationStep::NewAsChange => (payments, vec![]), + RotationStep::ForwardFromExisting | RotationStep::ClosingExisting => { + // Consume any payments the prior scheduler was unable to complete + // This should only actually matter once + let mut new_payments = self.existing.as_mut().unwrap().scheduler.consume_payments::(txn); + // Add the new payments + new_payments.extend(payments); + (vec![], new_payments) + } + } + } + + fn split_outputs_by_key(&self, outputs: Vec) -> (Vec, Vec) { + let mut existing_outputs = Vec::with_capacity(outputs.len()); + let mut new_outputs = vec![]; + + let existing_key = self.existing.as_ref().unwrap().key; + let new_key = self.new.as_ref().map(|new| new.key); + for output in outputs { + if output.key() == existing_key { + existing_outputs.push(output); + } else { + assert_eq!(Some(output.key()), new_key); + new_outputs.push(output); + } + } + + (existing_outputs, new_outputs) + } + + // Manually creates Plans for all External outputs needing forwarding/refunding. + // + // Returns created Plans and a map of forwarded output IDs to their associated InInstructions. + fn filter_outputs_due_to_forwarding( + &self, + existing_outputs: &mut Vec, + ) -> (Vec>, HashMap, InInstructionWithBalance>) { + // Manually create a Plan for all External outputs needing forwarding/refunding + + /* + Sending a Plan, with arbitrary data proxying the InInstruction, would require adding + a flow for networks which drop their data to still embed arbitrary data. It'd also have + edge cases causing failures. + + Instead, we save the InInstruction as we scan this output. Then, when the output is + successfully forwarded, we simply read it from the local database. This also saves the + costs of embedding arbitrary data. + + Since we can't rely on the Eventuality system to detect if it's a forwarded transaction, + due to the asynchonicity of the Eventuality system, we instead interpret an External + output with no InInstruction, which has an amount associated with an InInstruction + being forwarded, as having been forwarded. This does create a specific edge case where + a user who doesn't include an InInstruction may not be refunded however, if they share + an exact amount with an expected-to-be-forwarded transaction. This is deemed acceptable. + + TODO: Add a fourth address, forwarded_address, to prevent this. + */ + + let mut plans = vec![]; + let mut forwarding = HashMap::new(); + existing_outputs.retain(|output| { + if output.kind() == OutputType::External { + if let Some(instruction) = instruction_from_output::(output) { + // Build a dedicated Plan forwarding this + plans.push(Plan { + key: self.existing.as_ref().unwrap().key, + inputs: vec![output.clone()], + payments: vec![], + change: Some(N::address(self.new.as_ref().unwrap().key)), + }); + + // Set the instruction for this output to be returned + forwarding.insert(output.id().as_ref().to_vec(), instruction); + } + + // TODO: Refund here + false + } else { + true + } + }); + (plans, forwarding) + } + + // Filter newly received outputs due to the step being RotationStep::ClosingExisting. + fn filter_outputs_due_to_closing( + &mut self, + txn: &mut D::Transaction<'_>, + existing_outputs: &mut Vec, + ) -> Vec> { + /* + The document says to only handle outputs we created. We don't know what outputs we + created. We do have an ordered view of equivalent outputs however, and can assume the + first (and likely only) ones are the ones we created. + + Accordingly, only handling outputs we created should be definable as only handling + outputs from the resolution of Eventualities. + + This isn't feasible. It requires knowing what Eventualities were completed in this block, + when we handle this block, which we don't know without fully serialized scanning + Batch + publication. + + Take the following scenario: + 1) A network uses 10 confirmations. Block x is scanned, meaning x+9a exists. + 2) 67% of nodes process x, create, sign, and publish a TX, creating an Eventuality. + 3) A reorganization to a shorter chain occurs, including the published TX in x+1b. + 4) The 33% of nodes which are latent will be allowed to scan x+1b as soon as x+10b + exists. They won't wait for Serai to include the Batch for x until they try to scan + x+10b. + 5) These latent nodes will handle x+1b, post-create an Eventuality, post-learn x+1b + contained resolutions, changing how x+1b should've been interpreted. + + We either have to: + A) Fully serialize scanning (removing the ability to utilize throughput to allow higher + latency, at least while the step is `ClosingExisting`). + B) Create Eventualities immediately, which we can't do as then both the external + network's clock AND Serai's clock can trigger Eventualities, removing ordering. + We'd need to shift entirely to the external network's clock, only handling Burns + outside the parallelization window (which would be extremely latent). + C) Use a different mechanism to determine if we created an output. + D) Re-define which outputs are still to be handled after the 6 hour period expires, such + that the multisig's lifetime cannot be further extended yet it does fulfill its + responsibility. + + External outputs to the existing multisig will be: + - Scanned before the rotation and unused (as used External outputs become Change) + - Forwarded immediately upon scanning + - Not scanned before the cut off time (and accordingly dropped) + + For the first case, since they're scanned before the rotation and unused, they'll be + forwarded with all other available outputs (since they'll be available when scanned). + + Change outputs will be: + - Scanned before the rotation and forwarded with all other available outputs + - Forwarded immediately upon scanning + - Not scanned before the cut off time, requiring an extension exclusive to these outputs + + The important thing to note about honest Change outputs to the existing multisig is that + they'll only be created within `CONFIRMATIONS+1` blocks of the activation block. Also + important to note is that there's another explicit window of `CONFIRMATIONS` before the + 6 hour window. + + Eventualities are not guaranteed to be known before we scan the block containing their + resolution. They are guaranteed to be known within `CONFIRMATIONS-1` blocks however, due + to the limitation on how far we'll scan ahead. + + This means we will know of all Eventualities related to Change outputs we need to forward + before the 6 hour period begins (as forwarding outputs will not create any Change outputs + to the existing multisig). + + This means a definition of complete can be defined as: + 1) Handled all Branch outputs + 2) Forwarded all External outputs received before the end of 6 hour window + 3) Forwarded the results of all Eventualities with Change, which will have been created + before the 6 hour window + + How can we track and ensure this without needing to check if an output is from the + resolution of an Eventuality? + + 1) We only create Branch outputs before the 6 hour window starts. These are guaranteed + to appear within `CONFIRMATIONS` blocks. They will exist with arbitrary depth however, + meaning that upon completion they will spawn several more Eventualities. The further + created Eventualities re-risk being present after the 6 hour period ends. + + We can: + 1) Build a queue for Branch outputs, delaying their handling until relevant + Eventualities are guaranteed to be present. + + This solution would theoretically work for all outputs and allow collapsing this + problem to simply: + + > Accordingly, only handling outputs we created should be definable as only + handling outputs from the resolution of Eventualities. + + 2) Create all Eventualities under a Branch at time of Branch creation. + This idea fails as Plans are tightly bound to outputs. + + 3) Don't track Branch outputs by Eventualities, yet by the amount of Branch outputs + remaining. Any Branch output received, of a useful amount, is assumed to be our + own and handled. All other Branch outputs, even if they're the completion of some + Eventuality, are dropped. + + This avoids needing any additional queue, avoiding additional pipelining/latency. + + 2) External outputs are self-evident. We simply stop handling them at the cut-off point, + and only start checking after `CONFIRMATIONS` blocks if all Eventualities are + complete. + + 3) Since all Change Eventualities will be known prior to the 6 hour window's beginning, + we can safely check if a received Change output is the resolution of an Eventuality. + We only need to forward it if so. Forwarding it simply requires only checking if + Eventualities are complete after `CONFIRMATIONS` blocks, same as for straggling + External outputs. + */ + + let mut plans = vec![]; + existing_outputs.retain(|output| { + match output.kind() { + OutputType::External => false, + OutputType::Branch => { + let scheduler = &mut self.existing.as_mut().unwrap().scheduler; + // There *would* be a race condition here due to the fact we only mark a `Branch` output + // as needed when we process the block (and handle scheduling), yet actual `Branch` + // outputs may appear as soon as the next block (and we scan the next block before we + // process the prior block) + // + // Unlike Eventuality checking, which happens on scanning and is therefore asynchronous, + // all scheduling (and this check against the scheduler) happens on processing, which is + // synchronous + // + // While we could move Eventuality checking into the block processing, removing its + // asynchonicity, we could only check data the Scanner deems important. The Scanner won't + // deem important Eventuality resolutions which don't create an output to Serai unless + // it knows of the Eventuality. Accordingly, at best we could have a split role (the + // Scanner noting completion of Eventualities which don't have relevant outputs, the + // processing noting completion of ones which do) + // + // This is unnecessary, due to the current flow around Eventuality resolutions and the + // current bounds naturally found being sufficiently amenable, yet notable for the future + if scheduler.can_use_branch(output.amount()) { + // We could simply call can_use_branch, yet it'd have an edge case where if we receive + // two outputs for 100, and we could use one such output, we'd handle both. + // + // Individually schedule each output once confirming they're usable in order to avoid + // this. + let mut plan = scheduler.schedule::( + txn, + vec![output.clone()], + vec![], + self.new.as_ref().unwrap().key, + false, + ); + assert_eq!(plan.len(), 1); + let plan = plan.remove(0); + plans.push(plan); + } + false + } + OutputType::Change => { + // If the TX containing this output resolved an Eventuality... + if let Some(plan) = MultisigsDb::::resolved_plan(txn, output.tx_id()) { + // And the Eventuality had change... + // We need this check as Eventualities have a race condition and can't be relied + // on, as extensively detailed above. Eventualities explicitly with change do have + // a safe timing window however + if MultisigsDb::::plan_by_key_with_self_change( + txn, + // Pass the key so the DB checks the Plan's key is this multisig's, preventing a + // potential issue where the new multisig creates a Plan with change *and a + // payment to the existing multisig's change address* + self.existing.as_ref().unwrap().key, + plan, + ) { + // Then this is an honest change output we need to forward + // (or it's a payment to the change address in the same transaction as an honest + // change output, which is fine to let slip in) + return true; + } + } + false + } + } + }); + plans + } + + // Returns the Plans caused from a block being acknowledged. + // + // Will rotate keys if the block acknowledged is the retirement block. + async fn plans_from_block( + &mut self, + txn: &mut D::Transaction<'_>, + block_number: usize, + block_id: >::Id, + step: &mut RotationStep, + burns: Vec, + ) -> (bool, Vec>, HashMap, InInstructionWithBalance>) { + let (mut existing_payments, mut new_payments) = self.burns_to_payments(txn, *step, burns); + + // We now have to acknowledge the acknowledged block, if it's new + // It won't be if this block's `InInstruction`s were split into multiple `Batch`s + let (acquired_lock, (mut existing_outputs, new_outputs)) = { + let (acquired_lock, outputs) = if ScannerHandle::::db_scanned(txn) + .expect("published a Batch despite never scanning a block") < + block_number + { + let (is_retirement_block, outputs) = self.scanner.ack_block(txn, block_id.clone()).await; + if is_retirement_block { + let existing = self.existing.take().unwrap(); + assert!(existing.scheduler.empty()); + self.existing = self.new.take(); + *step = RotationStep::UseExisting; + assert!(existing_payments.is_empty()); + existing_payments = new_payments; + new_payments = vec![]; + } + (true, outputs) + } else { + (false, vec![]) + }; + (acquired_lock, self.split_outputs_by_key(outputs)) + }; + + let (mut plans, forwarded_external_outputs) = match *step { + RotationStep::UseExisting | RotationStep::NewAsChange => (vec![], HashMap::new()), + RotationStep::ForwardFromExisting => { + self.filter_outputs_due_to_forwarding(&mut existing_outputs) + } + RotationStep::ClosingExisting => { + (self.filter_outputs_due_to_closing(txn, &mut existing_outputs), HashMap::new()) + } + }; + + plans.extend({ + let existing = self.existing.as_mut().unwrap(); + let existing_key = existing.key; + self.existing.as_mut().unwrap().scheduler.schedule::( + txn, + existing_outputs, + existing_payments, + match *step { + RotationStep::UseExisting => existing_key, + RotationStep::NewAsChange | + RotationStep::ForwardFromExisting | + RotationStep::ClosingExisting => self.new.as_ref().unwrap().key, + }, + match *step { + RotationStep::UseExisting | RotationStep::NewAsChange => false, + RotationStep::ForwardFromExisting | RotationStep::ClosingExisting => true, + }, + ) + }); + + for plan in &plans { + assert_eq!(plan.key, self.existing.as_ref().unwrap().key); + if plan.change == Some(N::change_address(plan.key)) { + // Assert these are only created during the expected step + match *step { + RotationStep::UseExisting => {} + RotationStep::NewAsChange | + RotationStep::ForwardFromExisting | + RotationStep::ClosingExisting => panic!("change was set to self despite rotating"), + } + } + } + + if let Some(new) = self.new.as_mut() { + plans.extend(new.scheduler.schedule::(txn, new_outputs, new_payments, new.key, false)); + } + + (acquired_lock, plans, forwarded_external_outputs) + } + + /// Handle a SubstrateBlock event, building the relevant Plans. + pub async fn substrate_block( + &mut self, + txn: &mut D::Transaction<'_>, + network: &N, + context: SubstrateContext, + burns: Vec, + ) -> (bool, Vec<(::G, [u8; 32], N::SignableTransaction, N::Eventuality)>) + { + let mut block_id = >::Id::default(); + block_id.as_mut().copy_from_slice(context.network_latest_finalized_block.as_ref()); + let block_number = ScannerHandle::::block_number(txn, &block_id) + .expect("SubstrateBlock with context we haven't synced"); + + // Determine what step of rotation we're currently in + let mut step = self.current_rotation_step(block_number); + + // Get the Plans from this block + let (acquired_lock, plans, mut forwarded_external_outputs) = + self.plans_from_block(txn, block_number, block_id, &mut step, burns).await; + + let res = { + let mut res = Vec::with_capacity(plans.len()); + let fee = get_fee(network, block_number).await; + + for plan in plans { + let id = plan.id(); + info!("preparing plan {}: {:?}", hex::encode(id), plan); + + let key = plan.key; + let key_bytes = key.to_bytes(); + MultisigsDb::::save_active_plan( + txn, + key_bytes.as_ref(), + block_number.try_into().unwrap(), + &plan, + ); + + let to_be_forwarded = forwarded_external_outputs.remove(plan.inputs[0].id().as_ref()); + if to_be_forwarded.is_some() { + assert_eq!(plan.inputs.len(), 1); + } + let (tx, branches) = prepare_send(network, block_number, fee, plan).await; + + // If this is a Plan for an output we're forwarding, we need to save the InInstruction for + // its output under the amount successfully forwarded + if let Some(mut instruction) = to_be_forwarded { + // If we can't successfully create a forwarding TX, simply drop this + if let Some(tx) = &tx { + instruction.balance.amount.0 -= tx.0.fee(); + MultisigsDb::::save_forwarded_output(txn, instruction); + } + } + + for branch in branches { + let existing = self.existing.as_mut().unwrap(); + let to_use = if key == existing.key { + existing + } else { + let new = self + .new + .as_mut() + .expect("plan wasn't for existing multisig yet there wasn't a new multisig"); + assert_eq!(key, new.key); + new + }; + + to_use.scheduler.created_output::(txn, branch.expected, branch.actual); + } + + if let Some((tx, eventuality)) = tx { + // The main function we return to will send an event to the coordinator which must be + // fired before these registered Eventualities have their Completions fired + // Safety is derived from a mutable lock on the Scanner being preserved, preventing + // scanning (and detection of Eventuality resolutions) before it's released + // It's only released by the main function after it does what it will + self + .scanner + .register_eventuality(key_bytes.as_ref(), block_number, id, eventuality.clone()) + .await; + + res.push((key, id, tx, eventuality)); + } + + // TODO: If the TX is None, restore its inputs to the scheduler + // Otherwise, if the TX had a change output, dropping its inputs would burn funds + // Are there exceptional cases upon rotation? + } + res + }; + (acquired_lock, res) + } + + pub async fn release_scanner_lock(&mut self) { + self.scanner.release_lock().await; + } + + fn scanner_event_to_multisig_event( + &self, + txn: &mut D::Transaction<'_>, + msg: ScannerEvent, + ) -> MultisigEvent { + let (block_number, event) = match msg { + ScannerEvent::Block { is_retirement_block, block, outputs } => { + // Since the Scanner is asynchronous, the following is a concern for race conditions + // We safely know the step of a block since keys are declared, and the Scanner is safe + // with respect to the declaration of keys + // Accordingly, the following calls regarding new keys and step should be safe + let block_number = ScannerHandle::::block_number(txn, &block) + .expect("didn't have the block number for a block we just scanned"); + let step = self.current_rotation_step(block_number); + + let mut instructions = vec![]; + for output in outputs { + // If these aren't externally received funds, don't handle it as an instruction + if output.kind() != OutputType::External { + continue; + } + + // If this is an External transaction to the existing multisig, and we're either solely + // forwarding or closing the existing multisig, drop it + // In the case of the forwarding case, we'll report it once it hits the new multisig + if (match step { + RotationStep::UseExisting | RotationStep::NewAsChange => false, + RotationStep::ForwardFromExisting | RotationStep::ClosingExisting => true, + }) && (output.key() == self.existing.as_ref().unwrap().key) + { + continue; + } + + let instruction = if let Some(instruction) = instruction_from_output::(&output) { + instruction + } else { + if !output.data().is_empty() { + // TODO2: Refund + continue; + } + + if let Some(instruction) = + MultisigsDb::::take_forwarded_output(txn, output.amount()) + { + instruction + } else { + // TODO2: Refund + continue; + } + }; + + // Delay External outputs received to new multisig earlier than expected + if Some(output.key()) == self.new.as_ref().map(|new| new.key) { + match step { + RotationStep::UseExisting => { + MultisigsDb::::save_delayed_output(txn, instruction); + continue; + } + RotationStep::NewAsChange | + RotationStep::ForwardFromExisting | + RotationStep::ClosingExisting => {} + } + } + + instructions.push(instruction); + } + + // If any outputs were delayed, append them into this block + match step { + RotationStep::UseExisting => {} + RotationStep::NewAsChange | + RotationStep::ForwardFromExisting | + RotationStep::ClosingExisting => { + instructions.extend(MultisigsDb::::take_delayed_outputs(txn)); + } + } + + let mut block_hash = [0; 32]; + block_hash.copy_from_slice(block.as_ref()); + let mut batch_id = MultisigsDb::::next_batch_id(txn); + + // start with empty batch + let mut batches = vec![Batch { + network: N::NETWORK, + id: batch_id, + block: BlockHash(block_hash), + instructions: vec![], + }]; + + for instruction in instructions { + let batch = batches.last_mut().unwrap(); + batch.instructions.push(instruction); + + // check if batch is over-size + if batch.encode().len() > MAX_BATCH_SIZE { + // pop the last instruction so it's back in size + let instruction = batch.instructions.pop().unwrap(); + + // bump the id for the new batch + batch_id += 1; + + // make a new batch with this instruction included + batches.push(Batch { + network: N::NETWORK, + id: batch_id, + block: BlockHash(block_hash), + instructions: vec![instruction], + }); + } + } + + // Save the next batch ID + MultisigsDb::::set_next_batch_id(txn, batch_id + 1); + + ( + block_number, + MultisigEvent::Batches( + if is_retirement_block { + Some((self.existing.as_ref().unwrap().key, self.new.as_ref().unwrap().key)) + } else { + None + }, + batches, + ), + ) + } + + // This must be emitted before ScannerEvent::Block for all completions of known Eventualities + // within the block. Unknown Eventualities may have their Completed events emitted after + // ScannerEvent::Block however. + ScannerEvent::Completed(key, block_number, id, tx) => { + MultisigsDb::::resolve_plan(txn, &key, id, tx.id()); + (block_number, MultisigEvent::Completed(key, id, tx)) + } + }; + + // If we either received a Block event (which will be the trigger when we have no + // Plans/Eventualities leading into ClosingExisting), or we received the last Completed for + // this multisig, set its retirement block + let existing = self.existing.as_ref().unwrap(); + + // This multisig is closing + let closing = self.current_rotation_step(block_number) == RotationStep::ClosingExisting; + // There's nothing left in its Scheduler. This call is safe as: + // 1) When ClosingExisting, all outputs should've been already forwarded, preventing + // new UTXOs from accumulating. + // 2) No new payments should be issued. + // 3) While there may be plans, they'll be dropped to create Eventualities. + // If this Eventuality is resolved, the Plan has already been dropped. + // 4) If this Eventuality will trigger a Plan, it'll still be in the plans HashMap. + let scheduler_is_empty = closing && existing.scheduler.empty(); + // Nothing is still being signed + let no_active_plans = scheduler_is_empty && + MultisigsDb::::active_plans(txn, existing.key.to_bytes().as_ref()).is_empty(); + + self + .scanner + .multisig_completed + // The above explicitly included their predecessor to ensure short-circuiting, yet their + // names aren't defined as an aggregate check. Still including all three here ensures all are + // used in the final value + .send(closing && scheduler_is_empty && no_active_plans) + .unwrap(); + + event + } + + // async fn where dropping the Future causes no state changes + // This property is derived from recv having this property, and recv being the only async call + pub async fn next_event(&mut self, txn: &RwLock>) -> MultisigEvent { + let event = self.scanner.events.recv().await.unwrap(); + + // No further code is async + + self.scanner_event_to_multisig_event(&mut *txn.write().unwrap(), event) + } +} diff --git a/processor/src/multisigs/scanner.rs b/processor/src/multisigs/scanner.rs new file mode 100644 index 000000000..39b271c97 --- /dev/null +++ b/processor/src/multisigs/scanner.rs @@ -0,0 +1,728 @@ +use core::marker::PhantomData; +use std::{ + sync::Arc, + io::Read, + time::Duration, + collections::{VecDeque, HashSet, HashMap}, +}; + +use ciphersuite::group::GroupEncoding; +use frost::curve::Ciphersuite; + +use log::{info, debug, warn}; +use tokio::{ + sync::{RwLockReadGuard, RwLockWriteGuard, RwLock, mpsc}, + time::sleep, +}; + +use crate::{ + Get, DbTxn, Db, + networks::{Output, Transaction, EventualitiesTracker, Block, Network}, +}; + +#[derive(Clone, Debug)] +pub enum ScannerEvent { + // Block scanned + Block { is_retirement_block: bool, block: >::Id, outputs: Vec }, + // Eventuality completion found on-chain + Completed(Vec, usize, [u8; 32], N::Transaction), +} + +pub type ScannerEventChannel = mpsc::UnboundedReceiver>; + +#[derive(Clone, Debug)] +struct ScannerDb(PhantomData, PhantomData); +impl ScannerDb { + fn scanner_key(dst: &'static [u8], key: impl AsRef<[u8]>) -> Vec { + D::key(b"SCANNER", dst, key) + } + + fn block_key(number: usize) -> Vec { + Self::scanner_key(b"block_id", u64::try_from(number).unwrap().to_le_bytes()) + } + fn block_number_key(id: &>::Id) -> Vec { + Self::scanner_key(b"block_number", id) + } + fn save_block(txn: &mut D::Transaction<'_>, number: usize, id: &>::Id) { + txn.put(Self::block_number_key(id), u64::try_from(number).unwrap().to_le_bytes()); + txn.put(Self::block_key(number), id); + } + fn block(getter: &G, number: usize) -> Option<>::Id> { + getter.get(Self::block_key(number)).map(|id| { + let mut res = >::Id::default(); + res.as_mut().copy_from_slice(&id); + res + }) + } + fn block_number(getter: &G, id: &>::Id) -> Option { + getter + .get(Self::block_number_key(id)) + .map(|number| u64::from_le_bytes(number.try_into().unwrap()).try_into().unwrap()) + } + + fn keys_key() -> Vec { + Self::scanner_key(b"keys", b"") + } + fn register_key( + txn: &mut D::Transaction<'_>, + activation_number: usize, + key: ::G, + ) { + let mut keys = txn.get(Self::keys_key()).unwrap_or(vec![]); + + let key_bytes = key.to_bytes(); + + let key_len = key_bytes.as_ref().len(); + assert_eq!(keys.len() % (8 + key_len), 0); + + // Sanity check this key isn't already present + let mut i = 0; + while i < keys.len() { + if &keys[(i + 8) .. ((i + 8) + key_len)] == key_bytes.as_ref() { + panic!("adding {} as a key yet it was already present", hex::encode(key_bytes)); + } + i += 8 + key_len; + } + + keys.extend(u64::try_from(activation_number).unwrap().to_le_bytes()); + keys.extend(key_bytes.as_ref()); + txn.put(Self::keys_key(), keys); + } + fn keys(getter: &G) -> Vec<(usize, ::G)> { + let bytes_vec = getter.get(Self::keys_key()).unwrap_or(vec![]); + let mut bytes: &[u8] = bytes_vec.as_ref(); + + // Assumes keys will be 32 bytes when calculating the capacity + // If keys are larger, this may allocate more memory than needed + // If keys are smaller, this may require additional allocations + // Either are fine + let mut res = Vec::with_capacity(bytes.len() / (8 + 32)); + while !bytes.is_empty() { + let mut activation_number = [0; 8]; + bytes.read_exact(&mut activation_number).unwrap(); + let activation_number = u64::from_le_bytes(activation_number).try_into().unwrap(); + + res.push((activation_number, N::Curve::read_G(&mut bytes).unwrap())); + } + res + } + fn retire_key(txn: &mut D::Transaction<'_>) { + let keys = Self::keys(txn); + assert_eq!(keys.len(), 2); + txn.del(Self::keys_key()); + Self::register_key(txn, keys[1].0, keys[1].1); + } + + fn seen_key(id: &>::Id) -> Vec { + Self::scanner_key(b"seen", id) + } + fn seen(getter: &G, id: &>::Id) -> bool { + getter.get(Self::seen_key(id)).is_some() + } + + fn outputs_key(block: &>::Id) -> Vec { + Self::scanner_key(b"outputs", block.as_ref()) + } + fn save_outputs( + txn: &mut D::Transaction<'_>, + block: &>::Id, + outputs: &[N::Output], + ) { + let mut bytes = Vec::with_capacity(outputs.len() * 64); + for output in outputs { + output.write(&mut bytes).unwrap(); + } + txn.put(Self::outputs_key(block), bytes); + } + fn outputs( + txn: &D::Transaction<'_>, + block: &>::Id, + ) -> Option> { + let bytes_vec = txn.get(Self::outputs_key(block))?; + let mut bytes: &[u8] = bytes_vec.as_ref(); + + let mut res = vec![]; + while !bytes.is_empty() { + res.push(N::Output::read(&mut bytes).unwrap()); + } + Some(res) + } + + fn scanned_block_key() -> Vec { + Self::scanner_key(b"scanned_block", []) + } + + fn save_scanned_block(txn: &mut D::Transaction<'_>, block: usize) -> Vec { + let id = Self::block(txn, block); // It may be None for the first key rotated to + let outputs = + if let Some(id) = id.as_ref() { Self::outputs(txn, id).unwrap_or(vec![]) } else { vec![] }; + + // Mark all the outputs from this block as seen + for output in &outputs { + txn.put(Self::seen_key(&output.id()), b""); + } + + txn.put(Self::scanned_block_key(), u64::try_from(block).unwrap().to_le_bytes()); + + // Return this block's outputs so they can be pruned from the RAM cache + outputs + } + fn latest_scanned_block(getter: &G) -> Option { + getter + .get(Self::scanned_block_key()) + .map(|bytes| u64::from_le_bytes(bytes.try_into().unwrap()).try_into().unwrap()) + } + + fn retirement_block_key(key: &::G) -> Vec { + Self::scanner_key(b"retirement_block", key.to_bytes()) + } + fn save_retirement_block( + txn: &mut D::Transaction<'_>, + key: &::G, + block: usize, + ) { + txn.put(Self::retirement_block_key(key), u64::try_from(block).unwrap().to_le_bytes()); + } + fn retirement_block(getter: &G, key: &::G) -> Option { + getter + .get(Self::retirement_block_key(key)) + .map(|bytes| usize::try_from(u64::from_le_bytes(bytes.try_into().unwrap())).unwrap()) + } +} + +/// The Scanner emits events relating to the blockchain, notably received outputs. +/// +/// It WILL NOT fail to emit an event, even if it reboots at selected moments. +/// +/// It MAY fire the same event multiple times. +#[derive(Debug)] +pub struct Scanner { + _db: PhantomData, + + keys: Vec<(usize, ::G)>, + + eventualities: HashMap, EventualitiesTracker>, + + ram_scanned: Option, + ram_outputs: HashSet>, + + need_ack: VecDeque, + + events: mpsc::UnboundedSender>, +} + +#[derive(Clone, Debug)] +struct ScannerHold { + scanner: Arc>>>, +} +impl ScannerHold { + async fn read(&self) -> RwLockReadGuard<'_, Option>> { + loop { + let lock = self.scanner.read().await; + if lock.is_none() { + drop(lock); + tokio::task::yield_now().await; + continue; + } + return lock; + } + } + async fn write(&self) -> RwLockWriteGuard<'_, Option>> { + loop { + let lock = self.scanner.write().await; + if lock.is_none() { + drop(lock); + tokio::task::yield_now().await; + continue; + } + return lock; + } + } + // This is safe to not check if something else already acquired the Scanner as the only caller is + // sequential. + async fn long_term_acquire(&self) -> Scanner { + self.scanner.write().await.take().unwrap() + } + async fn restore(&self, scanner: Scanner) { + let _ = self.scanner.write().await.insert(scanner); + } +} + +#[derive(Debug)] +pub struct ScannerHandle { + scanner: ScannerHold, + held_scanner: Option>, + pub events: ScannerEventChannel, + pub multisig_completed: mpsc::UnboundedSender, +} + +impl ScannerHandle { + pub async fn ram_scanned(&self) -> usize { + self.scanner.read().await.as_ref().unwrap().ram_scanned.unwrap_or(0) + } + + /// Register a key to scan for. + pub async fn register_key( + &mut self, + txn: &mut D::Transaction<'_>, + activation_number: usize, + key: ::G, + ) { + let mut scanner_lock = self.scanner.write().await; + let scanner = scanner_lock.as_mut().unwrap(); + assert!( + activation_number > scanner.ram_scanned.unwrap_or(0), + "activation block of new keys was already scanned", + ); + + info!("Registering key {} in scanner at {activation_number}", hex::encode(key.to_bytes())); + + if scanner.keys.is_empty() { + assert!(scanner.ram_scanned.is_none()); + scanner.ram_scanned = Some(activation_number); + assert!(ScannerDb::::save_scanned_block(txn, activation_number).is_empty()); + } + + ScannerDb::::register_key(txn, activation_number, key); + scanner.keys.push((activation_number, key)); + #[cfg(not(test))] // TODO: A test violates this. Improve the test with a better flow + assert!(scanner.keys.len() <= 2); + + scanner.eventualities.insert(key.to_bytes().as_ref().to_vec(), EventualitiesTracker::new()); + } + + pub fn db_scanned(getter: &G) -> Option { + ScannerDb::::latest_scanned_block(getter) + } + + // This perform a database read which isn't safe with regards to if the value is set or not + // It may be set, when it isn't expected to be set, or not set, when it is expected to be set + // Since the value is static, if it's set, it's correctly set + pub fn block_number(getter: &G, id: &>::Id) -> Option { + ScannerDb::::block_number(getter, id) + } + + /// Acknowledge having handled a block. + /// + /// Creates a lock over the Scanner, preventing its independent scanning operations until + /// released. + /// + /// This must only be called on blocks which have been scanned in-memory. + pub async fn ack_block( + &mut self, + txn: &mut D::Transaction<'_>, + id: >::Id, + ) -> (bool, Vec) { + debug!("block {} acknowledged", hex::encode(&id)); + + let mut scanner = self.scanner.long_term_acquire().await; + + // Get the number for this block + let number = ScannerDb::::block_number(txn, &id) + .expect("main loop trying to operate on data we haven't scanned"); + log::trace!("block {} was {number}", hex::encode(&id)); + + let outputs = ScannerDb::::save_scanned_block(txn, number); + // This has a race condition if we try to ack a block we scanned on a prior boot, and we have + // yet to scan it on this boot + assert!(number <= scanner.ram_scanned.unwrap()); + for output in &outputs { + assert!(scanner.ram_outputs.remove(output.id().as_ref())); + } + + assert_eq!(scanner.need_ack.pop_front().unwrap(), number); + + self.held_scanner = Some(scanner); + + // Load the key from the DB, as it will have already been removed from RAM if retired + let key = ScannerDb::::keys(txn)[0].1; + let is_retirement_block = ScannerDb::::retirement_block(txn, &key) == Some(number); + if is_retirement_block { + ScannerDb::::retire_key(txn); + } + (is_retirement_block, outputs) + } + + pub async fn register_eventuality( + &mut self, + key: &[u8], + block_number: usize, + id: [u8; 32], + eventuality: N::Eventuality, + ) { + let mut lock; + // We won't use held_scanner if we're re-registering on boot + (if let Some(scanner) = self.held_scanner.as_mut() { + scanner + } else { + lock = Some(self.scanner.write().await); + lock.as_mut().unwrap().as_mut().unwrap() + }) + .eventualities + .get_mut(key) + .unwrap() + .register(block_number, id, eventuality) + } + + pub async fn release_lock(&mut self) { + self.scanner.restore(self.held_scanner.take().unwrap()).await + } +} + +impl Scanner { + #[allow(clippy::type_complexity, clippy::new_ret_no_self)] + pub fn new( + network: N, + db: D, + ) -> (ScannerHandle, Vec<(usize, ::G)>) { + let (events_send, events_recv) = mpsc::unbounded_channel(); + let (multisig_completed_send, multisig_completed_recv) = mpsc::unbounded_channel(); + + let keys = ScannerDb::::keys(&db); + let mut eventualities = HashMap::new(); + for key in &keys { + eventualities.insert(key.1.to_bytes().as_ref().to_vec(), EventualitiesTracker::new()); + } + + let ram_scanned = ScannerDb::::latest_scanned_block(&db); + + let scanner = ScannerHold { + scanner: Arc::new(RwLock::new(Some(Scanner { + _db: PhantomData, + + keys: keys.clone(), + + eventualities, + + ram_scanned, + ram_outputs: HashSet::new(), + + need_ack: VecDeque::new(), + + events: events_send, + }))), + }; + tokio::spawn(Scanner::run(db, network, scanner.clone(), multisig_completed_recv)); + + ( + ScannerHandle { + scanner, + held_scanner: None, + events: events_recv, + multisig_completed: multisig_completed_send, + }, + keys, + ) + } + + async fn emit(&mut self, event: ScannerEvent) -> bool { + if self.events.send(event).is_err() { + info!("Scanner handler was dropped. Shutting down?"); + return false; + } + true + } + + // An async function, to be spawned on a task, to discover and report outputs + async fn run( + mut db: D, + network: N, + scanner_hold: ScannerHold, + mut multisig_completed: mpsc::UnboundedReceiver, + ) { + loop { + let (ram_scanned, latest_block_to_scan) = { + // Sleep 5 seconds to prevent hammering the node/scanner lock + sleep(Duration::from_secs(5)).await; + + let ram_scanned = { + let scanner_lock = scanner_hold.read().await; + let scanner = scanner_lock.as_ref().unwrap(); + + // If we're not scanning for keys yet, wait until we are + if scanner.keys.is_empty() { + continue; + } + + let ram_scanned = scanner.ram_scanned.unwrap(); + // If a Batch has taken too long to be published, start waiting until it is before + // continuing scanning + // Solves a race condition around multisig rotation, documented in the relevant doc + // and demonstrated with mini + if let Some(needing_ack) = scanner.need_ack.front() { + let next = ram_scanned + 1; + let limit = needing_ack + N::CONFIRMATIONS; + assert!(next <= limit); + if next == limit { + continue; + } + }; + + ram_scanned + }; + + ( + ram_scanned, + loop { + break match network.get_latest_block_number().await { + // Only scan confirmed blocks, which we consider effectively finalized + // CONFIRMATIONS - 1 as whatever's in the latest block already has 1 confirm + Ok(latest) => latest.saturating_sub(N::CONFIRMATIONS.saturating_sub(1)), + Err(_) => { + warn!("couldn't get latest block number"); + sleep(Duration::from_secs(60)).await; + continue; + } + }; + }, + ) + }; + + for block_being_scanned in (ram_scanned + 1) ..= latest_block_to_scan { + // Redo the checks for if we're too far ahead + { + let needing_ack = { + let scanner_lock = scanner_hold.read().await; + let scanner = scanner_lock.as_ref().unwrap(); + scanner.need_ack.front().cloned() + }; + + if let Some(needing_ack) = needing_ack { + let limit = needing_ack + N::CONFIRMATIONS; + assert!(block_being_scanned <= limit); + if block_being_scanned == limit { + break; + } + } + } + + let block = match network.get_block(block_being_scanned).await { + Ok(block) => block, + Err(_) => { + warn!("couldn't get block {block_being_scanned}"); + break; + } + }; + let block_id = block.id(); + + info!("scanning block: {} ({block_being_scanned})", hex::encode(&block_id)); + + // These DB calls are safe, despite not having a txn, since they're static values + // There's no issue if they're written in advance of expected (such as on reboot) + // They're also only expected here + if let Some(id) = ScannerDb::::block(&db, block_being_scanned) { + if id != block_id { + panic!("reorg'd from finalized {} to {}", hex::encode(id), hex::encode(block_id)); + } + } else { + // TODO: Move this to an unwrap + if let Some(id) = ScannerDb::::block(&db, block_being_scanned.saturating_sub(1)) { + if id != block.parent() { + panic!( + "block {} doesn't build off expected parent {}", + hex::encode(block_id), + hex::encode(id), + ); + } + } + + let mut txn = db.txn(); + ScannerDb::::save_block(&mut txn, block_being_scanned, &block_id); + txn.commit(); + } + + // Scan new blocks + // TODO: This lock acquisition may be long-lived... + let mut scanner_lock = scanner_hold.write().await; + let scanner = scanner_lock.as_mut().unwrap(); + + let mut has_activation = false; + let mut outputs = vec![]; + let mut completion_block_numbers = vec![]; + for (activation_number, key) in scanner.keys.clone() { + if activation_number > block_being_scanned { + continue; + } + + if activation_number == block_being_scanned { + has_activation = true; + } + + let key_vec = key.to_bytes().as_ref().to_vec(); + + // TODO: These lines are the ones which will cause a really long-lived lock acquisiton + for output in network.get_outputs(&block, key).await { + assert_eq!(output.key(), key); + outputs.push(output); + } + + for (id, (block_number, tx)) in network + .get_eventuality_completions(scanner.eventualities.get_mut(&key_vec).unwrap(), &block) + .await + { + info!( + "eventuality {} resolved by {}, as found on chain", + hex::encode(id), + hex::encode(&tx.id()) + ); + + completion_block_numbers.push(block_number); + // This must be before the mission of ScannerEvent::Block, per commentary in mod.rs + if !scanner.emit(ScannerEvent::Completed(key_vec.clone(), block_number, id, tx)).await { + return; + } + } + } + + // Panic if we've already seen these outputs + for output in &outputs { + let id = output.id(); + info!( + "block {} had output {} worth {}", + hex::encode(&block_id), + hex::encode(&id), + output.amount(), + ); + + // On Bitcoin, the output ID should be unique for a given chain + // On Monero, it's trivial to make an output sharing an ID with another + // We should only scan outputs with valid IDs however, which will be unique + + /* + The safety of this code must satisfy the following conditions: + 1) seen is not set for the first occurrence + 2) seen is set for any future occurrence + + seen is only written to after this code completes. Accordingly, it cannot be set + before the first occurrence UNLESSS it's set, yet the last scanned block isn't. + They are both written in the same database transaction, preventing this. + + As for future occurrences, the RAM entry ensures they're handled properly even if + the database has yet to be set. + + On reboot, which will clear the RAM, if seen wasn't set, neither was latest scanned + block. Accordingly, this will scan from some prior block, re-populating the RAM. + + If seen was set, then this will be successfully read. + + There's also no concern ram_outputs was pruned, yet seen wasn't set, as pruning + from ram_outputs will acquire a write lock (preventing this code from acquiring + its own write lock and running), and during its holding of the write lock, it + commits the transaction setting seen and the latest scanned block. + + This last case isn't true. Committing seen/latest_scanned_block happens after + relinquishing the write lock. + + TODO2: Only update ram_outputs after committing the TXN in question. + */ + let seen = ScannerDb::::seen(&db, &id); + let id = id.as_ref().to_vec(); + if seen || scanner.ram_outputs.contains(&id) { + panic!("scanned an output multiple times"); + } + scanner.ram_outputs.insert(id); + } + + // We could remove this, if instead of doing the first block which passed + // requirements + CONFIRMATIONS, we simply emitted an event for every block where + // `number % CONFIRMATIONS == 0` (once at the final stage for the existing multisig) + // There's no need at this point, yet the latter may be more suitable for modeling... + async fn check_multisig_completed( + db: &mut D, + multisig_completed: &mut mpsc::UnboundedReceiver, + block_number: usize, + ) -> bool { + match multisig_completed.recv().await { + None => { + info!("Scanner handler was dropped. Shutting down?"); + false + } + Some(completed) => { + // Set the retirement block as block_number + CONFIRMATIONS + if completed { + let mut txn = db.txn(); + // The retiring key is the earliest one still around + let retiring_key = ScannerDb::::keys(&txn)[0].1; + // This value is static w.r.t. the key + ScannerDb::::save_retirement_block( + &mut txn, + &retiring_key, + block_number + N::CONFIRMATIONS, + ); + txn.commit(); + } + true + } + } + } + + drop(scanner_lock); + // Now that we've dropped the Scanner lock, we need to handle the multisig_completed + // channel before we decide if this block should be fired or not + // (holding the Scanner risks a deadlock) + for block_number in completion_block_numbers { + if !check_multisig_completed::(&mut db, &mut multisig_completed, block_number).await + { + return; + }; + } + + // Reacquire the scanner + let mut scanner_lock = scanner_hold.write().await; + let scanner = scanner_lock.as_mut().unwrap(); + + // Only emit an event if any of the following is true: + // - This is an activation block + // - This is a retirement block + // - There's outputs + // as only those are blocks are meaningful and warrant obtaining synchrony over + // TODO: Consider not obtaining synchrony over the retirement block depending on how the + // hand-off is implemented on the Substrate side of things + let is_retirement_block = + ScannerDb::::retirement_block(&db, &scanner.keys[0].1) == Some(block_being_scanned); + let sent_block = if has_activation || is_retirement_block || (!outputs.is_empty()) { + // Save the outputs to disk + let mut txn = db.txn(); + ScannerDb::::save_outputs(&mut txn, &block_id, &outputs); + txn.commit(); + + // Send all outputs + if !scanner + .emit(ScannerEvent::Block { is_retirement_block, block: block_id, outputs }) + .await + { + return; + } + + // Since we're creating a Batch, mark it as needing ack + scanner.need_ack.push_back(block_being_scanned); + true + } else { + false + }; + + // Remove it from memory + if is_retirement_block { + let retired = scanner.keys.remove(0).1; + scanner.eventualities.remove(retired.to_bytes().as_ref()); + } + + // Update ram_scanned + scanner.ram_scanned = Some(block_being_scanned); + + drop(scanner_lock); + // If we sent a Block event, once again check multisig_completed + if sent_block && + (!check_multisig_completed::( + &mut db, + &mut multisig_completed, + block_being_scanned, + ) + .await) + { + return; + } + } + } + } +} diff --git a/processor/src/scheduler.rs b/processor/src/multisigs/scheduler.rs similarity index 77% rename from processor/src/scheduler.rs rename to processor/src/multisigs/scheduler.rs index db0890438..4d0e42ff7 100644 --- a/processor/src/scheduler.rs +++ b/processor/src/multisigs/scheduler.rs @@ -6,7 +6,7 @@ use std::{ use ciphersuite::{group::GroupEncoding, Ciphersuite}; use crate::{ - networks::{Output, Network}, + networks::{OutputType, Output, Network}, DbTxn, Db, Payment, Plan, }; @@ -29,8 +29,6 @@ pub struct Scheduler { // queued_plans are for outputs which we will create, yet when created, will have their amount // reduced by the fee it cost to be created. The Scheduler will then be told how what amount the // output actually has, and it'll be moved into plans - // - // TODO2: Consider edge case where branch/change isn't mined yet keys are deprecated queued_plans: HashMap>>>, plans: HashMap>>>, @@ -46,6 +44,13 @@ fn scheduler_key(key: &G) -> Vec { } impl Scheduler { + pub fn empty(&self) -> bool { + self.queued_plans.is_empty() && + self.plans.is_empty() && + self.utxos.is_empty() && + self.payments.is_empty() + } + fn read(key: ::G, reader: &mut R) -> io::Result { let mut read_plans = || -> io::Result<_> { let mut all_plans = HashMap::new(); @@ -93,7 +98,7 @@ impl Scheduler { Ok(Scheduler { key, queued_plans, plans, utxos, payments }) } - // TODO: Get rid of this + // TODO2: Get rid of this // We reserialize the entire scheduler on any mutation to save it to the DB which is horrible // We should have an incremental solution fn serialize(&self) -> Vec { @@ -152,19 +157,16 @@ impl Scheduler { Self::read(key, reader) } - fn execute(&mut self, inputs: Vec, mut payments: Vec>) -> Plan { - // This must be equal to plan.key due to how networks detect they created outputs which are to - // the branch address - let branch_address = N::branch_address(self.key); - // created_output will be called any time we send to a branch address - // If it's called, and it wasn't expecting to be called, that's almost certainly an error - // The only way it wouldn't be is if someone on Serai triggered a burn to a branch, which is - // pointless anyways - // If we allow such behavior, we lose the ability to detect the aforementioned class of errors - // Ignore these payments so we can safely assert there - let mut payments = - payments.drain(..).filter(|payment| payment.address != branch_address).collect::>(); + pub fn can_use_branch(&self, amount: u64) -> bool { + self.plans.contains_key(&amount) + } + fn execute( + &mut self, + inputs: Vec, + mut payments: Vec>, + key_for_any_change: ::G, + ) -> Plan { let mut change = false; let mut max = N::MAX_OUTPUTS; @@ -184,6 +186,8 @@ impl Scheduler { amount }; + let branch_address = N::branch_address(self.key); + // If we have more payments than we can handle in a single TX, create plans for them // TODO2: This isn't perfect. For 258 outputs, and a MAX_OUTPUTS of 16, this will create: // 15 branches of 16 leaves @@ -207,37 +211,44 @@ impl Scheduler { payments.insert(0, Payment { address: branch_address.clone(), data: None, amount }); } - // TODO2: Use the latest key for change - // TODO2: Update rotation documentation - Plan { key: self.key, inputs, payments, change: Some(self.key).filter(|_| change) } + Plan { + key: self.key, + inputs, + payments, + change: Some(N::change_address(key_for_any_change)).filter(|_| change), + } } - fn add_outputs(&mut self, mut utxos: Vec) -> Vec> { + fn add_outputs( + &mut self, + mut utxos: Vec, + key_for_any_change: ::G, + ) -> Vec> { log::info!("adding {} outputs", utxos.len()); let mut txs = vec![]; for utxo in utxos.drain(..) { - // If we can fulfill planned TXs with this output, do so - // We could limit this to UTXOs where `utxo.kind() == OutputType::Branch`, yet there's no - // practical benefit in doing so - let amount = utxo.amount(); - if let Some(plans) = self.plans.get_mut(&amount) { - // Execute the first set of payments possible with an output of this amount - let payments = plans.pop_front().unwrap(); - // They won't be equal if we dropped payments due to being dust - assert!(amount >= payments.iter().map(|payment| payment.amount).sum::()); - - // If we've grabbed the last plan for this output amount, remove it from the map - if plans.is_empty() { - self.plans.remove(&amount); - } + if utxo.kind() == OutputType::Branch { + let amount = utxo.amount(); + if let Some(plans) = self.plans.get_mut(&amount) { + // Execute the first set of payments possible with an output of this amount + let payments = plans.pop_front().unwrap(); + // They won't be equal if we dropped payments due to being dust + assert!(amount >= payments.iter().map(|payment| payment.amount).sum::()); + + // If we've grabbed the last plan for this output amount, remove it from the map + if plans.is_empty() { + self.plans.remove(&amount); + } - // Create a TX for these payments - txs.push(self.execute(vec![utxo], payments)); - } else { - self.utxos.push(utxo); + // Create a TX for these payments + txs.push(self.execute(vec![utxo], payments, key_for_any_change)); + continue; + } } + + self.utxos.push(utxo); } log::info!("{} planned TXs have had their required inputs confirmed", txs.len()); @@ -249,9 +260,28 @@ impl Scheduler { &mut self, txn: &mut D::Transaction<'_>, utxos: Vec, - payments: Vec>, + mut payments: Vec>, + key_for_any_change: ::G, + force_spend: bool, ) -> Vec> { - let mut plans = self.add_outputs(utxos); + // Drop payments to our own branch address + /* + created_output will be called any time we send to a branch address. If it's called, and it + wasn't expecting to be called, that's almost certainly an error. The only way to guarantee + this however is to only have us send to a branch address when creating a branch, hence the + dropping of pointless payments. + + This is not comprehensive as a payment may still be made to another active multisig's branch + address, depending on timing. This is safe as the issue only occurs when a multisig sends to + its *own* branch address, since created_output is called on the signer's Scheduler. + */ + { + let branch_address = N::branch_address(self.key); + payments = + payments.drain(..).filter(|payment| payment.address != branch_address).collect::>(); + } + + let mut plans = self.add_outputs(utxos, key_for_any_change); log::info!("scheduling {} new payments", payments.len()); @@ -293,10 +323,14 @@ impl Scheduler { for chunk in utxo_chunks.drain(..) { // TODO: While payments have their TXs' fees deducted from themselves, that doesn't hold here - // We need to charge a fee before reporting incoming UTXOs to Substrate to cover aggregation - // TXs + // We need the documented, but not yet implemented, virtual amount scheme to solve this log::debug!("aggregating a chunk of {} inputs", N::MAX_INPUTS); - plans.push(Plan { key: self.key, inputs: chunk, payments: vec![], change: Some(self.key) }) + plans.push(Plan { + key: self.key, + inputs: chunk, + payments: vec![], + change: Some(N::change_address(key_for_any_change)), + }) } // We want to use all possible UTXOs for all possible payments @@ -326,12 +360,25 @@ impl Scheduler { // Now that we have the list of payments we can successfully handle right now, create the TX // for them if !executing.is_empty() { - plans.push(self.execute(utxos, executing)); + plans.push(self.execute(utxos, executing, key_for_any_change)); } else { // If we don't have any payments to execute, save these UTXOs for later self.utxos.extend(utxos); } + // If we're instructed to force a spend, do so + // This is used when an old multisig is retiring and we want to always transfer outputs to the + // new one, regardless if we currently have payments + if force_spend && (!self.utxos.is_empty()) { + assert!(self.utxos.len() <= N::MAX_INPUTS); + plans.push(Plan { + key: self.key, + inputs: self.utxos.drain(..).collect::>(), + payments: vec![], + change: Some(N::change_address(key_for_any_change)), + }); + } + txn.put(scheduler_key::(&self.key), self.serialize()); log::info!( @@ -342,6 +389,14 @@ impl Scheduler { plans } + pub fn consume_payments(&mut self, txn: &mut D::Transaction<'_>) -> Vec> { + let res: Vec<_> = self.payments.drain(..).collect(); + if !res.is_empty() { + txn.put(scheduler_key::(&self.key), self.serialize()); + } + res + } + // Note a branch output as having been created, with the amount it was actually created with, // or not having been created due to being too small // This can be called whenever, so long as it's properly ordered @@ -399,7 +454,7 @@ impl Scheduler { #[allow(clippy::unwrap_or_default)] self.plans.entry(actual).or_insert(VecDeque::new()).push_back(payments); - // TODO: This shows how ridiculous the serialize function is + // TODO2: This shows how ridiculous the serialize function is txn.put(scheduler_key::(&self.key), self.serialize()); } } diff --git a/processor/src/networks/bitcoin.rs b/processor/src/networks/bitcoin.rs index 9bcc70db6..823b45ce2 100644 --- a/processor/src/networks/bitcoin.rs +++ b/processor/src/networks/bitcoin.rs @@ -15,6 +15,7 @@ use tokio::time::sleep; use bitcoin_serai::{ bitcoin::{ hashes::Hash as HashTrait, + key::{Parity, XOnlyPublicKey}, consensus::{Encodable, Decodable}, script::Instruction, address::{NetworkChecked, Address as BAddress}, @@ -45,8 +46,9 @@ use serai_client::{ use crate::{ networks::{ NetworkError, Block as BlockTrait, OutputType, Output as OutputTrait, - Transaction as TransactionTrait, Eventuality as EventualityTrait, EventualitiesTracker, - PostFeeBranch, Network, drop_branches, amortize_fee, + Transaction as TransactionTrait, SignableTransaction as SignableTransactionTrait, + Eventuality as EventualityTrait, EventualitiesTracker, PostFeeBranch, Network, drop_branches, + amortize_fee, }, Plan, }; @@ -76,7 +78,7 @@ pub struct Output { data: Vec, } -impl OutputTrait for Output { +impl OutputTrait for Output { type Id = OutputId; fn kind(&self) -> OutputType { @@ -97,6 +99,24 @@ impl OutputTrait for Output { res } + fn tx_id(&self) -> [u8; 32] { + let mut hash = *self.output.outpoint().txid.as_raw_hash().as_byte_array(); + hash.reverse(); + hash + } + + fn key(&self) -> ProjectivePoint { + let script = &self.output.output().script_pubkey; + assert!(script.is_v1_p2tr()); + let Instruction::PushBytes(key) = script.instructions_minimal().last().unwrap().unwrap() else { + panic!("last item in v1 Taproot script wasn't bytes") + }; + let key = XOnlyPublicKey::from_slice(key.as_ref()) + .expect("last item in v1 Taproot script wasn't x-only public key"); + Secp256k1::read_G(&mut key.public_key(Parity::Even).serialize().as_slice()).unwrap() - + (ProjectivePoint::GENERATOR * self.output.offset()) + } + fn balance(&self) -> Balance { Balance { coin: SeraiCoin::Bitcoin, amount: Amount(self.output.value()) } } @@ -196,7 +216,6 @@ impl EventualityTrait for Eventuality { #[derive(Clone, Debug)] pub struct SignableTransaction { - keys: ThresholdKeys, transcript: RecommendedTranscript, actual: BSignableTransaction, } @@ -206,6 +225,11 @@ impl PartialEq for SignableTransaction { } } impl Eq for SignableTransaction {} +impl SignableTransactionTrait for SignableTransaction { + fn fee(&self) -> u64 { + self.actual.fee() + } +} impl BlockTrait for Block { type Id = [u8; 32]; @@ -221,6 +245,8 @@ impl BlockTrait for Block { hash } + // TODO: Don't use this block's time, use the network time at this block + // TODO: Confirm network time is monotonic, enabling its usage here fn time(&self) -> u64 { self.header.time.into() } @@ -231,7 +257,7 @@ impl BlockTrait for Block { } } -const KEY_DST: &[u8] = b"Bitcoin Key"; +const KEY_DST: &[u8] = b"Serai Bitcoin Output Offset"; lazy_static::lazy_static! { static ref BRANCH_OFFSET: Scalar = Secp256k1::hash_to_F(KEY_DST, b"branch"); static ref CHANGE_OFFSET: Scalar = Secp256k1::hash_to_F(KEY_DST, b"change"); @@ -313,6 +339,7 @@ impl Network for Bitcoin { const NETWORK: NetworkId = NetworkId::Bitcoin; const ID: &'static str = "Bitcoin"; + const ESTIMATED_BLOCK_TIME_IN_SECONDS: usize = 600; const CONFIRMATIONS: usize = 6; // 0.0001 BTC, 10,000 satoshis @@ -348,6 +375,11 @@ impl Network for Bitcoin { Self::address(key + (ProjectivePoint::GENERATOR * offsets[&OutputType::Branch])) } + fn change_address(key: ProjectivePoint) -> Self::Address { + let (_, offsets, _) = scanner(key); + Self::address(key + (ProjectivePoint::GENERATOR * offsets[&OutputType::Change])) + } + async fn get_latest_block_number(&self) -> Result { self.rpc.get_latest_block_number().await.map_err(|_| NetworkError::ConnectionError) } @@ -358,11 +390,7 @@ impl Network for Bitcoin { self.rpc.get_block(&block_hash).await.map_err(|_| NetworkError::ConnectionError) } - async fn get_outputs( - &self, - block: &Self::Block, - key: ProjectivePoint, - ) -> Result, NetworkError> { + async fn get_outputs(&self, block: &Self::Block, key: ProjectivePoint) -> Vec { let (scanner, _, kinds) = scanner(key); let mut outputs = vec![]; @@ -390,18 +418,20 @@ impl Network for Bitcoin { }; data.truncate(MAX_DATA_LEN.try_into().unwrap()); - outputs.push(Output { kind, output, data }) + let output = Output { kind, output, data }; + assert_eq!(output.tx_id(), tx.id()); + outputs.push(output); } } - Ok(outputs) + outputs } async fn get_eventuality_completions( &self, eventualities: &mut EventualitiesTracker, block: &Self::Block, - ) -> HashMap<[u8; 32], [u8; 32]> { + ) -> HashMap<[u8; 32], (usize, Transaction)> { let mut res = HashMap::new(); if eventualities.map.is_empty() { return res; @@ -410,7 +440,7 @@ impl Network for Bitcoin { async fn check_block( eventualities: &mut EventualitiesTracker, block: &Block, - res: &mut HashMap<[u8; 32], [u8; 32]>, + res: &mut HashMap<[u8; 32], (usize, Transaction)>, ) { for tx in &block.txdata[1 ..] { let input = &tx.input[0].previous_output; @@ -430,7 +460,7 @@ impl Network for Bitcoin { "dishonest multisig spent input on distinct set of outputs" ); - res.insert(plan, tx.id()); + res.insert(plan, (eventualities.block_number, tx.clone())); } } @@ -476,7 +506,6 @@ impl Network for Bitcoin { async fn prepare_send( &self, - keys: ThresholdKeys, _: usize, mut plan: Plan, fee: Fee, @@ -497,10 +526,7 @@ impl Network for Bitcoin { match BSignableTransaction::new( plan.inputs.iter().map(|input| input.output.clone()).collect(), &payments, - plan.change.map(|key| { - let (_, offsets, _) = scanner(key); - Self::address(key + (ProjectivePoint::GENERATOR * offsets[&OutputType::Change])).0 - }), + plan.change.as_ref().map(|change| change.0.clone()), None, fee.0, ) { @@ -544,7 +570,7 @@ impl Network for Bitcoin { Ok(( Some(( - SignableTransaction { keys, transcript: plan.transcript(), actual: signable }, + SignableTransaction { transcript: plan.transcript(), actual: signable }, Eventuality { plan_binding_input, outputs }, )), branch_outputs, @@ -553,13 +579,14 @@ impl Network for Bitcoin { async fn attempt_send( &self, + keys: ThresholdKeys, transaction: Self::SignableTransaction, ) -> Result { Ok( transaction .actual .clone() - .multisig(transaction.keys.clone(), transaction.transcript) + .multisig(keys.clone(), transaction.transcript) .expect("used the wrong keys"), ) } diff --git a/processor/src/networks/mod.rs b/processor/src/networks/mod.rs index 7e0666aa5..113cebb0b 100644 --- a/processor/src/networks/mod.rs +++ b/processor/src/networks/mod.rs @@ -1,4 +1,4 @@ -use core::fmt::Debug; +use core::{fmt::Debug, time::Duration}; use std::{io, collections::HashMap}; use async_trait::async_trait; @@ -12,6 +12,10 @@ use frost::{ use serai_client::primitives::{NetworkId, Balance}; +use log::error; + +use tokio::time::sleep; + #[cfg(feature = "bitcoin")] pub mod bitcoin; #[cfg(feature = "bitcoin")] @@ -90,14 +94,17 @@ impl OutputType { } } -pub trait Output: Send + Sync + Sized + Clone + PartialEq + Eq + Debug { +pub trait Output: Send + Sync + Sized + Clone + PartialEq + Eq + Debug { type Id: 'static + Id; fn kind(&self) -> OutputType; fn id(&self) -> Self::Id; + fn tx_id(&self) -> >::Id; + fn key(&self) -> ::G; fn balance(&self) -> Balance; + // TODO: Remove this? fn amount(&self) -> u64 { self.balance().amount.0 } @@ -117,6 +124,10 @@ pub trait Transaction: Send + Sync + Sized + Clone + Debug { async fn fee(&self, network: &N) -> u64; } +pub trait SignableTransaction: Send + Sync + Clone + Debug { + fn fee(&self) -> u64; +} + pub trait Eventuality: Send + Sync + Clone + Debug { fn lookup(&self) -> Vec; @@ -172,10 +183,11 @@ impl Default for EventualitiesTracker { } pub trait Block: Send + Sync + Sized + Clone + Debug { - // This is currently bounded to being 32-bytes. + // This is currently bounded to being 32 bytes. type Id: 'static + Id; fn id(&self) -> Self::Id; fn parent(&self) -> Self::Id; + // The monotonic network time at this block. fn time(&self) -> u64; fn median_fee(&self) -> N::Fee; } @@ -275,9 +287,9 @@ pub trait Network: 'static + Send + Sync + Clone + PartialEq + Eq + Debug { /// The type containing all information on a scanned output. // This is almost certainly distinct from the network's native output type. - type Output: Output; + type Output: Output; /// The type containing all information on a planned transaction, waiting to be signed. - type SignableTransaction: Send + Sync + Clone + Debug; + type SignableTransaction: SignableTransaction; /// The type containing all information to check if a plan was completed. /// /// This must be binding to both the outputs expected and the plan ID. @@ -302,6 +314,8 @@ pub trait Network: 'static + Send + Sync + Clone + PartialEq + Eq + Debug { const NETWORK: NetworkId; /// String ID for this network. const ID: &'static str; + /// The estimated amount of time a block will take. + const ESTIMATED_BLOCK_TIME_IN_SECONDS: usize; /// The amount of confirmations required to consider a block 'final'. const CONFIRMATIONS: usize; /// The maximum amount of inputs which will fit in a TX. @@ -322,8 +336,9 @@ pub trait Network: 'static + Send + Sync + Clone + PartialEq + Eq + Debug { /// Address for the given group key to receive external coins to. fn address(key: ::G) -> Self::Address; /// Address for the given group key to use for scheduled branches. - // This is purely used for debugging purposes. Any output may be used to execute a branch. fn branch_address(key: ::G) -> Self::Address; + /// Address for the given group key to use for change. + fn change_address(key: ::G) -> Self::Address; /// Get the latest block's number. async fn get_latest_block_number(&self) -> Result; @@ -334,24 +349,26 @@ pub trait Network: 'static + Send + Sync + Clone + PartialEq + Eq + Debug { &self, block: &Self::Block, key: ::G, - ) -> Result, NetworkError>; + ) -> Vec; /// Get the registered eventualities completed within this block, and any prior blocks which /// registered eventualities may have been completed in. /// - /// This will panic if not fed a new block. + /// This may panic if not fed a block greater than the tracker's block number. + // TODO: get_eventuality_completions_internal + provided get_eventuality_completions for common + // code async fn get_eventuality_completions( &self, eventualities: &mut EventualitiesTracker, block: &Self::Block, - ) -> HashMap<[u8; 32], >::Id>; + ) -> HashMap<[u8; 32], (usize, Self::Transaction)>; /// Prepare a SignableTransaction for a transaction. + /// /// Returns None for the transaction if the SignableTransaction was dropped due to lack of value. #[rustfmt::skip] async fn prepare_send( &self, - keys: ThresholdKeys, block_number: usize, plan: Plan, fee: Self::Fee, @@ -363,6 +380,7 @@ pub trait Network: 'static + Send + Sync + Clone + PartialEq + Eq + Debug { /// Attempt to sign a SignableTransaction. async fn attempt_send( &self, + keys: ThresholdKeys, transaction: Self::SignableTransaction, ) -> Result; @@ -396,3 +414,35 @@ pub trait Network: 'static + Send + Sync + Clone + PartialEq + Eq + Debug { #[cfg(test)] async fn test_send(&self, key: Self::Address) -> Self::Block; } + +// TODO: Move into above trait +pub async fn get_latest_block_number(network: &N) -> usize { + loop { + match network.get_latest_block_number().await { + Ok(number) => { + return number; + } + Err(e) => { + error!( + "couldn't get the latest block number in main's error-free get_block. {} {}", + "this should only happen if the node is offline. error: ", e + ); + sleep(Duration::from_secs(10)).await; + } + } + } +} + +pub async fn get_block(network: &N, block_number: usize) -> N::Block { + loop { + match network.get_block(block_number).await { + Ok(block) => { + return block; + } + Err(e) => { + error!("couldn't get block {block_number} in main's error-free get_block. error: {}", e); + sleep(Duration::from_secs(10)).await; + } + } + } +} diff --git a/processor/src/networks/monero.rs b/processor/src/networks/monero.rs index 32617fcde..82f04326e 100644 --- a/processor/src/networks/monero.rs +++ b/processor/src/networks/monero.rs @@ -37,8 +37,9 @@ use crate::{ Payment, Plan, additional_key, networks::{ NetworkError, Block as BlockTrait, OutputType, Output as OutputTrait, - Transaction as TransactionTrait, Eventuality as EventualityTrait, EventualitiesTracker, - PostFeeBranch, Network, drop_branches, amortize_fee, + Transaction as TransactionTrait, SignableTransaction as SignableTransactionTrait, + Eventuality as EventualityTrait, EventualitiesTracker, PostFeeBranch, Network, drop_branches, + amortize_fee, }, }; @@ -49,7 +50,7 @@ const EXTERNAL_SUBADDRESS: Option = SubaddressIndex::new(0, 0); const BRANCH_SUBADDRESS: Option = SubaddressIndex::new(1, 0); const CHANGE_SUBADDRESS: Option = SubaddressIndex::new(2, 0); -impl OutputTrait for Output { +impl OutputTrait for Output { // While we could use (tx, o), using the key ensures we won't be susceptible to the burning bug. // While we already are immune, thanks to using featured address, this doesn't hurt and is // technically more efficient. @@ -68,6 +69,14 @@ impl OutputTrait for Output { self.0.output.data.key.compress().to_bytes() } + fn tx_id(&self) -> [u8; 32] { + self.0.output.absolute.tx + } + + fn key(&self) -> EdwardsPoint { + EdwardsPoint(self.0.output.data.key - (EdwardsPoint::generator().0 * self.0.key_offset())) + } + fn balance(&self) -> Balance { Balance { coin: SeraiCoin::Monero, amount: Amount(self.0.commitment().amount) } } @@ -130,10 +139,14 @@ impl EventualityTrait for Eventuality { #[derive(Clone, Debug)] pub struct SignableTransaction { - keys: ThresholdKeys, transcript: RecommendedTranscript, actual: MSignableTransaction, } +impl SignableTransactionTrait for SignableTransaction { + fn fee(&self) -> u64 { + self.actual.fee() + } +} impl BlockTrait for Block { type Id = [u8; 32]; @@ -145,6 +158,7 @@ impl BlockTrait for Block { self.header.previous } + // TODO: Check Monero enforces this to be monotonic and sane fn time(&self) -> u64 { self.header.timestamp } @@ -227,6 +241,7 @@ impl Network for Monero { const NETWORK: NetworkId = NetworkId::Monero; const ID: &'static str = "Monero"; + const ESTIMATED_BLOCK_TIME_IN_SECONDS: usize = 120; const CONFIRMATIONS: usize = 10; // wallet2 will not create a transaction larger than 100kb, and Monero won't relay a transaction @@ -250,6 +265,10 @@ impl Network for Monero { Self::address_internal(key, BRANCH_SUBADDRESS) } + fn change_address(key: EdwardsPoint) -> Self::Address { + Self::address_internal(key, CHANGE_SUBADDRESS) + } + async fn get_latest_block_number(&self) -> Result { // Monero defines height as chain length, so subtract 1 for block number Ok(self.rpc.get_height().await.map_err(|_| NetworkError::ConnectionError)? - 1) @@ -267,15 +286,19 @@ impl Network for Monero { ) } - async fn get_outputs( - &self, - block: &Block, - key: EdwardsPoint, - ) -> Result, NetworkError> { - let mut txs = Self::scanner(key) - .scan(&self.rpc, block) - .await - .map_err(|_| NetworkError::ConnectionError)? + async fn get_outputs(&self, block: &Block, key: EdwardsPoint) -> Vec { + let outputs = loop { + match Self::scanner(key).scan(&self.rpc, block).await { + Ok(outputs) => break outputs, + Err(e) => { + log::error!("couldn't scan block {}: {e:?}", hex::encode(block.id())); + sleep(Duration::from_secs(60)).await; + continue; + } + } + }; + + let mut txs = outputs .iter() .filter_map(|outputs| Some(outputs.not_locked()).filter(|outputs| !outputs.is_empty())) .collect::>(); @@ -305,14 +328,14 @@ impl Network for Monero { } } - Ok(outputs) + outputs } async fn get_eventuality_completions( &self, eventualities: &mut EventualitiesTracker, block: &Block, - ) -> HashMap<[u8; 32], [u8; 32]> { + ) -> HashMap<[u8; 32], (usize, Transaction)> { let mut res = HashMap::new(); if eventualities.map.is_empty() { return res; @@ -322,7 +345,7 @@ impl Network for Monero { network: &Monero, eventualities: &mut EventualitiesTracker, block: &Block, - res: &mut HashMap<[u8; 32], [u8; 32]>, + res: &mut HashMap<[u8; 32], (usize, Transaction)>, ) { for hash in &block.txs { let tx = { @@ -339,7 +362,7 @@ impl Network for Monero { if let Some((_, eventuality)) = eventualities.map.get(&tx.prefix.extra) { if eventuality.matches(&tx) { - res.insert(eventualities.map.remove(&tx.prefix.extra).unwrap().0, tx.hash()); + res.insert(eventualities.map.remove(&tx.prefix.extra).unwrap().0, (block.number(), tx)); } } } @@ -373,7 +396,6 @@ impl Network for Monero { async fn prepare_send( &self, - keys: ThresholdKeys, block_number: usize, mut plan: Plan, fee: Fee, @@ -457,9 +479,7 @@ impl Network for Monero { Some(Zeroizing::new(plan.id())), inputs.clone(), payments, - plan.change.map(|key| { - Change::fingerprintable(Self::address_internal(key, CHANGE_SUBADDRESS).into()) - }), + plan.change.map(|change| Change::fingerprintable(change.into())), vec![], fee, ) { @@ -509,7 +529,6 @@ impl Network for Monero { let branch_outputs = amortize_fee(&mut plan, tx_fee); let signable = SignableTransaction { - keys, transcript, actual: match signable(plan, Some(tx_fee))? { Some(signable) => signable, @@ -522,9 +541,10 @@ impl Network for Monero { async fn attempt_send( &self, + keys: ThresholdKeys, transaction: SignableTransaction, ) -> Result { - match transaction.actual.clone().multisig(transaction.keys.clone(), transaction.transcript) { + match transaction.actual.clone().multisig(keys, transaction.transcript) { Ok(machine) => Ok(machine), Err(e) => panic!("failed to create a multisig machine for TX: {e}"), } diff --git a/processor/src/plan.rs b/processor/src/plan.rs index 129966046..3f005865b 100644 --- a/processor/src/plan.rs +++ b/processor/src/plan.rs @@ -24,6 +24,7 @@ impl Payment { } pub fn write(&self, writer: &mut W) -> io::Result<()> { + // TODO: Don't allow creating Payments with an Address which can't be serialized let address: Vec = self .address .clone() @@ -74,7 +75,7 @@ pub struct Plan { pub key: ::G, pub inputs: Vec, pub payments: Vec>, - pub change: Option<::G>, + pub change: Option, } impl core::fmt::Debug for Plan { fn fmt(&self, fmt: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { @@ -83,7 +84,7 @@ impl core::fmt::Debug for Plan { .field("key", &hex::encode(self.key.to_bytes())) .field("inputs", &self.inputs) .field("payments", &self.payments) - .field("change", &self.change.map(|change| hex::encode(change.to_bytes()))) + .field("change", &self.change.as_ref().map(|change| change.to_string())) .finish() } } @@ -105,8 +106,8 @@ impl Plan { payment.transcript(&mut transcript); } - if let Some(change) = self.change { - transcript.append_message(b"change", change.to_bytes()); + if let Some(change) = &self.change { + transcript.append_message(b"change", change.to_string()); } transcript @@ -132,12 +133,23 @@ impl Plan { payment.write(writer)?; } - writer.write_all(&[u8::from(self.change.is_some())])?; - if let Some(change) = &self.change { - writer.write_all(change.to_bytes().as_ref())?; - } - - Ok(()) + // TODO: Have Plan construction fail if change cannot be serialized + let change = if let Some(change) = &self.change { + change.clone().try_into().map_err(|_| { + io::Error::new( + io::ErrorKind::Other, + format!( + "an address we said to use as change couldn't be convered to a Vec: {}", + change.to_string(), + ), + ) + })? + } else { + vec![] + }; + assert!(serai_client::primitives::MAX_ADDRESS_LEN <= u8::MAX.into()); + writer.write_all(&[u8::try_from(change.len()).unwrap()])?; + writer.write_all(&change) } pub fn read(reader: &mut R) -> io::Result { @@ -156,9 +168,20 @@ impl Plan { payments.push(Payment::::read(reader)?); } - let mut buf = [0; 1]; - reader.read_exact(&mut buf)?; - let change = if buf[0] == 1 { Some(N::Curve::read_G(reader)?) } else { None }; + let mut len = [0; 1]; + reader.read_exact(&mut len)?; + let mut change = vec![0; usize::from(len[0])]; + reader.read_exact(&mut change)?; + let change = if change.is_empty() { + None + } else { + Some(N::Address::try_from(change).map_err(|_| { + io::Error::new( + io::ErrorKind::Other, + "couldn't deserialize an Address serialized into a Plan", + ) + })?) + }; Ok(Plan { key, inputs, payments, change }) } diff --git a/processor/src/scanner.rs b/processor/src/scanner.rs deleted file mode 100644 index cdf438dd0..000000000 --- a/processor/src/scanner.rs +++ /dev/null @@ -1,525 +0,0 @@ -use core::marker::PhantomData; -use std::{ - sync::Arc, - time::Duration, - collections::{HashSet, HashMap}, -}; - -use ciphersuite::group::GroupEncoding; -use frost::curve::Ciphersuite; - -use log::{info, debug, warn}; -use tokio::{ - sync::{RwLock, mpsc}, - time::sleep, -}; - -use crate::{ - Get, DbTxn, Db, - networks::{Output, Transaction, EventualitiesTracker, Block, Network}, -}; - -#[derive(Clone, Debug)] -pub enum ScannerEvent { - // Block scanned - Block { block: >::Id, outputs: Vec }, - // Eventuality completion found on-chain - Completed([u8; 32], >::Id), -} - -pub type ScannerEventChannel = mpsc::UnboundedReceiver>; - -#[derive(Clone, Debug)] -struct ScannerDb(PhantomData, PhantomData); -impl ScannerDb { - fn scanner_key(dst: &'static [u8], key: impl AsRef<[u8]>) -> Vec { - D::key(b"SCANNER", dst, key) - } - - fn block_key(number: usize) -> Vec { - Self::scanner_key(b"block_id", u64::try_from(number).unwrap().to_le_bytes()) - } - fn block_number_key(id: &>::Id) -> Vec { - Self::scanner_key(b"block_number", id) - } - fn save_block(txn: &mut D::Transaction<'_>, number: usize, id: &>::Id) { - txn.put(Self::block_number_key(id), u64::try_from(number).unwrap().to_le_bytes()); - txn.put(Self::block_key(number), id); - } - fn block(getter: &G, number: usize) -> Option<>::Id> { - getter.get(Self::block_key(number)).map(|id| { - let mut res = >::Id::default(); - res.as_mut().copy_from_slice(&id); - res - }) - } - fn block_number(getter: &G, id: &>::Id) -> Option { - getter - .get(Self::block_number_key(id)) - .map(|number| u64::from_le_bytes(number.try_into().unwrap()).try_into().unwrap()) - } - - fn active_keys_key() -> Vec { - Self::scanner_key(b"active_keys", b"") - } - fn add_active_key(txn: &mut D::Transaction<'_>, key: ::G) { - let mut keys = txn.get(Self::active_keys_key()).unwrap_or(vec![]); - - let key_bytes = key.to_bytes(); - - let key_len = key_bytes.as_ref().len(); - assert_eq!(keys.len() % key_len, 0); - - // Don't add this key if it's already present - let mut i = 0; - while i < keys.len() { - if &keys[i .. (i + key_len)] == key_bytes.as_ref() { - debug!("adding {} as an active key yet it was already present", hex::encode(key_bytes)); - return; - } - i += key_len; - } - - keys.extend(key_bytes.as_ref()); - txn.put(Self::active_keys_key(), keys); - } - fn active_keys(getter: &G) -> Vec<::G> { - let bytes_vec = getter.get(Self::active_keys_key()).unwrap_or(vec![]); - let mut bytes: &[u8] = bytes_vec.as_ref(); - - // Assumes keys will be 32 bytes when calculating the capacity - // If keys are larger, this may allocate more memory than needed - // If keys are smaller, this may require additional allocations - // Either are fine - let mut res = Vec::with_capacity(bytes.len() / 32); - while !bytes.is_empty() { - res.push(N::Curve::read_G(&mut bytes).unwrap()); - } - res - } - - fn seen_key(id: &::Id) -> Vec { - Self::scanner_key(b"seen", id) - } - fn seen(getter: &G, id: &::Id) -> bool { - getter.get(Self::seen_key(id)).is_some() - } - - fn next_batch_key() -> Vec { - Self::scanner_key(b"next_batch", []) - } - fn outputs_key( - key: &::G, - block: &>::Id, - ) -> Vec { - Self::scanner_key(b"outputs", [key.to_bytes().as_ref(), block.as_ref()].concat()) - } - fn save_outputs( - txn: &mut D::Transaction<'_>, - key: &::G, - block: &>::Id, - outputs: &[N::Output], - ) { - let mut bytes = Vec::with_capacity(outputs.len() * 64); - for output in outputs { - output.write(&mut bytes).unwrap(); - } - txn.put(Self::outputs_key(key, block), bytes); - - // This is a new set of outputs, which are expected to be handled in a perfectly ordered - // fashion - - // TODO2: This is not currently how this works - // There may be new blocks 0 .. 5, which A will scan, yet then B may be activated at block 4 - // This would cause - // 0a, 1a, 2a, 3a, 4a, 5a, 4b, 5b - // when it should be - // 0a, 1a, 2a, 3a, 4a, 4b, 5a, 5b - } - fn outputs( - txn: &D::Transaction<'_>, - key: &::G, - block: &>::Id, - ) -> Option> { - let bytes_vec = txn.get(Self::outputs_key(key, block))?; - let mut bytes: &[u8] = bytes_vec.as_ref(); - - let mut res = vec![]; - while !bytes.is_empty() { - res.push(N::Output::read(&mut bytes).unwrap()); - } - Some(res) - } - - fn scanned_block_key(key: &::G) -> Vec { - Self::scanner_key(b"scanned_block", key.to_bytes()) - } - - #[allow(clippy::type_complexity)] - fn save_scanned_block( - txn: &mut D::Transaction<'_>, - key: &::G, - block: usize, - ) -> Vec { - let id = Self::block(txn, block); // It may be None for the first key rotated to - let outputs = if let Some(id) = id.as_ref() { - Self::outputs(txn, key, id).unwrap_or(vec![]) - } else { - vec![] - }; - - // Mark all the outputs from this block as seen - for output in &outputs { - txn.put(Self::seen_key(&output.id()), b""); - } - - txn.put(Self::scanned_block_key(key), u64::try_from(block).unwrap().to_le_bytes()); - - // Return this block's outputs so they can be pruned from the RAM cache - outputs - } - fn latest_scanned_block(getter: &G, key: ::G) -> usize { - let bytes = getter - .get(Self::scanned_block_key(&key)) - .expect("asking for latest scanned block of key which wasn't rotated to"); - u64::from_le_bytes(bytes.try_into().unwrap()).try_into().unwrap() - } -} - -/// The Scanner emits events relating to the blockchain, notably received outputs. -/// It WILL NOT fail to emit an event, even if it reboots at selected moments. -/// It MAY fire the same event multiple times. -#[derive(Debug)] -pub struct Scanner { - network: N, - db: D, - keys: Vec<::G>, - - eventualities: EventualitiesTracker, - - ram_scanned: HashMap, usize>, - ram_outputs: HashSet>, - - events: mpsc::UnboundedSender>, -} - -#[derive(Debug)] -pub struct ScannerHandle { - scanner: Arc>>, - pub events: ScannerEventChannel, -} - -impl ScannerHandle { - pub async fn ram_scanned(&self) -> usize { - let mut res = None; - for scanned in self.scanner.read().await.ram_scanned.values() { - if res.is_none() { - res = Some(*scanned); - } - // Returns the lowest scanned value so no matter the keys interacted with, this is - // sufficiently scanned - res = Some(res.unwrap().min(*scanned)); - } - res.unwrap_or(0) - } - - pub async fn register_eventuality( - &mut self, - block_number: usize, - id: [u8; 32], - eventuality: N::Eventuality, - ) { - self.scanner.write().await.eventualities.register(block_number, id, eventuality) - } - - pub async fn drop_eventuality(&mut self, id: [u8; 32]) { - self.scanner.write().await.eventualities.drop(id); - } - - /// Rotate the key being scanned for. - /// - /// If no key has been prior set, this will become the key with no further actions. - /// - /// If a key has been prior set, both keys will be scanned for as detailed in the Multisig - /// documentation. The old key will eventually stop being scanned for, leaving just the - /// updated-to key. - pub async fn rotate_key( - &mut self, - txn: &mut D::Transaction<'_>, - activation_number: usize, - key: ::G, - ) { - let mut scanner = self.scanner.write().await; - if !scanner.keys.is_empty() { - // Protonet will have a single, static validator set - // TODO2 - panic!("only a single key is supported at this time"); - } - - info!("Rotating scanner to key {} at {activation_number}", hex::encode(key.to_bytes())); - - let outputs = ScannerDb::::save_scanned_block(txn, &key, activation_number); - scanner.ram_scanned.insert(key.to_bytes().as_ref().to_vec(), activation_number); - assert!(outputs.is_empty()); - - ScannerDb::::add_active_key(txn, key); - scanner.keys.push(key); - } - - // This perform a database read which isn't safe with regards to if the value is set or not - // It may be set, when it isn't expected to be set, or not set, when it is expected to be set - // Since the value is static, if it's set, it's correctly set - pub async fn block_number(&self, id: &>::Id) -> Option { - ScannerDb::::block_number(&self.scanner.read().await.db, id) - } - - // Set the next batch ID to use - pub fn set_next_batch_id(&self, txn: &mut D::Transaction<'_>, batch: u32) { - txn.put(ScannerDb::::next_batch_key(), batch.to_le_bytes()); - } - - // Get the next batch ID - pub fn next_batch_id(&self, txn: &D::Transaction<'_>) -> u32 { - txn - .get(ScannerDb::::next_batch_key()) - .map_or(0, |v| u32::from_le_bytes(v.try_into().unwrap())) - } - - /// Acknowledge having handled a block for a key. - pub async fn ack_up_to_block( - &mut self, - txn: &mut D::Transaction<'_>, - key: ::G, - id: >::Id, - ) -> Vec { - let mut scanner = self.scanner.write().await; - debug!("Block {} acknowledged", hex::encode(&id)); - - // Get the number for this block - let number = ScannerDb::::block_number(txn, &id) - .expect("main loop trying to operate on data we haven't scanned"); - // Get the number of the last block we acknowledged - let prior = ScannerDb::::latest_scanned_block(txn, key); - - let mut outputs = vec![]; - for number in (prior + 1) ..= number { - outputs.extend(ScannerDb::::save_scanned_block(txn, &key, number)); - } - - for output in &outputs { - assert!(scanner.ram_outputs.remove(output.id().as_ref())); - } - - outputs - } -} - -impl Scanner { - #[allow(clippy::new_ret_no_self)] - pub fn new(network: N, db: D) -> (ScannerHandle, Vec<::G>) { - let (events_send, events_recv) = mpsc::unbounded_channel(); - - let keys = ScannerDb::::active_keys(&db); - let mut ram_scanned = HashMap::new(); - for key in keys.clone() { - ram_scanned.insert( - key.to_bytes().as_ref().to_vec(), - ScannerDb::::latest_scanned_block(&db, key), - ); - } - - let scanner = Arc::new(RwLock::new(Scanner { - network, - db, - keys: keys.clone(), - - eventualities: EventualitiesTracker::new(), - - ram_scanned, - ram_outputs: HashSet::new(), - - events: events_send, - })); - tokio::spawn(Scanner::run(scanner.clone())); - - (ScannerHandle { scanner, events: events_recv }, keys) - } - - fn emit(&mut self, event: ScannerEvent) -> bool { - if self.events.send(event).is_err() { - info!("Scanner handler was dropped. Shutting down?"); - return false; - } - true - } - - // An async function, to be spawned on a task, to discover and report outputs - async fn run(scanner: Arc>) { - loop { - // Only check every five seconds for new blocks - sleep(Duration::from_secs(5)).await; - - // Scan new blocks - { - let mut scanner = scanner.write().await; - let latest = scanner.network.get_latest_block_number().await; - let latest = match latest { - // Only scan confirmed blocks, which we consider effectively finalized - // CONFIRMATIONS - 1 as whatever's in the latest block already has 1 confirm - Ok(latest) => latest.saturating_sub(N::CONFIRMATIONS.saturating_sub(1)), - Err(_) => { - warn!("couldn't get latest block number"); - sleep(Duration::from_secs(60)).await; - continue; - } - }; - - for key in scanner.keys.clone() { - let key_vec = key.to_bytes().as_ref().to_vec(); - let latest_scanned = scanner.ram_scanned[&key_vec]; - - for i in (latest_scanned + 1) ..= latest { - // TODO2: Check for key deprecation - - let block = match scanner.network.get_block(i).await { - Ok(block) => block, - Err(_) => { - warn!("couldn't get block {i}"); - break; - } - }; - let block_id = block.id(); - - // These block calls are safe, despite not having a txn, since they're static values - // only written to/read by this thread - // There's also no error caused by them being unexpectedly written (if the commit is - // made and then the processor suddenly reboots) - // There's also no issue if this code is run multiple times (due to code after - // aborting) - if let Some(id) = ScannerDb::::block(&scanner.db, i) { - if id != block_id { - panic!("reorg'd from finalized {} to {}", hex::encode(id), hex::encode(block_id)); - } - } else { - info!("Found new block: {}", hex::encode(&block_id)); - - if let Some(id) = ScannerDb::::block(&scanner.db, i.saturating_sub(1)) { - if id != block.parent() { - panic!( - "block {} doesn't build off expected parent {}", - hex::encode(block_id), - hex::encode(id), - ); - } - } - - let mut txn = scanner.db.txn(); - ScannerDb::::save_block(&mut txn, i, &block_id); - txn.commit(); - } - - let outputs = match scanner.network.get_outputs(&block, key).await { - Ok(outputs) => outputs, - Err(_) => { - warn!("couldn't scan block {i}"); - break; - } - }; - - // Write this number as scanned so we won't perform any of the following mutations - // multiple times - scanner.ram_scanned.insert(key_vec.clone(), i); - - // Panic if we've already seen these outputs - for output in &outputs { - let id = output.id(); - info!( - "block {} had output {} worth {}", - hex::encode(&block_id), - hex::encode(&id), - output.amount(), - ); - - // On Bitcoin, the output ID should be unique for a given chain - // On Monero, it's trivial to make an output sharing an ID with another - // We should only scan outputs with valid IDs however, which will be unique - - /* - The safety of this code must satisfy the following conditions: - 1) seen is not set for the first occurrence - 2) seen is set for any future occurrence - - seen is only written to after this code completes. Accordingly, it cannot be set - before the first occurrence UNLESSS it's set, yet the last scanned block isn't. - They are both written in the same database transaction, preventing this. - - As for future occurrences, the RAM entry ensures they're handled properly even if - the database has yet to be set. - - On reboot, which will clear the RAM, if seen wasn't set, neither was latest scanned - block. Accordingly, this will scan from some prior block, re-populating the RAM. - - If seen was set, then this will be successfully read. - - There's also no concern ram_outputs was pruned, yet seen wasn't set, as pruning - from ram_outputs will acquire a write lock (preventing this code from acquiring - its own write lock and running), and during its holding of the write lock, it - commits the transaction setting seen and the latest scanned block. - - This last case isn't true. Committing seen/latest_scanned_block happens after - relinquishing the write lock. - - TODO: Only update ram_outputs after committing the TXN in question. - */ - let seen = ScannerDb::::seen(&scanner.db, &id); - let id = id.as_ref().to_vec(); - if seen || scanner.ram_outputs.contains(&id) { - panic!("scanned an output multiple times"); - } - scanner.ram_outputs.insert(id); - } - - // Clone network because we can't borrow it while also mutably borrowing the - // eventualities - // Thankfully, network is written to be a cheap clone - let network = scanner.network.clone(); - // TODO: This get_eventuality_completions call will panic if called multiple times over - // the same blocks (such as when checking multiple keys under the current layout), - // as get_eventuality_completions assumes it's always only fed a future block - for (id, tx) in - network.get_eventuality_completions(&mut scanner.eventualities, &block).await - { - // This should only happen if there's a P2P net desync or there's a malicious - // validator - warn!( - "eventuality {} resolved by {}, as found on chain. this should not happen", - hex::encode(id), - hex::encode(&tx) - ); - - if !scanner.emit(ScannerEvent::Completed(id, tx)) { - return; - } - } - - // Don't emit an event if there's not any outputs - if outputs.is_empty() { - continue; - } - - // Save the outputs to disk - let mut txn = scanner.db.txn(); - ScannerDb::::save_outputs(&mut txn, &key, &block_id, &outputs); - txn.commit(); - - // Send all outputs - // TODO2: Fire this with all outputs for all keys, not for each key - if !scanner.emit(ScannerEvent::Block { block: block_id, outputs }) { - return; - } - } - } - } - } - } -} diff --git a/processor/src/signer.rs b/processor/src/signer.rs index a9afa8dbd..e098c07c9 100644 --- a/processor/src/signer.rs +++ b/processor/src/signer.rs @@ -139,10 +139,6 @@ impl Signer { } } - pub fn keys(&self) -> ThresholdKeys { - self.keys.clone() - } - fn verify_id(&self, id: &SignId) -> Result<(), ()> { // Check the attempt lines up match self.attempt.get(&id.id) { @@ -202,28 +198,42 @@ impl Signer { self.events.push_back(SignerEvent::SignedTransaction { id, tx: tx_id }); } - pub async fn eventuality_completion( + pub fn completed(&mut self, txn: &mut D::Transaction<'_>, id: [u8; 32], tx: N::Transaction) { + let first_completion = !self.already_completed(txn, id); + + // Save this completion to the DB + SignerDb::::save_transaction(txn, &tx); + SignerDb::::complete(txn, id, &tx.id()); + + if first_completion { + self.complete(id, tx.id()); + } + } + + // Doesn't use any loops/retries since we'll eventually get this from the Scanner anyways + async fn claimed_eventuality_completion( &mut self, txn: &mut D::Transaction<'_>, id: [u8; 32], tx_id: &>::Id, - ) { + ) -> bool { if let Some(eventuality) = SignerDb::::eventuality(txn, id) { // Transaction hasn't hit our mempool/was dropped for a different signature // The latter can happen given certain latency conditions/a single malicious signer - // In the case of a single malicious signer, they can drag multiple honest - // validators down with them, so we unfortunately can't slash on this case + // In the case of a single malicious signer, they can drag multiple honest validators down + // with them, so we unfortunately can't slash on this case let Ok(tx) = self.network.get_transaction(tx_id).await else { warn!( - "a validator claimed {} completed {} yet we didn't have that TX in our mempool", + "a validator claimed {} completed {} yet we didn't have that TX in our mempool {}", hex::encode(tx_id), hex::encode(id), + "(or had another connectivity issue)", ); - return; + return false; }; if self.network.confirm_completion(&eventuality, &tx) { - info!("eventuality for {} resolved in TX {}", hex::encode(id), hex::encode(tx_id)); + info!("signer eventuality for {} resolved in TX {}", hex::encode(id), hex::encode(tx_id)); let first_completion = !self.already_completed(txn, id); @@ -233,6 +243,7 @@ impl Signer { if first_completion { self.complete(id, tx.id()); + return true; } } else { warn!( @@ -242,12 +253,17 @@ impl Signer { ); } } else { - warn!( - "signer {} informed of the completion of plan {}. that plan was not recognized", + // If we don't have this in RAM, it should be because we already finished signing it + // TODO: Will the coordinator ever send us Completed for an unknown ID? + assert!(SignerDb::::completed(txn, id).is_some()); + info!( + "signer {} informed of the eventuality completion for plan {}, {}", hex::encode(self.keys.group_key().to_bytes()), hex::encode(id), + "which we already marked as completed", ); } + false } async fn attempt(&mut self, txn: &mut D::Transaction<'_>, id: [u8; 32], attempt: u32) { @@ -311,7 +327,7 @@ impl Signer { SignerDb::::attempt(txn, &id); // Attempt to create the TX - let machine = match self.network.attempt_send(tx).await { + let machine = match self.network.attempt_send(self.keys.clone(), tx).await { Err(e) => { error!("failed to attempt {}, #{}: {:?}", hex::encode(id.id), id.attempt, e); return; @@ -481,7 +497,7 @@ impl Signer { } tx.as_mut().copy_from_slice(&tx_vec); - self.eventuality_completion(txn, id, &tx).await; + self.claimed_eventuality_completion(txn, id, &tx).await; } } } diff --git a/processor/src/substrate_signer.rs b/processor/src/substrate_signer.rs index 64ae1fbbf..e7c0ded61 100644 --- a/processor/src/substrate_signer.rs +++ b/processor/src/substrate_signer.rs @@ -4,6 +4,7 @@ use std::collections::{VecDeque, HashMap}; use rand_core::OsRng; use transcript::{Transcript, RecommendedTranscript}; +use ciphersuite::group::GroupEncoding; use frost::{ curve::Ristretto, ThresholdKeys, @@ -177,9 +178,7 @@ impl SubstrateSigner { // Update the attempt number self.attempt.insert(id, attempt); - // Doesn't set key since there's only one key active at a time - // TODO: BatchSignId - let id = SignId { key: vec![], id, attempt }; + let id = SignId { key: self.keys.group_key().to_bytes().to_vec(), id, attempt }; info!("signing batch {} #{}", hex::encode(id.id), id.attempt); // If we reboot mid-sign, the current design has us abort all signs and wait for latter @@ -208,7 +207,7 @@ impl SubstrateSigner { // b"substrate" is a literal from sp-core let machine = AlgorithmMachine::new(Schnorrkel::new(b"substrate"), self.keys.clone()); - // TODO: Use a seeded RNG here so we don't produce distinct messages with the same purpose + // TODO: Use a seeded RNG here so we don't produce distinct messages with the same intent // This is also needed so we don't preprocess, send preprocess, reboot before ack'ing the // message, send distinct preprocess, and then attempt a signing session premised on the former // with the latter diff --git a/processor/src/tests/addresses.rs b/processor/src/tests/addresses.rs index 00a88805a..9d726e030 100644 --- a/processor/src/tests/addresses.rs +++ b/processor/src/tests/addresses.rs @@ -12,11 +12,12 @@ use serai_db::{DbTxn, MemDb}; use crate::{ Plan, Db, networks::{OutputType, Output, Block, Network}, - scanner::{ScannerEvent, Scanner, ScannerHandle}, + multisigs::scanner::{ScannerEvent, Scanner, ScannerHandle}, tests::sign, }; async fn spend( + db: &mut D, network: &N, keys: &HashMap>, scanner: &mut ScannerHandle, @@ -32,10 +33,14 @@ async fn spend( keys.clone(), network .prepare_send( - keys.clone(), network.get_latest_block_number().await.unwrap() - N::CONFIRMATIONS, // Send to a change output - Plan { key, inputs: outputs.clone(), payments: vec![], change: Some(key) }, + Plan { + key, + inputs: outputs.clone(), + payments: vec![], + change: Some(N::change_address(key)), + }, network.get_fee().await, ) .await @@ -51,13 +56,19 @@ async fn spend( network.mine_block().await; } match timeout(Duration::from_secs(30), scanner.events.recv()).await.unwrap().unwrap() { - ScannerEvent::Block { block: _, outputs } => { + ScannerEvent::Block { is_retirement_block, block, outputs } => { + scanner.multisig_completed.send(false).unwrap(); + assert!(!is_retirement_block); assert_eq!(outputs.len(), 1); // Make sure this is actually a change output assert_eq!(outputs[0].kind(), OutputType::Change); + let mut txn = db.txn(); + assert_eq!(scanner.ack_block(&mut txn, block).await.1, outputs); + scanner.release_lock().await; + txn.commit(); outputs } - ScannerEvent::Completed(_, _) => { + ScannerEvent::Completed(_, _, _, _) => { panic!("unexpectedly got eventuality completion"); } } @@ -76,11 +87,14 @@ pub async fn test_addresses(network: N) { } let mut db = MemDb::new(); - let (mut scanner, active_keys) = Scanner::new(network.clone(), db.clone()); - assert!(active_keys.is_empty()); + let (mut scanner, current_keys) = Scanner::new(network.clone(), db.clone()); + assert!(current_keys.is_empty()); let mut txn = db.txn(); - scanner.rotate_key(&mut txn, network.get_latest_block_number().await.unwrap(), key).await; + scanner.register_key(&mut txn, network.get_latest_block_number().await.unwrap(), key).await; txn.commit(); + for _ in 0 .. N::CONFIRMATIONS { + network.mine_block().await; + } // Receive funds to the branch address and make sure it's properly identified let block_id = network.test_send(N::branch_address(key)).await.id(); @@ -88,19 +102,25 @@ pub async fn test_addresses(network: N) { // Verify the Scanner picked them up let outputs = match timeout(Duration::from_secs(30), scanner.events.recv()).await.unwrap().unwrap() { - ScannerEvent::Block { block, outputs } => { + ScannerEvent::Block { is_retirement_block, block, outputs } => { + scanner.multisig_completed.send(false).unwrap(); + assert!(!is_retirement_block); assert_eq!(block, block_id); assert_eq!(outputs.len(), 1); assert_eq!(outputs[0].kind(), OutputType::Branch); + let mut txn = db.txn(); + assert_eq!(scanner.ack_block(&mut txn, block).await.1, outputs); + scanner.release_lock().await; + txn.commit(); outputs } - ScannerEvent::Completed(_, _) => { + ScannerEvent::Completed(_, _, _, _) => { panic!("unexpectedly got eventuality completion"); } }; // Spend the branch output, creating a change output and ensuring we actually get change - let outputs = spend(&network, &keys, &mut scanner, outputs).await; + let outputs = spend(&mut db, &network, &keys, &mut scanner, outputs).await; // Also test spending the change output - spend(&network, &keys, &mut scanner, outputs).await; + spend(&mut db, &network, &keys, &mut scanner, outputs).await; } diff --git a/processor/src/tests/literal/mod.rs b/processor/src/tests/literal/mod.rs index d94943616..c98913cdd 100644 --- a/processor/src/tests/literal/mod.rs +++ b/processor/src/tests/literal/mod.rs @@ -16,6 +16,7 @@ mod bitcoin { bitcoin_signer, bitcoin_wallet, bitcoin_addresses, + bitcoin_no_deadlock_in_multisig_completed, ); } @@ -39,5 +40,6 @@ mod monero { monero_signer, monero_wallet, monero_addresses, + monero_no_deadlock_in_multisig_completed, ); } diff --git a/processor/src/tests/mod.rs b/processor/src/tests/mod.rs index 084d60b39..13af66bfd 100644 --- a/processor/src/tests/mod.rs +++ b/processor/src/tests/mod.rs @@ -2,7 +2,7 @@ mod key_gen; pub(crate) use key_gen::test_key_gen; mod scanner; -pub(crate) use scanner::test_scanner; +pub(crate) use scanner::{test_scanner, test_no_deadlock_in_multisig_completed}; mod signer; pub(crate) use signer::{sign, test_signer}; @@ -59,8 +59,12 @@ macro_rules! test_network { $signer: ident, $wallet: ident, $addresses: ident, + $no_deadlock_in_multisig_completed: ident, ) => { - use $crate::tests::{test_key_gen, test_scanner, test_signer, test_wallet, test_addresses}; + use $crate::tests::{ + test_key_gen, test_scanner, test_no_deadlock_in_multisig_completed, test_signer, test_wallet, + test_addresses, + }; // This doesn't interact with a node and accordingly doesn't need to be run sequentially #[tokio::test] @@ -93,6 +97,12 @@ macro_rules! test_network { test_addresses($network().await).await; } } + + async_sequential! { + async fn $no_deadlock_in_multisig_completed() { + test_no_deadlock_in_multisig_completed($network().await).await; + } + } }; } diff --git a/processor/src/tests/scanner.rs b/processor/src/tests/scanner.rs index 45d7abef2..747538c09 100644 --- a/processor/src/tests/scanner.rs +++ b/processor/src/tests/scanner.rs @@ -1,19 +1,17 @@ use core::time::Duration; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use rand_core::OsRng; -use frost::Participant; +use frost::{Participant, tests::key_gen}; -use tokio::time::timeout; - -use serai_client::primitives::BlockHash; +use tokio::{sync::Mutex, time::timeout}; use serai_db::{DbTxn, Db, MemDb}; use crate::{ networks::{OutputType, Output, Block, Network}, - scanner::{ScannerEvent, Scanner, ScannerHandle}, + multisigs::scanner::{ScannerEvent, Scanner, ScannerHandle}, }; pub async fn test_scanner(network: N) { @@ -32,16 +30,19 @@ pub async fn test_scanner(network: N) { let db = MemDb::new(); let new_scanner = || async { let mut db = db.clone(); - let (mut scanner, active_keys) = Scanner::new(network.clone(), db.clone()); - let mut first = first.lock().unwrap(); + let (mut scanner, current_keys) = Scanner::new(network.clone(), db.clone()); + let mut first = first.lock().await; if *first { - assert!(active_keys.is_empty()); + assert!(current_keys.is_empty()); let mut txn = db.txn(); - scanner.rotate_key(&mut txn, activation_number, group_key).await; + scanner.register_key(&mut txn, activation_number, group_key).await; txn.commit(); + for _ in 0 .. N::CONFIRMATIONS { + network.mine_block().await; + } *first = false; } else { - assert_eq!(active_keys.len(), 1); + assert_eq!(current_keys.len(), 1); } scanner }; @@ -55,13 +56,15 @@ pub async fn test_scanner(network: N) { let verify_event = |mut scanner: ScannerHandle| async { let outputs = match timeout(Duration::from_secs(30), scanner.events.recv()).await.unwrap().unwrap() { - ScannerEvent::Block { block, outputs } => { + ScannerEvent::Block { is_retirement_block, block, outputs } => { + scanner.multisig_completed.send(false).unwrap(); + assert!(!is_retirement_block); assert_eq!(block, block_id); assert_eq!(outputs.len(), 1); assert_eq!(outputs[0].kind(), OutputType::External); outputs } - ScannerEvent::Completed(_, _) => { + ScannerEvent::Completed(_, _, _, _) => { panic!("unexpectedly got eventuality completion"); } }; @@ -73,22 +76,10 @@ pub async fn test_scanner(network: N) { verify_event(new_scanner().await).await; // Acknowledge the block - - // Acknowledging it should yield a list of all blocks since the last acknowledged block - let mut blocks = vec![]; - let mut curr_block = activation_number + 1; - loop { - let block = network.get_block(curr_block).await.unwrap().id(); - blocks.push(BlockHash(block.as_ref().try_into().unwrap())); - if block == block_id { - break; - } - curr_block += 1; - } - let mut cloned_db = db.clone(); let mut txn = cloned_db.txn(); - assert_eq!(scanner.ack_up_to_block(&mut txn, keys.group_key(), block_id).await, outputs); + assert_eq!(scanner.ack_block(&mut txn, block_id).await.1, outputs); + scanner.release_lock().await; txn.commit(); // There should be no more events @@ -97,3 +88,67 @@ pub async fn test_scanner(network: N) { // Create a new scanner off the current DB and make sure it also does nothing assert!(timeout(Duration::from_secs(30), new_scanner().await.events.recv()).await.is_err()); } + +pub async fn test_no_deadlock_in_multisig_completed(network: N) { + // Mine blocks so there's a confirmed block + for _ in 0 .. N::CONFIRMATIONS { + network.mine_block().await; + } + + let mut db = MemDb::new(); + let (mut scanner, current_keys) = Scanner::new(network.clone(), db.clone()); + assert!(current_keys.is_empty()); + + let mut txn = db.txn(); + // Register keys to cause Block events at CONFIRMATIONS (dropped since first keys), + // CONFIRMATIONS + 1, and CONFIRMATIONS + 2 + for i in 0 .. 3 { + scanner + .register_key( + &mut txn, + network.get_latest_block_number().await.unwrap() + N::CONFIRMATIONS + i, + { + let mut keys = key_gen(&mut OsRng); + for (_, keys) in keys.iter_mut() { + N::tweak_keys(keys); + } + keys[&Participant::new(1).unwrap()].group_key() + }, + ) + .await; + } + txn.commit(); + + for _ in 0 .. (3 * N::CONFIRMATIONS) { + network.mine_block().await; + } + + let block_id = + match timeout(Duration::from_secs(30), scanner.events.recv()).await.unwrap().unwrap() { + ScannerEvent::Block { is_retirement_block, block, outputs: _ } => { + scanner.multisig_completed.send(false).unwrap(); + assert!(!is_retirement_block); + block + } + ScannerEvent::Completed(_, _, _, _) => { + panic!("unexpectedly got eventuality completion"); + } + }; + + match timeout(Duration::from_secs(30), scanner.events.recv()).await.unwrap().unwrap() { + ScannerEvent::Block { .. } => {} + ScannerEvent::Completed(_, _, _, _) => { + panic!("unexpectedly got eventuality completion"); + } + }; + + // The ack_block acquisiton shows the Scanner isn't maintaining the lock on its own thread after + // emitting the Block event + // TODO: This is incomplete. Also test after emitting Completed + let mut txn = db.txn(); + assert_eq!(scanner.ack_block(&mut txn, block_id).await.1, vec![]); + scanner.release_lock().await; + txn.commit(); + + scanner.multisig_completed.send(false).unwrap(); +} diff --git a/processor/src/tests/signer.rs b/processor/src/tests/signer.rs index 1e0c53b83..c5a0e6c5a 100644 --- a/processor/src/tests/signer.rs +++ b/processor/src/tests/signer.rs @@ -153,7 +153,7 @@ pub async fn test_signer(network: N) { } let key = keys[&Participant::new(1).unwrap()].group_key(); - let outputs = network.get_outputs(&network.test_send(N::address(key)).await, key).await.unwrap(); + let outputs = network.get_outputs(&network.test_send(N::address(key)).await, key).await; let sync_block = network.get_latest_block_number().await.unwrap() - N::CONFIRMATIONS; let fee = network.get_fee().await; @@ -163,13 +163,12 @@ pub async fn test_signer(network: N) { for (i, keys) in keys.drain() { let (signable, eventuality) = network .prepare_send( - keys.clone(), sync_block, Plan { key, inputs: outputs.clone(), payments: vec![Payment { address: N::address(key), data: None, amount }], - change: Some(key), + change: Some(N::change_address(key)), }, fee, ) @@ -194,8 +193,7 @@ pub async fn test_signer(network: N) { &network.get_block(network.get_latest_block_number().await.unwrap()).await.unwrap(), key, ) - .await - .unwrap(); + .await; assert_eq!(outputs.len(), 2); // Adjust the amount for the fees let amount = amount - tx.fee(&network).await; diff --git a/processor/src/tests/substrate_signer.rs b/processor/src/tests/substrate_signer.rs index 583e5ca89..a457b56fd 100644 --- a/processor/src/tests/substrate_signer.rs +++ b/processor/src/tests/substrate_signer.rs @@ -26,7 +26,11 @@ async fn test_substrate_signer() { let id: u32 = 5; let block = BlockHash([0xaa; 32]); - let mut actual_id = SignId { key: vec![], id: [0; 32], attempt: 0 }; + let mut actual_id = SignId { + key: keys.values().next().unwrap().group_key().to_bytes().to_vec(), + id: [0; 32], + attempt: 0, + }; let batch = Batch { network: NetworkId::Monero, diff --git a/processor/src/tests/wallet.rs b/processor/src/tests/wallet.rs index 8b22685cf..8bb84d457 100644 --- a/processor/src/tests/wallet.rs +++ b/processor/src/tests/wallet.rs @@ -11,13 +11,20 @@ use serai_db::{DbTxn, Db, MemDb}; use crate::{ Payment, Plan, networks::{Output, Transaction, Block, Network}, - scanner::{ScannerEvent, Scanner}, - scheduler::Scheduler, + multisigs::{ + scanner::{ScannerEvent, Scanner}, + scheduler::Scheduler, + }, tests::sign, }; // Tests the Scanner, Scheduler, and Signer together pub async fn test_wallet(network: N) { + // Mine blocks so there's a confirmed block + for _ in 0 .. N::CONFIRMATIONS { + network.mine_block().await; + } + let mut keys = key_gen(&mut OsRng); for (_, keys) in keys.iter_mut() { N::tweak_keys(keys); @@ -25,27 +32,36 @@ pub async fn test_wallet(network: N) { let key = keys[&Participant::new(1).unwrap()].group_key(); let mut db = MemDb::new(); - let (mut scanner, active_keys) = Scanner::new(network.clone(), db.clone()); - assert!(active_keys.is_empty()); + let (mut scanner, current_keys) = Scanner::new(network.clone(), db.clone()); + assert!(current_keys.is_empty()); let (block_id, outputs) = { let mut txn = db.txn(); - scanner.rotate_key(&mut txn, network.get_latest_block_number().await.unwrap(), key).await; + scanner.register_key(&mut txn, network.get_latest_block_number().await.unwrap(), key).await; txn.commit(); + for _ in 0 .. N::CONFIRMATIONS { + network.mine_block().await; + } let block = network.test_send(N::address(key)).await; let block_id = block.id(); match timeout(Duration::from_secs(30), scanner.events.recv()).await.unwrap().unwrap() { - ScannerEvent::Block { block, outputs } => { + ScannerEvent::Block { is_retirement_block, block, outputs } => { + scanner.multisig_completed.send(false).unwrap(); + assert!(!is_retirement_block); assert_eq!(block, block_id); assert_eq!(outputs.len(), 1); (block_id, outputs) } - ScannerEvent::Completed(_, _) => { + ScannerEvent::Completed(_, _, _, _) => { panic!("unexpectedly got eventuality completion"); } } }; + let mut txn = db.txn(); + assert_eq!(scanner.ack_block(&mut txn, block_id.clone()).await.1, outputs); + scanner.release_lock().await; + txn.commit(); let mut txn = db.txn(); let mut scheduler = Scheduler::new::(&mut txn, key); @@ -54,6 +70,8 @@ pub async fn test_wallet(network: N) { &mut txn, outputs.clone(), vec![Payment { address: N::address(key), data: None, amount }], + key, + false, ); txn.commit(); assert_eq!( @@ -62,7 +80,7 @@ pub async fn test_wallet(network: N) { key, inputs: outputs.clone(), payments: vec![Payment { address: N::address(key), data: None, amount }], - change: Some(key), + change: Some(N::change_address(key)), }] ); @@ -78,7 +96,7 @@ pub async fn test_wallet(network: N) { let mut eventualities = vec![]; for (i, keys) in keys.drain() { let (signable, eventuality) = network - .prepare_send(keys.clone(), network.get_block_number(&block_id).await, plans[0].clone(), fee) + .prepare_send(network.get_block_number(&block_id).await, plans[0].clone(), fee) .await .unwrap() .0 @@ -93,8 +111,7 @@ pub async fn test_wallet(network: N) { network.mine_block().await; let block_number = network.get_latest_block_number().await.unwrap(); let block = network.get_block(block_number).await.unwrap(); - let first_outputs = outputs; - let outputs = network.get_outputs(&block, key).await.unwrap(); + let outputs = network.get_outputs(&block, key).await; assert_eq!(outputs.len(), 2); let amount = amount - tx.fee(&network).await; assert!((outputs[0].amount() == amount) || (outputs[1].amount() == amount)); @@ -108,20 +125,20 @@ pub async fn test_wallet(network: N) { } match timeout(Duration::from_secs(30), scanner.events.recv()).await.unwrap().unwrap() { - ScannerEvent::Block { block: block_id, outputs: these_outputs } => { + ScannerEvent::Block { is_retirement_block, block: block_id, outputs: these_outputs } => { + scanner.multisig_completed.send(false).unwrap(); + assert!(!is_retirement_block); assert_eq!(block_id, block.id()); assert_eq!(these_outputs, outputs); } - ScannerEvent::Completed(_, _) => { + ScannerEvent::Completed(_, _, _, _) => { panic!("unexpectedly got eventuality completion"); } } // Check the Scanner DB can reload the outputs let mut txn = db.txn(); - assert_eq!( - scanner.ack_up_to_block(&mut txn, key, block.id()).await, - [first_outputs, outputs].concat().to_vec() - ); + assert_eq!(scanner.ack_block(&mut txn, block.id()).await.1, outputs); + scanner.release_lock().await; txn.commit(); } diff --git a/substrate/client/Cargo.toml b/substrate/client/Cargo.toml index bd9048582..bd72e1723 100644 --- a/substrate/client/Cargo.toml +++ b/substrate/client/Cargo.toml @@ -36,6 +36,8 @@ lazy_static = "1" rand_core = "0.6" +blake2 = "0.10" + ciphersuite = { path = "../../crypto/ciphersuite", features = ["ristretto"] } frost = { package = "modular-frost", path = "../../crypto/frost", features = ["tests"] } schnorrkel = { path = "../../crypto/schnorrkel", package = "frost-schnorrkel" } diff --git a/substrate/client/tests/batch.rs b/substrate/client/tests/batch.rs index 84ae0d6be..9fef6bed4 100644 --- a/substrate/client/tests/batch.rs +++ b/substrate/client/tests/batch.rs @@ -1,5 +1,12 @@ use rand_core::{RngCore, OsRng}; +use blake2::{ + digest::{consts::U32, Digest}, + Blake2b, +}; + +use scale::Encode; + use serai_client::{ primitives::{Amount, NetworkId, Coin, Balance, BlockHash, SeraiAddress}, in_instructions::{ @@ -38,12 +45,20 @@ serai_test!( }], }; - let block = provide_batch(batch).await; + let block = provide_batch(batch.clone()).await; let serai = serai().await; assert_eq!(serai.get_latest_block_for_network(block, network).await.unwrap(), Some(block_hash)); let batches = serai.get_batch_events(block).await.unwrap(); - assert_eq!(batches, vec![InInstructionsEvent::Batch { network, id, block: block_hash }]); + assert_eq!( + batches, + vec![InInstructionsEvent::Batch { + network, + id, + block: block_hash, + instructions_hash: Blake2b::::digest(batch.instructions.encode()).into(), + }] + ); assert_eq!( serai.get_mint_events(block).await.unwrap(), diff --git a/substrate/client/tests/burn.rs b/substrate/client/tests/burn.rs index ddaf260c4..8b7f5cc87 100644 --- a/substrate/client/tests/burn.rs +++ b/substrate/client/tests/burn.rs @@ -1,5 +1,12 @@ use rand_core::{RngCore, OsRng}; +use blake2::{ + digest::{consts::U32, Digest}, + Blake2b, +}; + +use scale::Encode; + use sp_core::Pair; use serai_client::{ @@ -45,11 +52,19 @@ serai_test!( }], }; - let block = provide_batch(batch).await; + let block = provide_batch(batch.clone()).await; let serai = serai().await; let batches = serai.get_batch_events(block).await.unwrap(); - assert_eq!(batches, vec![InInstructionsEvent::Batch { network, id, block: block_hash }]); + assert_eq!( + batches, + vec![InInstructionsEvent::Batch { + network, + id, + block: block_hash, + instructions_hash: Blake2b::::digest(batch.instructions.encode()).into(), + }] + ); assert_eq!( serai.get_mint_events(block).await.unwrap(), diff --git a/substrate/client/tests/common/in_instructions.rs b/substrate/client/tests/common/in_instructions.rs index bb5315065..aefcb7fc8 100644 --- a/substrate/client/tests/common/in_instructions.rs +++ b/substrate/client/tests/common/in_instructions.rs @@ -1,3 +1,10 @@ +use blake2::{ + digest::{consts::U32, Digest}, + Blake2b, +}; + +use scale::Encode; + use sp_core::Pair; use serai_client::{ @@ -38,7 +45,12 @@ pub async fn provide_batch(batch: Batch) -> [u8; 32] { // TODO: impl From for BatchEvent? assert_eq!( batches, - vec![InInstructionsEvent::Batch { network: batch.network, id: batch.id, block: batch.block }], + vec![InInstructionsEvent::Batch { + network: batch.network, + id: batch.id, + block: batch.block, + instructions_hash: Blake2b::::digest(batch.instructions.encode()).into(), + }], ); // TODO: Check the tokens events diff --git a/substrate/in-instructions/pallet/Cargo.toml b/substrate/in-instructions/pallet/Cargo.toml index 5261e70c8..d8a312d7f 100644 --- a/substrate/in-instructions/pallet/Cargo.toml +++ b/substrate/in-instructions/pallet/Cargo.toml @@ -17,9 +17,10 @@ thiserror = { version = "1", optional = true } scale = { package = "parity-scale-codec", version = "3", default-features = false, features = ["derive", "max-encoded-len"] } scale-info = { version = "2", default-features = false, features = ["derive"] } +sp-core = { git = "https://github.com/serai-dex/substrate", default-features = false } +sp-io = { git = "https://github.com/serai-dex/substrate", default-features = false } sp-application-crypto = { git = "https://github.com/serai-dex/substrate", default-features = false } sp-runtime = { git = "https://github.com/serai-dex/substrate", default-features = false } -sp-core = { git = "https://github.com/serai-dex/substrate", default-features = false } frame-system = { git = "https://github.com/serai-dex/substrate", default-features = false } frame-support = { git = "https://github.com/serai-dex/substrate", default-features = false } @@ -37,6 +38,9 @@ std = [ "scale/std", "scale-info/std", + "sp-core/std", + "sp-io/std", + "sp-application-crypto/std", "sp-runtime/std", "frame-system/std", diff --git a/substrate/in-instructions/pallet/src/lib.rs b/substrate/in-instructions/pallet/src/lib.rs index dc9961861..83b882d23 100644 --- a/substrate/in-instructions/pallet/src/lib.rs +++ b/substrate/in-instructions/pallet/src/lib.rs @@ -4,6 +4,7 @@ use scale::Encode; +use sp_io::hashing::blake2_256; use sp_runtime::RuntimeDebug; use serai_primitives::{BlockHash, NetworkId}; @@ -45,7 +46,7 @@ pub mod pallet { #[pallet::event] #[pallet::generate_deposit(fn deposit_event)] pub enum Event { - Batch { network: NetworkId, id: u32, block: BlockHash }, + Batch { network: NetworkId, id: u32, block: BlockHash, instructions_hash: [u8; 32] }, InstructionFailure { network: NetworkId, id: u32, index: u32 }, } @@ -123,6 +124,7 @@ pub mod pallet { network: batch.network, id: batch.id, block: batch.block, + instructions_hash: blake2_256(&batch.instructions.encode()), }); for (i, instruction) in batch.instructions.into_iter().enumerate() { // TODO: Check this balance's coin belongs to this network diff --git a/tests/coordinator/Cargo.toml b/tests/coordinator/Cargo.toml index ced0d65e7..d52789f11 100644 --- a/tests/coordinator/Cargo.toml +++ b/tests/coordinator/Cargo.toml @@ -19,12 +19,15 @@ hex = "0.4" zeroize = { version = "1", default-features = false } rand_core = { version = "0.6", default-features = false } +blake2 = "0.10" ciphersuite = { path = "../../crypto/ciphersuite", default-features = false, features = ["ristretto", "secp256k1"] } schnorrkel = "0.11" dkg = { path = "../../crypto/dkg", default-features = false, features = ["tests"] } messages = { package = "serai-processor-messages", path = "../../processor/messages" } +scale = { package = "parity-scale-codec", version = "3" } + serai-client = { path = "../../substrate/client", features = ["serai"] } serai-message-queue = { path = "../../message-queue" } diff --git a/tests/coordinator/src/lib.rs b/tests/coordinator/src/lib.rs index c68bdb577..afad025e2 100644 --- a/tests/coordinator/src/lib.rs +++ b/tests/coordinator/src/lib.rs @@ -200,7 +200,7 @@ impl Processor { Serai::new(&self.serai_rpc).await.unwrap() } - /// Send a message to a processor as its coordinator. + /// Send a message to the coordinator as a processor. pub async fn send_message(&mut self, msg: impl Into) { let msg: ProcessorMessage = msg.into(); self @@ -217,14 +217,14 @@ impl Processor { self.next_send_id += 1; } - /// Receive a message from a processor as its coordinator. + /// Receive a message from the coordinator as a processor. pub async fn recv_message(&mut self) -> CoordinatorMessage { - let msg = tokio::time::timeout(Duration::from_secs(10), self.queue.next(self.next_recv_id)) + let msg = tokio::time::timeout(Duration::from_secs(10), self.queue.next(Service::Coordinator)) .await .unwrap(); assert_eq!(msg.from, Service::Coordinator); assert_eq!(msg.id, self.next_recv_id); - self.queue.ack(self.next_recv_id).await; + self.queue.ack(Service::Coordinator, msg.id).await; self.next_recv_id += 1; serde_json::from_slice(&msg.msg).unwrap() } diff --git a/tests/coordinator/src/tests/batch.rs b/tests/coordinator/src/tests/batch.rs index e0257dc3a..ea41807cc 100644 --- a/tests/coordinator/src/tests/batch.rs +++ b/tests/coordinator/src/tests/batch.rs @@ -7,10 +7,15 @@ use std::{ use zeroize::Zeroizing; use rand_core::{RngCore, OsRng}; +use blake2::{ + digest::{consts::U32, Digest}, + Blake2b, +}; use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto, Secp256k1}; - use dkg::Participant; +use scale::Encode; + use serai_client::{ primitives::{NetworkId, BlockHash, Signature}, in_instructions::{ @@ -22,15 +27,24 @@ use messages::{sign::SignId, SubstrateContext, CoordinatorMessage}; use crate::{*, tests::*}; -pub async fn batch( +pub async fn batch( processors: &mut [Processor], substrate_key: &Zeroizing<::F>, - network_key: &Zeroizing, batch: Batch, ) -> u64 { let mut id = [0; 32]; OsRng.fill_bytes(&mut id); - let id = SignId { key: vec![], id, attempt: 0 }; + let id = SignId { + key: (::generator() * **substrate_key).to_bytes().to_vec(), + id, + attempt: 0, + }; + + for processor in processors.iter_mut() { + processor + .send_message(messages::substrate::ProcessorMessage::Batch { batch: batch.clone() }) + .await; + } // Select a random participant to exclude, so we know for sure who *is* participating assert_eq!(COORDINATORS - THRESHOLD, 1); @@ -165,7 +179,7 @@ pub async fn batch( for processor in processors.iter_mut() { processor - .send_message(messages::substrate::ProcessorMessage::Update { batch: batch.clone() }) + .send_message(messages::substrate::ProcessorMessage::SignedBatch { batch: batch.clone() }) .await; } @@ -191,7 +205,8 @@ pub async fn batch( InInstructionsEvent::Batch { network: batch.batch.network, id: batch.batch.id, - block: batch.batch.block + block: batch.batch.block, + instructions_hash: Blake2b::::digest(batch.batch.instructions.encode()).into(), } ); break 'outer; @@ -213,7 +228,6 @@ pub async fn batch( }, network: batch.batch.network, block: last_serai_block, - key: (C::generator() * **network_key).to_bytes().as_ref().to_vec(), burns: vec![], batches: vec![batch.batch.id], } @@ -257,11 +271,10 @@ async fn batch_test() { } let mut processors = new_processors; - let (substrate_key, network_key) = key_gen::(&mut processors).await; - batch::( + let (substrate_key, _) = key_gen::(&mut processors).await; + batch( &mut processors, &substrate_key, - &network_key, Batch { network: NetworkId::Bitcoin, id: 0, diff --git a/tests/coordinator/src/tests/sign.rs b/tests/coordinator/src/tests/sign.rs index 816caf87c..1a835b205 100644 --- a/tests/coordinator/src/tests/sign.rs +++ b/tests/coordinator/src/tests/sign.rs @@ -232,10 +232,9 @@ async fn sign_test() { let balance = Balance { coin: Coin::Bitcoin, amount }; let coin_block = BlockHash([0x33; 32]); - let block_included_in = batch::( + let block_included_in = batch( &mut processors, &substrate_key, - &network_key, Batch { network: NetworkId::Bitcoin, id: 0, @@ -346,7 +345,6 @@ async fn sign_test() { }, network: NetworkId::Bitcoin, block: last_serai_block.number(), - key: (Secp256k1::generator() * *network_key).to_bytes().to_vec(), burns: vec![OutInstructionWithBalance { instruction: out_instruction.clone(), balance: Balance { coin: Coin::Bitcoin, amount } diff --git a/tests/message-queue/src/lib.rs b/tests/message-queue/src/lib.rs index f1a548440..567cb244c 100644 --- a/tests/message-queue/src/lib.rs +++ b/tests/message-queue/src/lib.rs @@ -47,6 +47,7 @@ pub fn instance( hex::encode((Ristretto::generator() * priv_keys[&NetworkId::Monero]).to_bytes()), ), ("DB_PATH".to_string(), "./message-queue-db".to_string()), + ("RUST_LOG".to_string(), "serai_message_queue=trace,".to_string()), ] .into(), ); @@ -105,28 +106,54 @@ fn basic_functionality() { // Successfully get it let bitcoin = MessageQueue::new( Service::Processor(NetworkId::Bitcoin), - rpc, + rpc.clone(), Zeroizing::new(priv_keys[&NetworkId::Bitcoin]), ); - let msg = bitcoin.next(0).await; + let msg = bitcoin.next(Service::Coordinator).await; assert_eq!(msg.from, Service::Coordinator); assert_eq!(msg.id, 0); assert_eq!(&msg.msg, b"Hello, World!"); // If we don't ack it, it should continue to be returned - assert_eq!(msg, bitcoin.next(0).await); + assert_eq!(msg, bitcoin.next(Service::Coordinator).await); // Acknowledging it should yield the next message - bitcoin.ack(0).await; + bitcoin.ack(Service::Coordinator, 0).await; - let next_msg = bitcoin.next(1).await; + let next_msg = bitcoin.next(Service::Coordinator).await; assert!(msg != next_msg); assert_eq!(next_msg.from, Service::Coordinator); assert_eq!(next_msg.id, 1); assert_eq!(&next_msg.msg, b"Hello, World, again!"); - bitcoin.ack(1).await; + bitcoin.ack(Service::Coordinator, 1).await; // No further messages should be available - tokio::time::timeout(core::time::Duration::from_secs(10), bitcoin.next(2)).await.unwrap_err(); + tokio::time::timeout(core::time::Duration::from_secs(10), bitcoin.next(Service::Coordinator)) + .await + .unwrap_err(); + + // Queueing to a distinct processor should work, with a unique ID + coordinator + .queue( + Metadata { + from: Service::Coordinator, + to: Service::Processor(NetworkId::Monero), + // Intents should be per-from-to, making this valid + intent: b"intent".to_vec(), + }, + b"Hello, World!".to_vec(), + ) + .await; + + let monero = MessageQueue::new( + Service::Processor(NetworkId::Monero), + rpc, + Zeroizing::new(priv_keys[&NetworkId::Monero]), + ); + assert_eq!(monero.next(Service::Coordinator).await.id, 0); + monero.ack(Service::Coordinator, 0).await; + tokio::time::timeout(core::time::Duration::from_secs(10), monero.next(Service::Coordinator)) + .await + .unwrap_err(); }); } diff --git a/tests/processor/src/lib.rs b/tests/processor/src/lib.rs index 8004e81df..b924acdf2 100644 --- a/tests/processor/src/lib.rs +++ b/tests/processor/src/lib.rs @@ -54,6 +54,7 @@ pub fn processor_instance( ("NETWORK_RPC_LOGIN".to_string(), format!("{RPC_USER}:{RPC_PASS}")), ("NETWORK_RPC_PORT".to_string(), port.to_string()), ("DB_PATH".to_string(), "./processor-db".to_string()), + ("RUST_LOG".to_string(), "serai_processor=trace,".to_string()), ] .into(), ) @@ -221,13 +222,15 @@ impl Coordinator { /// Receive a message from a processor as its coordinator. pub async fn recv_message(&mut self) -> ProcessorMessage { - let msg = - tokio::time::timeout(core::time::Duration::from_secs(10), self.queue.next(self.next_recv_id)) - .await - .unwrap(); + let msg = tokio::time::timeout( + core::time::Duration::from_secs(10), + self.queue.next(Service::Processor(self.network)), + ) + .await + .unwrap(); assert_eq!(msg.from, Service::Processor(self.network)); assert_eq!(msg.id, self.next_recv_id); - self.queue.ack(self.next_recv_id).await; + self.queue.ack(Service::Processor(self.network), msg.id).await; self.next_recv_id += 1; serde_json::from_slice(&msg.msg).unwrap() } diff --git a/tests/processor/src/tests/batch.rs b/tests/processor/src/tests/batch.rs index 71fd2a535..b78836649 100644 --- a/tests/processor/src/tests/batch.rs +++ b/tests/processor/src/tests/batch.rs @@ -10,7 +10,7 @@ use messages::{sign::SignId, SubstrateContext}; use serai_client::{ primitives::{BlockHash, crypto::RuntimePublic, PublicKey, SeraiAddress, NetworkId}, in_instructions::primitives::{ - InInstruction, InInstructionWithBalance, SignedBatch, batch_message, + InInstruction, InInstructionWithBalance, Batch, SignedBatch, batch_message, }, }; @@ -18,6 +18,8 @@ use crate::{*, tests::*}; pub(crate) async fn recv_batch_preprocesses( coordinators: &mut [Coordinator], + substrate_key: &[u8; 32], + batch: &Batch, attempt: u32, ) -> (SignId, HashMap>) { let mut id = None; @@ -26,8 +28,18 @@ pub(crate) async fn recv_batch_preprocesses( for (i, coordinator) in coordinators.iter_mut().enumerate() { let i = Participant::new(u16::try_from(i).unwrap() + 1).unwrap(); - let msg = coordinator.recv_message().await; - match msg { + if attempt == 0 { + match coordinator.recv_message().await { + messages::ProcessorMessage::Substrate(messages::substrate::ProcessorMessage::Batch { + batch: sent_batch, + }) => { + assert_eq!(&sent_batch, batch); + } + _ => panic!("processor didn't send batch"), + } + } + + match coordinator.recv_message().await { messages::ProcessorMessage::Coordinator( messages::coordinator::ProcessorMessage::BatchPreprocess { id: this_id, @@ -36,7 +48,7 @@ pub(crate) async fn recv_batch_preprocesses( }, ) => { if id.is_none() { - assert!(this_id.key.is_empty()); + assert_eq!(&this_id.key, substrate_key); assert_eq!(this_id.attempt, attempt); id = Some(this_id.clone()); block = Some(this_block); @@ -121,9 +133,9 @@ pub(crate) async fn sign_batch( if preprocesses.contains_key(&i) { match coordinator.recv_message().await { - messages::ProcessorMessage::Substrate(messages::substrate::ProcessorMessage::Update { - batch: this_batch, - }) => { + messages::ProcessorMessage::Substrate( + messages::substrate::ProcessorMessage::SignedBatch { batch: this_batch }, + ) => { if batch.is_none() { assert!(PublicKey::from_raw(key) .verify(&batch_message(&this_batch.batch), &this_batch.signature)); @@ -149,7 +161,6 @@ pub(crate) async fn substrate_block( context: _, network: sent_network, block: sent_block, - key: _, burns: _, batches: _, } => { @@ -231,8 +242,23 @@ fn batch_test() { // The scanner works on a 5s interval, so this leaves a few s for any processing/latency tokio::time::sleep(Duration::from_secs(10)).await; - // Make sure the proceessors picked it up by checking they're trying to sign a batch for it - let (mut id, mut preprocesses) = recv_batch_preprocesses(&mut coordinators, 0).await; + let expected_batch = Batch { + network, + id: i, + block: BlockHash(block_with_tx.unwrap()), + instructions: if let Some(instruction) = instruction { + vec![InInstructionWithBalance { instruction, balance: balance_sent }] + } else { + // This shouldn't have an instruction as we didn't add any data into the TX we sent + // Empty batches remain valuable as they let us achieve consensus on the block and spend + // contained outputs + vec![] + }, + }; + + // Make sure the processors picked it up by checking they're trying to sign a batch for it + let (mut id, mut preprocesses) = + recv_batch_preprocesses(&mut coordinators, &key_pair.0 .0, &expected_batch, 0).await; // Trigger a random amount of re-attempts for attempt in 1 ..= u32::try_from(OsRng.next_u64() % 4).unwrap() { // TODO: Double check how the processor handles this ID field @@ -245,27 +271,16 @@ fn batch_test() { }) .await; } - (id, preprocesses) = recv_batch_preprocesses(&mut coordinators, attempt).await; + (id, preprocesses) = + recv_batch_preprocesses(&mut coordinators, &key_pair.0 .0, &expected_batch, attempt) + .await; } // Continue with signing the batch let batch = sign_batch(&mut coordinators, key_pair.0 .0, id, preprocesses).await; // Check it - assert_eq!(batch.batch.network, network); - assert_eq!(batch.batch.id, i); - assert_eq!(batch.batch.block, BlockHash(block_with_tx.unwrap())); - if let Some(instruction) = instruction { - assert_eq!( - batch.batch.instructions, - vec![InInstructionWithBalance { instruction, balance: balance_sent }] - ); - } else { - // This shouldn't have an instruction as we didn't add any data into the TX we sent - // Empty batches remain valuable as they let us achieve consensus on the block and spend - // contained outputs - assert!(batch.batch.instructions.is_empty()); - } + assert_eq!(batch.batch, expected_batch); // Fire a SubstrateBlock let serai_time = @@ -280,8 +295,6 @@ fn batch_test() { }, network, block: substrate_block_num + u64::from(i), - // TODO: Should we use the network key here? Or should we only use the Ristretto key? - key: key_pair.1.to_vec(), burns: vec![], batches: vec![batch.batch.id], }, diff --git a/tests/processor/src/tests/send.rs b/tests/processor/src/tests/send.rs index 6a32a8ae4..67703b99b 100644 --- a/tests/processor/src/tests/send.rs +++ b/tests/processor/src/tests/send.rs @@ -9,6 +9,7 @@ use messages::{sign::SignId, SubstrateContext}; use serai_client::{ primitives::{BlockHash, NetworkId}, + in_instructions::primitives::Batch, tokens::primitives::{OutInstruction, OutInstructionWithBalance}, }; @@ -186,17 +187,18 @@ fn send_test() { // The scanner works on a 5s interval, so this leaves a few s for any processing/latency tokio::time::sleep(Duration::from_secs(10)).await; + let expected_batch = + Batch { network, id: 0, block: BlockHash(block_with_tx.unwrap()), instructions: vec![] }; + // Make sure the proceessors picked it up by checking they're trying to sign a batch for it - let (id, preprocesses) = recv_batch_preprocesses(&mut coordinators, 0).await; + let (id, preprocesses) = + recv_batch_preprocesses(&mut coordinators, &key_pair.0 .0, &expected_batch, 0).await; // Continue with signing the batch let batch = sign_batch(&mut coordinators, key_pair.0 .0, id, preprocesses).await; // Check it - assert_eq!(batch.batch.network, network); - assert_eq!(batch.batch.id, 0); - assert_eq!(batch.batch.block, BlockHash(block_with_tx.unwrap())); - assert!(batch.batch.instructions.is_empty()); + assert_eq!(batch.batch, expected_batch); // Fire a SubstrateBlock with a burn let substrate_block_num = (OsRng.next_u64() % 4_000_000_000u64) + 1; @@ -213,8 +215,6 @@ fn send_test() { }, network, block: substrate_block_num, - // TODO: Should we use the network key here? Or should we only use the Ristretto key? - key: key_pair.1.to_vec(), burns: vec![OutInstructionWithBalance { instruction: OutInstruction { address: wallet.address(), data: None }, balance: balance_sent,