diff --git a/.github/actions/bitcoin/action.yml b/.github/actions/bitcoin/action.yml index e15cc26e9..903108680 100644 --- a/.github/actions/bitcoin/action.yml +++ b/.github/actions/bitcoin/action.yml @@ -37,11 +37,4 @@ runs: - name: Bitcoin Regtest Daemon shell: bash - run: | - RPC_USER=serai - RPC_PASS=seraidex - - bitcoind -txindex -regtest \ - -rpcuser=$RPC_USER -rpcpassword=$RPC_PASS \ - -rpcbind=127.0.0.1 -rpcbind=$(hostname) -rpcallowip=0.0.0.0/0 \ - -daemon + run: PATH=$PATH:/usr/bin ./orchestration/dev/coins/bitcoin/run.sh -daemon diff --git a/.github/actions/monero-wallet-rpc/action.yml b/.github/actions/monero-wallet-rpc/action.yml index 0e8aa7c79..3192bb95b 100644 --- a/.github/actions/monero-wallet-rpc/action.yml +++ b/.github/actions/monero-wallet-rpc/action.yml @@ -5,7 +5,7 @@ inputs: version: description: "Version to download and run" required: false - default: v0.18.2.0 + default: v0.18.3.1 runs: using: "composite" @@ -41,4 +41,9 @@ runs: - name: Monero Wallet RPC shell: bash - run: ./monero-wallet-rpc --disable-rpc-login --rpc-bind-port 6061 --allow-mismatched-daemon-version --wallet-dir ./ --detach + run: | + ./monero-wallet-rpc --allow-mismatched-daemon-version \ + --daemon-address 0.0.0.0:18081 --daemon-login serai:seraidex \ + --disable-rpc-login --rpc-bind-port 18082 \ + --wallet-dir ./ \ + --detach diff --git a/.github/actions/monero/action.yml b/.github/actions/monero/action.yml index cb80c841f..8dff093a5 100644 --- a/.github/actions/monero/action.yml +++ b/.github/actions/monero/action.yml @@ -5,7 +5,7 @@ inputs: version: description: "Version to download and run" required: false - default: v0.18.2.0 + default: v0.18.3.1 runs: using: "composite" @@ -14,7 +14,7 @@ runs: id: cache-monerod uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 with: - path: monerod + path: /usr/bin/monerod key: monerod-${{ runner.os }}-${{ runner.arch }}-${{ inputs.version }} - name: Download the Monero Daemon @@ -37,8 +37,10 @@ runs: wget https://downloads.getmonero.org/cli/$FILE tar -xvf $FILE - mv monero-x86_64-linux-gnu-${{ inputs.version }}/monerod monerod + sudo mv monero-x86_64-linux-gnu-${{ inputs.version }}/monerod /usr/bin/monerod + sudo chmod 777 /usr/bin/monerod + sudo chmod +x /usr/bin/monerod - name: Monero Regtest Daemon shell: bash - run: ./monerod --regtest --offline --fixed-difficulty=1 --detach + run: PATH=$PATH:/usr/bin ./orchestration/dev/coins/monero/run.sh --detach diff --git a/.github/actions/test-dependencies/action.yml b/.github/actions/test-dependencies/action.yml index e4492dbb6..a19e17046 100644 --- a/.github/actions/test-dependencies/action.yml +++ b/.github/actions/test-dependencies/action.yml @@ -5,7 +5,7 @@ inputs: monero-version: description: "Monero version to download and run as a regtest node" required: false - default: v0.18.2.0 + default: v0.18.3.1 bitcoin-version: description: "Bitcoin version to download and run as a regtest node" diff --git a/.github/nightly-version b/.github/nightly-version index d7aace133..4a8f1e33b 100644 --- a/.github/nightly-version +++ b/.github/nightly-version @@ -1 +1 @@ -nightly-2024-02-01 +nightly-2024-02-07 diff --git a/.github/workflows/coordinator-tests.yml b/.github/workflows/coordinator-tests.yml index cf32e9305..7cc4d7b36 100644 --- a/.github/workflows/coordinator-tests.yml +++ b/.github/workflows/coordinator-tests.yml @@ -9,9 +9,8 @@ on: - "crypto/**" - "coins/**" - "message-queue/**" - - "orchestration/message-queue/**" - "coordinator/**" - - "orchestration/coordinator/**" + - "orchestration/**" - "tests/docker/**" - "tests/coordinator/**" @@ -21,9 +20,8 @@ on: - "crypto/**" - "coins/**" - "message-queue/**" - - "orchestration/message-queue/**" - "coordinator/**" - - "orchestration/coordinator/**" + - "orchestration/**" - "tests/docker/**" - "tests/coordinator/**" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 707112649..2f74aba27 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -73,14 +73,6 @@ jobs: - name: Run rustfmt run: cargo +${{ steps.nightly.outputs.version }} fmt -- --check - dockerfiles: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac - - name: Verify Dockerfiles are up to date - # Runs the file which generates them and checks the diff has no lines - run: cd orchestration && ./dockerfiles.sh && git diff | wc -l | grep -x "0" - machete: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/message-queue-tests.yml b/.github/workflows/message-queue-tests.yml index e6a5cfbf3..273af237f 100644 --- a/.github/workflows/message-queue-tests.yml +++ b/.github/workflows/message-queue-tests.yml @@ -8,7 +8,7 @@ on: - "common/**" - "crypto/**" - "message-queue/**" - - "orchestration/message-queue/**" + - "orchestration/**" - "tests/docker/**" - "tests/message-queue/**" @@ -17,7 +17,7 @@ on: - "common/**" - "crypto/**" - "message-queue/**" - - "orchestration/message-queue/**" + - "orchestration/**" - "tests/docker/**" - "tests/message-queue/**" diff --git a/.github/workflows/processor-tests.yml b/.github/workflows/processor-tests.yml index f124cecea..88f4429cf 100644 --- a/.github/workflows/processor-tests.yml +++ b/.github/workflows/processor-tests.yml @@ -9,9 +9,8 @@ on: - "crypto/**" - "coins/**" - "message-queue/**" - - "orchestration/message-queue/**" - "processor/**" - - "orchestration/processor/**" + - "orchestration/**" - "tests/docker/**" - "tests/processor/**" @@ -21,9 +20,8 @@ on: - "crypto/**" - "coins/**" - "message-queue/**" - - "orchestration/message-queue/**" - "processor/**" - - "orchestration/processor/**" + - "orchestration/**" - "tests/docker/**" - "tests/processor/**" diff --git a/.gitignore b/.gitignore index c4220b672..b8df85f9a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ target -.vscode +Dockerfile +!orchestration/runtime/Dockerfile .test-logs + +.vscode diff --git a/Cargo.lock b/Cargo.lock index 11791cf14..b1d9755c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -139,9 +139,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2faccea4cc4ab4a667ce676a30e8ec13922a692c99bb8f5b11f1502c72e04220" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" @@ -747,9 +747,9 @@ checksum = "c3ac9f8b63eca6fd385229b3675f6cc0dc5c8a5c8a54a59d4f52ffd670d87b0c" [[package]] name = "bytemuck" -version = "1.14.1" +version = "1.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2490600f404f2b94c167e31d3ed1d5f3c225a0f3b80230053b3e0b7b962bd9" +checksum = "ea31d69bda4949c1c1562c1e6f042a1caefac98cdc8a298260a2ff41c1e2d42b" [[package]] name = "byteorder" @@ -1289,9 +1289,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.1" +version = "4.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89b8c6a2e4b1f45971ad09761aafb85514a84744b67a95e32c3cc1352d1f65c" +checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348" dependencies = [ "cfg-if", "cpufeatures", @@ -1675,9 +1675,9 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f628eaec48bfd21b865dc2950cfa014450c01d2fa2b69a86c2fd5844ec523c0" +checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" dependencies = [ "curve25519-dalek", "ed25519", @@ -2114,9 +2114,9 @@ dependencies = [ [[package]] name = "fiat-crypto" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27573eac26f4dd11e2b1916c3fe1baa56407c83c71a773a8ba17ec0bca03b6b7" +checksum = "1676f435fc1dadde4d03e43f5d62b259e1ce5f40bd4ffb21db2b42ebe59c1382" [[package]] name = "file-per-thread-logger" @@ -2783,9 +2783,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" +checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" [[package]] name = "hex" @@ -3271,9 +3271,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" dependencies = [ "wasm-bindgen", ] @@ -4694,9 +4694,9 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214" +checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" dependencies = [ "num-traits", ] @@ -4713,11 +4713,10 @@ dependencies = [ [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] @@ -4735,9 +4734,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", "libm", @@ -5291,9 +5290,9 @@ checksum = "626dec3cac7cc0e1577a2ec3fc496277ec2baa084bebad95bb6fdbfae235f84c" [[package]] name = "polling" -version = "3.3.2" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545c980a3880efd47b2e262f6a4bb6daad6555cf3367aa9c4e52895f69537a41" +checksum = "30054e72317ab98eddd8561db0f6524df3367636884b7b21b703e4b280a84a14" dependencies = [ "cfg-if", "concurrent-queue", @@ -7629,8 +7628,10 @@ dependencies = [ "clap", "frame-benchmarking", "futures-util", + "hex", "jsonrpsee", "pallet-transaction-payment-rpc", + "rand_core", "sc-authority-discovery", "sc-basic-authorship", "sc-cli", @@ -7647,6 +7648,8 @@ dependencies = [ "sc-telemetry", "sc-transaction-pool", "sc-transaction-pool-api", + "schnorrkel", + "serai-env", "serai-runtime", "sp-api", "sp-block-builder", @@ -7654,10 +7657,26 @@ dependencies = [ "sp-consensus-babe", "sp-core", "sp-io", + "sp-keystore", "sp-timestamp", "substrate-build-script-utils", "substrate-frame-rpc-system", "tokio", + "zeroize", +] + +[[package]] +name = "serai-orchestrator" +version = "0.0.1" +dependencies = [ + "ciphersuite", + "flexible-transcript", + "hex", + "home", + "rand_chacha", + "rand_core", + "zalloc", + "zeroize", ] [[package]] @@ -9069,13 +9088,12 @@ checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" [[package]] name = "tempfile" -version = "3.9.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.4.1", "rustix", "windows-sys 0.52.0", ] @@ -9804,9 +9822,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -9814,9 +9832,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" dependencies = [ "bumpalo", "log", @@ -9829,9 +9847,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" +checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" dependencies = [ "cfg-if", "js-sys", @@ -9841,9 +9859,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9851,9 +9869,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", @@ -9864,9 +9882,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" [[package]] name = "wasm-encoder" @@ -10171,9 +10189,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" dependencies = [ "js-sys", "wasm-bindgen", @@ -10406,9 +10424,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.37" +version = "0.5.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7cad8365489051ae9f054164e459304af2e7e9bb407c958076c8bf4aef52da5" +checksum = "5389a154b01683d28c77f8f68f49dea75f0a4da32557a58f68ee51ebba472d29" dependencies = [ "memchr", ] @@ -10453,9 +10471,9 @@ dependencies = [ [[package]] name = "x25519-dalek" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb66477291e7e8d2b0ff1bcb900bf29489a9692816d79874bea351e7a8b6de96" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" dependencies = [ "curve25519-dalek", "rand_core", @@ -10569,10 +10587,8 @@ dependencies = [ [[package]] name = "zstd" version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe 5.0.2+zstd.1.5.2", + "zstd 0.12.4", ] [[package]] @@ -10581,17 +10597,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" dependencies = [ - "zstd-safe 6.0.6", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", + "zstd-safe", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cfdf13ebd..25601c46e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,18 @@ [workspace] resolver = "2" members = [ + # Version patches + "patches/zstd", + "patches/proc-macro-crate", + + # std patches + "patches/matches", + "patches/is-terminal", + + # Rewrites/redirects + "patches/option-ext", + "patches/directories-next", + "common/std-shims", "common/zalloc", "common/db", @@ -57,6 +69,8 @@ members = [ "substrate/client", + "orchestration", + "mini", "tests/no-std", @@ -67,12 +81,6 @@ members = [ "tests/coordinator", "tests/full-stack", "tests/reproducible-runtime", - - "patches/is-terminal", - "patches/matches", - "patches/option-ext", - "patches/directories-next", - "patches/proc-macro-crate", ] # Always compile Monero (and a variety of dependencies) with optimizations due @@ -102,6 +110,11 @@ lazy_static = { git = "https://github.com/rust-lang-nursery/lazy-static.rs", rev # Needed due to dockertest's usage of `Rc`s when we need `Arc`s dockertest = { git = "https://github.com/kayabaNerve/dockertest-rs", branch = "arc" } +# wasmtime pulls in an old version for this +zstd = { path = "patches/zstd" } +# proc-macro-crate 2 binds to an old version of toml for msrv so we patch to 3 +proc-macro-crate = { path = "patches/proc-macro-crate" } + # is-terminal now has an std-based solution with an equivalent API is-terminal = { path = "patches/is-terminal" } # So does matches @@ -115,9 +128,6 @@ matches = { path = "patches/matches" } option-ext = { path = "patches/option-ext" } directories-next = { path = "patches/directories-next" } -# proc-macro-crate 2 binds to an old version of toml for msrv so we patch to 3 -proc-macro-crate = { path = "patches/proc-macro-crate" } - [workspace.lints.clippy] unwrap_or_default = "allow" borrow_as_ptr = "deny" diff --git a/coins/bitcoin/tests/runner.rs b/coins/bitcoin/tests/runner.rs index 6491d5384..bc471279a 100644 --- a/coins/bitcoin/tests/runner.rs +++ b/coins/bitcoin/tests/runner.rs @@ -12,7 +12,7 @@ pub fn SEQUENTIAL() -> &'static Mutex<()> { #[allow(dead_code)] pub(crate) async fn rpc() -> Rpc { - let rpc = Rpc::new("http://serai:seraidex@127.0.0.1:18443".to_string()).await.unwrap(); + let rpc = Rpc::new("http://serai:seraidex@127.0.0.1:8332".to_string()).await.unwrap(); // If this node has already been interacted with, clear its chain if rpc.get_latest_block_number().await.unwrap() > 0 { diff --git a/coins/monero/tests/runner.rs b/coins/monero/tests/runner.rs index fa0b48df4..ff80656e7 100644 --- a/coins/monero/tests/runner.rs +++ b/coins/monero/tests/runner.rs @@ -86,7 +86,7 @@ pub fn check_weight_and_fee(tx: &Transaction, fee_rate: Fee) { } pub async fn rpc() -> Rpc { - let rpc = HttpRpc::new("http://127.0.0.1:18081".to_string()).await.unwrap(); + let rpc = HttpRpc::new("http://serai:seraidex@127.0.0.1:18081".to_string()).await.unwrap(); // Only run once if rpc.get_height().await.unwrap() != 1 { diff --git a/coins/monero/tests/wallet2_compatibility.rs b/coins/monero/tests/wallet2_compatibility.rs index 4b638c248..2554325fe 100644 --- a/coins/monero/tests/wallet2_compatibility.rs +++ b/coins/monero/tests/wallet2_compatibility.rs @@ -35,7 +35,7 @@ async fn make_integrated_address(rpc: &Rpc, payment_id: [u8; 8]) -> Str } async fn initialize_rpcs() -> (Rpc, Rpc, String) { - let wallet_rpc = HttpRpc::new("http://127.0.0.1:6061".to_string()).await.unwrap(); + let wallet_rpc = HttpRpc::new("http://127.0.0.1:18082".to_string()).await.unwrap(); let daemon_rpc = runner::rpc().await; #[derive(Debug, Deserialize)] diff --git a/common/env/src/lib.rs b/common/env/src/lib.rs index 7b5cd049d..bace84fd3 100644 --- a/common/env/src/lib.rs +++ b/common/env/src/lib.rs @@ -3,6 +3,7 @@ // Obtain a variable from the Serai environment/secret store. pub fn var(variable: &str) -> Option { - // TODO: Move this to Kubernetes + // TODO: Move this to a proper secret store + // TODO: Unset this variable std::env::var(variable).ok() } diff --git a/deny.toml b/deny.toml index 1fe2cd86e..2e516b996 100644 --- a/deny.toml +++ b/deny.toml @@ -65,6 +65,8 @@ exceptions = [ { allow = ["AGPL-3.0"], name = "serai-runtime" }, { allow = ["AGPL-3.0"], name = "serai-node" }, + { allow = ["AGPL-3.0"], name = "serai-orchestrator" }, + { allow = ["AGPL-3.0"], name = "mini-serai" }, { allow = ["AGPL-3.0"], name = "serai-docker-tests" }, diff --git a/docs/Getting Started.md b/docs/Getting Started.md index 4500efc1e..0034d69d4 100644 --- a/docs/Getting Started.md +++ b/docs/Getting Started.md @@ -70,24 +70,22 @@ Running tests requires: - A properly configured Bitcoin regtest node (available via Docker) - A properly configured Monero regtest node (available via Docker) - A properly configured monero-wallet-rpc instance (available via Docker) -- A debug Serai node (`cd substrate/node && cargo build`) + +To start the required daemons, one may run: ``` -cargo test --all-features +cargo run -p serai-orchestrator -- key_gen dev +cargo run -p serai-orchestrator -- setup dev ``` -### Run Serai in Development Mode +and then: ``` -./target/release/serai-node --dev +cargo run -p serai-orchestrator -- start dev bitcoin-daemon monero-daemon monero-wallet-rpc ``` -### Run Serai with Orchestration - -Under `/orchestration`, you can find our orchestration components for running -the entire infrastructure of Serai in a local environment using Docker Compose -or Kubernetes. - -[Run Serai with Docker Compose](../orchestration/README.md) +Finally, to run the tests: -[Run Serai with Kubernetes](../orchestration/kubernetes/README.md) +``` +cargo test --all-features +``` diff --git a/orchestration/Cargo.toml b/orchestration/Cargo.toml new file mode 100644 index 000000000..fca380661 --- /dev/null +++ b/orchestration/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "serai-orchestrator" +version = "0.0.1" +description = "Generates Dockerfiles for Serai" +license = "AGPL-3.0-only" +repository = "https://github.com/serai-dex/serai/tree/develop/orchestration/" +authors = ["Luke Parker "] +keywords = [] +edition = "2021" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[lints] +workspace = true + +[dependencies] +hex = { version = "0.4", default-features = false, features = ["std"] } + +zeroize = { version = "1", default-features = false, features = ["std"] } +rand_core = { version = "0.6", default-features = false, features = ["std", "getrandom"] } +rand_chacha = { version = "0.3", default-features = false, features = ["std"] } + +transcript = { package = "flexible-transcript", path = "../crypto/transcript", default-features = false, features = ["std", "recommended"] } +ciphersuite = { path = "../crypto/ciphersuite", default-features = false, features = ["std", "ristretto"] } + +zalloc = { path = "../common/zalloc" } + +home = "0.5" diff --git a/orchestration/Dockerfile.parts/Dockerfile.alpine.start b/orchestration/Dockerfile.parts/Dockerfile.alpine.start deleted file mode 100644 index 0a51fc6a3..000000000 --- a/orchestration/Dockerfile.parts/Dockerfile.alpine.start +++ /dev/null @@ -1,6 +0,0 @@ -FROM alpine:latest as image - -COPY --from=mimalloc libmimalloc.so /usr/lib -ENV LD_PRELOAD=libmimalloc.so - -RUN apk update && apk upgrade diff --git a/orchestration/Dockerfile.parts/Dockerfile.debian.start b/orchestration/Dockerfile.parts/Dockerfile.debian.start deleted file mode 100644 index c22664686..000000000 --- a/orchestration/Dockerfile.parts/Dockerfile.debian.start +++ /dev/null @@ -1,6 +0,0 @@ -FROM debian:bookworm-slim as image - -COPY --from=mimalloc libmimalloc.so /usr/lib -RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload - -RUN apt update && apt upgrade -y && apt autoremove -y && apt clean diff --git a/orchestration/Dockerfile.parts/Dockerfile.serai.build b/orchestration/Dockerfile.parts/Dockerfile.serai.build deleted file mode 100644 index 8bc5a498d..000000000 --- a/orchestration/Dockerfile.parts/Dockerfile.serai.build +++ /dev/null @@ -1,39 +0,0 @@ -FROM rust:1.75-slim-bookworm as builder - -COPY --from=mimalloc libmimalloc.so /usr/lib -RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload - -RUN apt update && apt upgrade -y && apt autoremove -y && apt clean - -# Add dev dependencies -RUN apt install -y pkg-config clang - -# Dependencies for the Serai node -RUN apt install -y make protobuf-compiler - -# Add the wasm toolchain -RUN rustup target add wasm32-unknown-unknown - -# Add files for build -ADD common /serai/common -ADD crypto /serai/crypto -ADD coins /serai/coins -ADD message-queue /serai/message-queue -ADD processor /serai/processor -ADD coordinator /serai/coordinator -ADD substrate /serai/substrate -ADD mini /serai/mini -ADD tests /serai/tests -ADD patches /serai/patches -ADD Cargo.toml /serai -ADD Cargo.lock /serai -ADD AGPL-3.0 /serai - -WORKDIR /serai - -# Mount the caches and build -RUN --mount=type=cache,target=/root/.cargo \ - --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/usr/local/cargo/git \ - --mount=type=cache,target=/serai/target \ - mkdir /serai/bin && \ diff --git a/orchestration/Dockerfile.parts/mimalloc/Dockerfile.alpine b/orchestration/Dockerfile.parts/mimalloc/Dockerfile.alpine deleted file mode 100644 index 913284352..000000000 --- a/orchestration/Dockerfile.parts/mimalloc/Dockerfile.alpine +++ /dev/null @@ -1,11 +0,0 @@ -FROM alpine:latest as mimalloc - -RUN apk update && apk upgrade && apk --no-cache add gcc g++ libc-dev make cmake git -RUN git clone https://github.com/microsoft/mimalloc && \ - cd mimalloc && \ - git checkout 43ce4bd7fd34bcc730c1c7471c99995597415488 && \ - mkdir -p out/secure && \ - cd out/secure && \ - cmake -DMI_SECURE=ON ../.. && \ - make && \ - cp ./libmimalloc-secure.so ../../../libmimalloc.so diff --git a/orchestration/Dockerfile.parts/mimalloc/Dockerfile.debian b/orchestration/Dockerfile.parts/mimalloc/Dockerfile.debian deleted file mode 100644 index def8ad82d..000000000 --- a/orchestration/Dockerfile.parts/mimalloc/Dockerfile.debian +++ /dev/null @@ -1,11 +0,0 @@ -FROM debian:bookworm-slim as mimalloc - -RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git -RUN git clone https://github.com/microsoft/mimalloc && \ - cd mimalloc && \ - git checkout 43ce4bd7fd34bcc730c1c7471c99995597415488 && \ - mkdir -p out/secure && \ - cd out/secure && \ - cmake -DMI_SECURE=ON ../.. && \ - make && \ - cp ./libmimalloc-secure.so ../../../libmimalloc.so diff --git a/orchestration/README.md b/orchestration/README.md index fb307a2e9..b502da10b 100644 --- a/orchestration/README.md +++ b/orchestration/README.md @@ -1,65 +1,12 @@ -# Deploy +# Orchestration -## Run with Docker Compose +This folder contains the tool which generates various dockerfiles and manage +deployments of Serai. -Running the Serai infrastructure is easy with Docker. +To start, run: -We utilize compose profiles to easily orchestrate various pieces of the -infrastructure. +```sh +cargo run -p serai-orchestrator +``` -**Example:** `docker compose --profile cluster-coins-sm up` - -All commands are assumed to be ran from `/deploy`, not the root folder. - -### Profiles: - -* `bitcoin` - Bitcoin node -* `monero` - Monero node -* `ethereum` - Ethereum node -* `coins` - Nodes for all external networks (BTC, ETH, XMR) - -* `message-queue` - The message queue service. -* `processor` - Serai processor for one external network. -* `coordinator` - Serai coordinator for the entire Serai stack. - -* `serai` - Serai node -* `cluster-sm` - "Alice", "Bob", "Charlie", and "Dave" Serai nodes, all as - validators (enough to achieve BFT with one faulty node) -* `cluster-lg` - `cluster-sm` with non-validators "Eve" and "Ferdie" - -You can supply one or more profiles to the docker compose command to orchestrate -the desired components. - -**Example:** `docker compose --profile coins --profile serai up` - -## Orchestration Approach - -### Builds - -The Serai infrastructure is locally compiled. This may take several minutes. - -Images for external networks download binaries, before verifying their checksums -and signatures. - -**Stage 1 -- Builder** -* Configure environment. -* Get the binary. -* Verify binary using GPG. -* Decompress binary to prepare image. - -**Stage 2 -- Image** -* Copy needed files from builder. -* Move executables to bin folder. -* Copy scripts folder. -* Expose necessary ports. -* Map necessary volumes. - -### Entrypoint - -The Serai node and external networks' nodes are each started from an entrypoint -script inside the `/scripts `folder. - -To update the scripts on the image you must rebuild the updated images using the -`--build` flag after `up` in `docker compose`. - -**Example:** `docker compose --profile bitcoin up --build` +to generate all of the dockerfiles needed for development. diff --git a/orchestration/coins/bitcoin/Dockerfile.bitcoin b/orchestration/coins/bitcoin/Dockerfile.bitcoin deleted file mode 100644 index b2aca0cb5..000000000 --- a/orchestration/coins/bitcoin/Dockerfile.bitcoin +++ /dev/null @@ -1,22 +0,0 @@ -FROM alpine:latest as bitcoin - -ENV BITCOIN_VERSION=26.0 - -RUN apk --no-cache add git gnupg - -# Download Bitcoin -RUN wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/bitcoin-${BITCOIN_VERSION}-$(uname -m)-linux-gnu.tar.gz \ - && wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS \ - && wget https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/SHA256SUMS.asc - -# Verify all sigs and check for a valid signature from laanwj -- 71A3 -RUN git clone https://github.com/bitcoin-core/guix.sigs && \ - cd guix.sigs/builder-keys && \ - find . -iname '*.gpg' -exec gpg --import {} \; && \ - gpg --verify --status-fd 1 --verify ../../SHA256SUMS.asc ../../SHA256SUMS | grep "^\[GNUPG:\] VALIDSIG.*71A3B16735405025D447E8F274810B012346C9A6" - -RUN grep bitcoin-${BITCOIN_VERSION}-$(uname -m)-linux-gnu.tar.gz SHA256SUMS | sha256sum -c - -# Prepare Image -RUN tar xzvf bitcoin-${BITCOIN_VERSION}-$(uname -m)-linux-gnu.tar.gz -RUN mv bitcoin-${BITCOIN_VERSION}/bin/bitcoind . diff --git a/orchestration/coins/bitcoin/Dockerfile.bitcoin.end b/orchestration/coins/bitcoin/Dockerfile.bitcoin.end deleted file mode 100644 index 8c6a77fc0..000000000 --- a/orchestration/coins/bitcoin/Dockerfile.bitcoin.end +++ /dev/null @@ -1,10 +0,0 @@ -# Switch to a non-root user -RUN useradd --system --create-home --shell /sbin/nologin bitcoin -USER bitcoin -WORKDIR /home/bitcoin - -COPY --from=bitcoin --chown=bitcoin bitcoind /bin -COPY ./scripts /scripts - -EXPOSE 8332 8333 18332 18333 18443 18444 -# VOLUME ["/home/bitcoin/.bitcoin"] diff --git a/orchestration/coins/bitcoin/scripts/entry-dev.sh b/orchestration/coins/bitcoin/scripts/entry-dev.sh deleted file mode 100755 index 702dac893..000000000 --- a/orchestration/coins/bitcoin/scripts/entry-dev.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -RPC_USER="${RPC_USER:=serai}" -RPC_PASS="${RPC_PASS:=seraidex}" - -bitcoind -txindex -regtest \ - -rpcuser=$RPC_USER -rpcpassword=$RPC_PASS \ - -rpcbind=0.0.0.0 -rpcallowip=0.0.0.0/0 diff --git a/orchestration/coins/ethereum/Dockerfile b/orchestration/coins/ethereum/Dockerfile deleted file mode 100644 index d78229a7f..000000000 --- a/orchestration/coins/ethereum/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# Prepare Environment -FROM alpine:latest as builder - -ENV GETH_VERSION=1.10.23-d901d853 - -WORKDIR /home/ethereum - -RUN apk update \ - && apk --no-cache add ca-certificates gnupg bash su-exec - -# Get Binary -RUN wget https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-${GETH_VERSION}.tar.gz\ - && wget https://gethstore.blob.core.windows.net/builds/geth-linux-amd64-${GETH_VERSION}.tar.gz.asc - -# Verify Binary -# Refer to https://geth.ethereum.org/downloads/#openpgp_signatures for the PGP -# PGP keys of builders and developers -ENV KEYS 9BA28146 E058A81C 05A5DDF0 1CCB7DD2 - -RUN gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys ${KEYS} \ - && gpg --verify geth-linux-amd64-${GETH_VERSION}.tar.gz.asc geth-linux-amd64-${GETH_VERSION}.tar.gz - -# Prepare Image -RUN tar xzvf geth-linux-amd64-${GETH_VERSION}.tar.gz - -# Prepare Image -FROM ubuntu:latest as image - -WORKDIR /home/ethereum -COPY --from=builder /home/ethereum/* . -RUN mv * /bin/ -COPY ./scripts /scripts - -EXPOSE 8545 8546 30303 30303/udp - -# Run -CMD ["geth"] diff --git a/orchestration/coins/monero-wallet-rpc/Dockerfile b/orchestration/coins/monero-wallet-rpc/Dockerfile deleted file mode 100644 index 98fb760b0..000000000 --- a/orchestration/coins/monero-wallet-rpc/Dockerfile +++ /dev/null @@ -1,50 +0,0 @@ -FROM debian:bookworm-slim as mimalloc - -RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git -RUN git clone https://github.com/microsoft/mimalloc && \ - cd mimalloc && \ - git checkout 43ce4bd7fd34bcc730c1c7471c99995597415488 && \ - mkdir -p out/secure && \ - cd out/secure && \ - cmake -DMI_SECURE=ON ../.. && \ - make && \ - cp ./libmimalloc-secure.so ../../../libmimalloc.so -FROM alpine:latest as monero - -# https://downloads.getmonero.org/cli/monero-linux-x64-v0.18.3.1.tar.bz2 -# Verification will fail if MONERO_VERSION doesn't match the latest -# due to the way monero publishes releases. They overwrite a single hashes.txt -# file with each release, meaning we can only grab the SHA256 of the latest -# release. -# Most publish a asc file for each release / build architecture ¯\_(ツ)_/¯ -ENV MONERO_VERSION=0.18.3.1 - -RUN apk --no-cache add gnupg - -# Download Monero -RUN wget https://downloads.getmonero.org/cli/monero-linux-x64-v${MONERO_VERSION}.tar.bz2 - -# Verify Binary -- fingerprint from https://github.com/monero-project/monero-site/issues/1949 -ADD ./temp/hashes-v${MONERO_VERSION}.txt . -RUN gpg --keyserver hkp://keyserver.ubuntu.com:80 --keyserver-options no-self-sigs-only --receive-keys 81AC591FE9C4B65C5806AFC3F0AF4D462A0BDF92 && \ - gpg --verify hashes-v${MONERO_VERSION}.txt && \ - grep "$(sha256sum monero-linux-x64-v${MONERO_VERSION}.tar.bz2 | cut -c 1-64)" hashes-v${MONERO_VERSION}.txt - -# Extract it -RUN tar -xvjf monero-linux-x64-v${MONERO_VERSION}.tar.bz2 --strip-components=1 -FROM debian:bookworm-slim as image - -COPY --from=mimalloc libmimalloc.so /usr/lib -RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload - -RUN apt update && apt upgrade -y && apt autoremove -y && apt clean -# Switch to a non-root user -# System user (not a human), shell of nologin, no password assigned -RUN useradd --system --create-home --shell /sbin/nologin monero -USER monero - -WORKDIR /home/monero -COPY --from=monero --chown=monero monero-wallet-rpc /bin -ADD scripts /scripts - -EXPOSE 6061 diff --git a/orchestration/coins/monero-wallet-rpc/Dockerfile.monero-wallet-rpc.end b/orchestration/coins/monero-wallet-rpc/Dockerfile.monero-wallet-rpc.end deleted file mode 100644 index bf143fb67..000000000 --- a/orchestration/coins/monero-wallet-rpc/Dockerfile.monero-wallet-rpc.end +++ /dev/null @@ -1,10 +0,0 @@ -# Switch to a non-root user -# System user (not a human), shell of nologin, no password assigned -RUN useradd --system --create-home --shell /sbin/nologin monero -USER monero - -WORKDIR /home/monero -COPY --from=monero --chown=monero monero-wallet-rpc /bin -ADD scripts /scripts - -EXPOSE 6061 diff --git a/orchestration/coins/monero-wallet-rpc/scripts/entry-dev.sh b/orchestration/coins/monero-wallet-rpc/scripts/entry-dev.sh deleted file mode 100644 index fbf1edb99..000000000 --- a/orchestration/coins/monero-wallet-rpc/scripts/entry-dev.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh - -monero-wallet-rpc --disable-rpc-login --rpc-bind-port 6061 --rpc-bind-ip=0.0.0.0 --confirm-external-bind --daemon-address monero:18081 --allow-mismatched-daemon-version --wallet-dir /home/monero diff --git a/orchestration/coins/monero/Dockerfile b/orchestration/coins/monero/Dockerfile deleted file mode 100644 index 77c5f4878..000000000 --- a/orchestration/coins/monero/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -FROM alpine:latest as mimalloc - -RUN apk update && apk upgrade && apk --no-cache add gcc g++ libc-dev make cmake git -RUN git clone https://github.com/microsoft/mimalloc && \ - cd mimalloc && \ - git checkout 43ce4bd7fd34bcc730c1c7471c99995597415488 && \ - mkdir -p out/secure && \ - cd out/secure && \ - cmake -DMI_SECURE=ON ../.. && \ - make && \ - cp ./libmimalloc-secure.so ../../../libmimalloc.so -FROM alpine:latest as monero - -# https://downloads.getmonero.org/cli/monero-linux-x64-v0.18.3.1.tar.bz2 -# Verification will fail if MONERO_VERSION doesn't match the latest -# due to the way monero publishes releases. They overwrite a single hashes.txt -# file with each release, meaning we can only grab the SHA256 of the latest -# release. -# Most publish a asc file for each release / build architecture ¯\_(ツ)_/¯ -ENV MONERO_VERSION=0.18.3.1 - -RUN apk --no-cache add gnupg - -# Download Monero -RUN wget https://downloads.getmonero.org/cli/monero-linux-x64-v${MONERO_VERSION}.tar.bz2 - -# Verify Binary -- fingerprint from https://github.com/monero-project/monero-site/issues/1949 -ADD ./temp/hashes-v${MONERO_VERSION}.txt . -RUN gpg --keyserver hkp://keyserver.ubuntu.com:80 --keyserver-options no-self-sigs-only --receive-keys 81AC591FE9C4B65C5806AFC3F0AF4D462A0BDF92 && \ - gpg --verify hashes-v${MONERO_VERSION}.txt && \ - grep "$(sha256sum monero-linux-x64-v${MONERO_VERSION}.tar.bz2 | cut -c 1-64)" hashes-v${MONERO_VERSION}.txt - -# Extract it -RUN tar -xvjf monero-linux-x64-v${MONERO_VERSION}.tar.bz2 --strip-components=1 -FROM alpine:latest as image - -COPY --from=mimalloc libmimalloc.so /usr/lib -ENV LD_PRELOAD=libmimalloc.so - -RUN apk update && apk upgrade -RUN apk --no-cache add gcompat - -# Switch to a non-root user -# System user (not a human), shell of nologin, no password assigned -RUN adduser -S -s /sbin/nologin -D monero -USER monero - -WORKDIR /home/monero -COPY --from=monero --chown=monero monerod /bin -ADD scripts /scripts - -EXPOSE 18080 18081 -# VOLUME /home/monero/.bitmonero diff --git a/orchestration/coins/monero/Dockerfile.monero b/orchestration/coins/monero/Dockerfile.monero deleted file mode 100644 index 5e23a8012..000000000 --- a/orchestration/coins/monero/Dockerfile.monero +++ /dev/null @@ -1,23 +0,0 @@ -FROM alpine:latest as monero - -# https://downloads.getmonero.org/cli/monero-linux-x64-v0.18.3.1.tar.bz2 -# Verification will fail if MONERO_VERSION doesn't match the latest -# due to the way monero publishes releases. They overwrite a single hashes.txt -# file with each release, meaning we can only grab the SHA256 of the latest -# release. -# Most publish a asc file for each release / build architecture ¯\_(ツ)_/¯ -ENV MONERO_VERSION=0.18.3.1 - -RUN apk --no-cache add gnupg - -# Download Monero -RUN wget https://downloads.getmonero.org/cli/monero-linux-x64-v${MONERO_VERSION}.tar.bz2 - -# Verify Binary -- fingerprint from https://github.com/monero-project/monero-site/issues/1949 -ADD ./temp/hashes-v${MONERO_VERSION}.txt . -RUN gpg --keyserver hkp://keyserver.ubuntu.com:80 --keyserver-options no-self-sigs-only --receive-keys 81AC591FE9C4B65C5806AFC3F0AF4D462A0BDF92 && \ - gpg --verify hashes-v${MONERO_VERSION}.txt && \ - grep "$(sha256sum monero-linux-x64-v${MONERO_VERSION}.tar.bz2 | cut -c 1-64)" hashes-v${MONERO_VERSION}.txt - -# Extract it -RUN tar -xvjf monero-linux-x64-v${MONERO_VERSION}.tar.bz2 --strip-components=1 diff --git a/orchestration/coins/monero/Dockerfile.monero.end b/orchestration/coins/monero/Dockerfile.monero.end deleted file mode 100644 index 2101dbe06..000000000 --- a/orchestration/coins/monero/Dockerfile.monero.end +++ /dev/null @@ -1,13 +0,0 @@ -RUN apk --no-cache add gcompat - -# Switch to a non-root user -# System user (not a human), shell of nologin, no password assigned -RUN adduser -S -s /sbin/nologin -D monero -USER monero - -WORKDIR /home/monero -COPY --from=monero --chown=monero monerod /bin -ADD scripts /scripts - -EXPOSE 18080 18081 -# VOLUME /home/monero/.bitmonero diff --git a/orchestration/coins/monero/scripts/entry-dev.sh b/orchestration/coins/monero/scripts/entry-dev.sh deleted file mode 100755 index b5367f0cb..000000000 --- a/orchestration/coins/monero/scripts/entry-dev.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -RPC_USER="${RPC_USER:=serai}" -RPC_PASS="${RPC_PASS:=seraidex}" - -# Run Monero -# TODO: Restore Auth -monerod --non-interactive --regtest --offline --fixed-difficulty=1 \ - --no-zmq --rpc-bind-ip=0.0.0.0 --confirm-external-bind \ - --rpc-access-control-origins * --disable-rpc-ban diff --git a/orchestration/coordinator/Dockerfile b/orchestration/coordinator/Dockerfile deleted file mode 100644 index f30734c56..000000000 --- a/orchestration/coordinator/Dockerfile +++ /dev/null @@ -1,73 +0,0 @@ -FROM debian:bookworm-slim as mimalloc - -RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git -RUN git clone https://github.com/microsoft/mimalloc && \ - cd mimalloc && \ - git checkout 43ce4bd7fd34bcc730c1c7471c99995597415488 && \ - mkdir -p out/secure && \ - cd out/secure && \ - cmake -DMI_SECURE=ON ../.. && \ - make && \ - cp ./libmimalloc-secure.so ../../../libmimalloc.so -FROM rust:1.75-slim-bookworm as builder - -COPY --from=mimalloc libmimalloc.so /usr/lib -RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload - -RUN apt update && apt upgrade -y && apt autoremove -y && apt clean - -# Add dev dependencies -RUN apt install -y pkg-config clang - -# Dependencies for the Serai node -RUN apt install -y make protobuf-compiler - -# Add the wasm toolchain -RUN rustup target add wasm32-unknown-unknown - -# Add files for build -ADD common /serai/common -ADD crypto /serai/crypto -ADD coins /serai/coins -ADD message-queue /serai/message-queue -ADD processor /serai/processor -ADD coordinator /serai/coordinator -ADD substrate /serai/substrate -ADD mini /serai/mini -ADD tests /serai/tests -ADD patches /serai/patches -ADD Cargo.toml /serai -ADD Cargo.lock /serai -ADD AGPL-3.0 /serai - -WORKDIR /serai - -# Mount the caches and build -RUN --mount=type=cache,target=/root/.cargo \ - --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/usr/local/cargo/git \ - --mount=type=cache,target=/serai/target \ - mkdir /serai/bin && \ - cargo build -p serai-coordinator --features "parity-db longer-reattempts" && \ - mv /serai/target/debug/serai-coordinator /serai/bin -FROM debian:bookworm-slim as image - -COPY --from=mimalloc libmimalloc.so /usr/lib -RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload - -RUN apt update && apt upgrade -y && apt autoremove -y && apt clean -# Install ca-certificates -RUN apt install -y ca-certificates - -# Switch to a non-root user -RUN useradd --system --create-home --shell /sbin/nologin coordinator -USER coordinator - -WORKDIR /home/coordinator - -# Copy the Coordinator binary and relevant license -COPY --from=builder --chown=processor /serai/bin/serai-coordinator /bin/ -COPY --from=builder --chown=processor /serai/AGPL-3.0 . - -# Run coordinator -CMD ["serai-coordinator"] diff --git a/orchestration/coordinator/Dockerfile.coordinator b/orchestration/coordinator/Dockerfile.coordinator deleted file mode 100644 index 8c1f6a22e..000000000 --- a/orchestration/coordinator/Dockerfile.coordinator +++ /dev/null @@ -1,2 +0,0 @@ - cargo build -p serai-coordinator --features "parity-db longer-reattempts" && \ - mv /serai/target/debug/serai-coordinator /serai/bin diff --git a/orchestration/coordinator/Dockerfile.coordinator.end b/orchestration/coordinator/Dockerfile.coordinator.end deleted file mode 100644 index 9c8bcd3dc..000000000 --- a/orchestration/coordinator/Dockerfile.coordinator.end +++ /dev/null @@ -1,15 +0,0 @@ -# Install ca-certificates -RUN apt install -y ca-certificates - -# Switch to a non-root user -RUN useradd --system --create-home --shell /sbin/nologin coordinator -USER coordinator - -WORKDIR /home/coordinator - -# Copy the Coordinator binary and relevant license -COPY --from=builder --chown=processor /serai/bin/serai-coordinator /bin/ -COPY --from=builder --chown=processor /serai/AGPL-3.0 . - -# Run coordinator -CMD ["serai-coordinator"] diff --git a/orchestration/coordinator/scripts/entry-dev.sh b/orchestration/coordinator/scripts/entry-dev.sh deleted file mode 100644 index 187bf7983..000000000 --- a/orchestration/coordinator/scripts/entry-dev.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -export MESSAGE_QUEUE_KEY="0000000000000000000000000000000000000000000000000000000000000000" -export MESSAGE_QUEUE_RPC="http://127.0.0.1:2287" - -export DB_PATH="./coordinator-db" -export SERAI_HOSTNAME="127.0.0.1" - -serai-coordinator diff --git a/orchestration/dev/coins/bitcoin/run.sh b/orchestration/dev/coins/bitcoin/run.sh new file mode 100755 index 000000000..da7c95a8b --- /dev/null +++ b/orchestration/dev/coins/bitcoin/run.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +RPC_USER="${RPC_USER:=serai}" +RPC_PASS="${RPC_PASS:=seraidex}" + +bitcoind -txindex -regtest --port=8333 \ + -rpcuser=$RPC_USER -rpcpassword=$RPC_PASS \ + -rpcbind=0.0.0.0 -rpcallowip=0.0.0.0/0 -rpcport=8332 \ + $1 diff --git a/orchestration/coins/ethereum/scripts/entry-dev.sh b/orchestration/dev/coins/ethereum/run.sh similarity index 100% rename from orchestration/coins/ethereum/scripts/entry-dev.sh rename to orchestration/dev/coins/ethereum/run.sh diff --git a/orchestration/dev/coins/monero-wallet-rpc/run.sh b/orchestration/dev/coins/monero-wallet-rpc/run.sh new file mode 100755 index 000000000..5c7aeb70c --- /dev/null +++ b/orchestration/dev/coins/monero-wallet-rpc/run.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +monero-wallet-rpc \ + --allow-mismatched-daemon-version \ + --daemon-address serai-dev-monero:18081 --daemon-login serai:seraidex \ + --disable-rpc-login --rpc-bind-ip=0.0.0.0 --rpc-bind-port 18082 --confirm-external-bind \ + --wallet-dir /home/monero diff --git a/orchestration/coins/monero/temp/hashes-v0.18.3.1.txt b/orchestration/dev/coins/monero/hashes-v0.18.3.1.txt similarity index 100% rename from orchestration/coins/monero/temp/hashes-v0.18.3.1.txt rename to orchestration/dev/coins/monero/hashes-v0.18.3.1.txt diff --git a/orchestration/dev/coins/monero/run.sh b/orchestration/dev/coins/monero/run.sh new file mode 100755 index 000000000..675d44382 --- /dev/null +++ b/orchestration/dev/coins/monero/run.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +RPC_USER="${RPC_USER:=serai}" +RPC_PASS="${RPC_PASS:=seraidex}" + +# Run Monero +monerod --non-interactive --regtest --offline --fixed-difficulty=1 \ + --no-zmq --rpc-bind-ip=0.0.0.0 --rpc-bind-port=18081 --confirm-external-bind \ + --rpc-access-control-origins "*" --disable-rpc-ban \ + --rpc-login=$RPC_USER:$RPC_PASS \ + $1 diff --git a/orchestration/dev/coordinator/.folder b/orchestration/dev/coordinator/.folder new file mode 100644 index 000000000..e69de29bb diff --git a/orchestration/dev/message-queue/.folder b/orchestration/dev/message-queue/.folder new file mode 100644 index 000000000..e69de29bb diff --git a/orchestration/dev/processor/bitcoin/.folder b/orchestration/dev/processor/bitcoin/.folder new file mode 100644 index 000000000..e69de29bb diff --git a/orchestration/dev/processor/ethereum/.folder b/orchestration/dev/processor/ethereum/.folder new file mode 100644 index 000000000..e69de29bb diff --git a/orchestration/dev/processor/monero/.folder b/orchestration/dev/processor/monero/.folder new file mode 100644 index 000000000..e69de29bb diff --git a/orchestration/dev/serai/run.sh b/orchestration/dev/serai/run.sh new file mode 100755 index 000000000..44e9969f8 --- /dev/null +++ b/orchestration/dev/serai/run.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +serai-node --unsafe-rpc-external --rpc-cors all --chain local --$SERAI_NAME diff --git a/orchestration/docker-compose.yml b/orchestration/docker-compose.yml deleted file mode 100644 index ae8637bb9..000000000 --- a/orchestration/docker-compose.yml +++ /dev/null @@ -1,221 +0,0 @@ -version: "3.9" -name: serai-dev - -volumes: - serai: - serai-alice: - serai-bob: - serai-charlie: - serai-dave: - serai-eve: - serai-ferdie: - -services: - # Coin services - - bitcoin: - profiles: - - bitcoin - - coins - build: - context: ./coins/bitcoin/ - restart: unless-stopped - volumes: - - "./coins/bitcoin/scripts:/scripts" - entrypoint: /scripts/entry-dev.sh - # TODO: Use expose, not ports - ports: - - "18443:18443" - - ethereum: - profiles: - - ethereum - - coins - build: - context: ./coins/ethereum/ - restart: unless-stopped - volumes: - - "./coins/ethereum/scripts:/scripts" - entrypoint: /scripts/entry-dev.sh - - monero: - profiles: - - monero - - coins - build: - context: ./coins/monero/ - restart: unless-stopped - volumes: - - "./coins/monero/scripts:/scripts" - entrypoint: /scripts/entry-dev.sh - # TODO: Use expose, not ports - ports: - - "18081:18081" - - monero-wallet-rpc: - profiles: - - monero - - coins - build: - context: ./coins/monero-wallet-rpc/ - restart: unless-stopped - volumes: - - "./coins/monero-wallet-rpc/scripts:/scripts" - entrypoint: /scripts/entry-dev.sh - # TODO: Use expose, not ports - ports: - - "6061:6061" - - # Infrastructure - - message-queue: - profiles: - - message-queue - build: - context: ../ - dockerfile: ./orchestration/message-queue/Dockerfile - restart: unless-stopped - volumes: - - "./message-queue/scripts:/scripts" - entrypoint: /scripts/entry-dev.sh - expose: - - "2287" - - bitcoin-processor: - profiles: - - bitcoin-processor - build: - context: ../ - dockerfile: ./orchestration/processor/bitcoin/Dockerfile - restart: unless-stopped - volumes: - - "./processor/scripts:/scripts" - entrypoint: /scripts/entry-dev.sh - - monero-processor: - profiles: - - monero-processor - build: - context: ../ - dockerfile: ./orchestration/processor/monero/Dockerfile - restart: unless-stopped - volumes: - - "./processor/scripts:/scripts" - entrypoint: /scripts/entry-dev.sh - - coordinator: - profiles: - - coordinator - build: - context: ../ - dockerfile: ./orchestration/coordinator/Dockerfile - restart: unless-stopped - volumes: - - "./coordinator/scripts:/scripts" - entrypoint: /scripts/entry-dev.sh - - # Serai runtime - - runtime: - profiles: - - runtime - build: - context: ../ - dockerfile: ./orchestration/runtime/Dockerfile - entrypoint: | - sh -c "cd /serai/substrate/runtime && cargo clean && cargo build --release && \ - sha256sum /serai/target/release/wbuild/serai-runtime/serai_runtime.wasm" - - # Serai nodes - - _serai: - &serai_defaults - restart: unless-stopped - # image: serai:dev - profiles: - - _ - build: - context: ../ - dockerfile: ./orchestration/serai/Dockerfile - args: - TAG: serai - entrypoint: /scripts/entry-dev.sh - volumes: - - "./serai/scripts:/scripts" - - serai: - <<: *serai_defaults - hostname: serai - profiles: - - serai - environment: - CHAIN: local - NAME: node - - serai-alice: - <<: *serai_defaults - hostname: serai-alice - profiles: - - alice - - cluster-sm - - cluster-lg - environment: - CHAIN: local - NAME: alice - VALIDATOR: true - - serai-bob: - <<: *serai_defaults - hostname: serai-bob - profiles: - - bob - - cluster-sm - - cluster-lg - environment: - CHAIN: local - NAME: bob - VALIDATOR: true - - serai-charlie: - <<: *serai_defaults - hostname: serai-charlie - profiles: - - charlie - - cluster-sm - - cluster-lg - environment: - CHAIN: local - NAME: charlie - VALIDATOR: true - - serai-dave: - <<: *serai_defaults - hostname: serai-dave - profiles: - - dave - - cluster-sm - - cluster-lg - environment: - CHAIN: local - NAME: dave - VALIDATOR: true - - serai-eve: - <<: *serai_defaults - hostname: serai-eve - profiles: - - eve - - cluster-lg - environment: - CHAIN: local - NAME: eve - - serai-ferdie: - <<: *serai_defaults - hostname: serai-ferdie - profiles: - - ferdie - - cluster-lg - environment: - CHAIN: local - NAME: ferdie diff --git a/orchestration/dockerfiles.sh b/orchestration/dockerfiles.sh deleted file mode 100755 index 40b8c1119..000000000 --- a/orchestration/dockerfiles.sh +++ /dev/null @@ -1,70 +0,0 @@ -# Bitcoin -rm ./coins/bitcoin/Dockerfile -cat \ - ./Dockerfile.parts/mimalloc/Dockerfile.debian \ - ./coins/bitcoin/Dockerfile.bitcoin \ - ./Dockerfile.parts/Dockerfile.debian.start \ - ./coins/bitcoin/Dockerfile.bitcoin.end >> ./coins/bitcoin/Dockerfile - -# Monero -rm ./coins/monero/Dockerfile -cat \ - ./Dockerfile.parts/mimalloc/Dockerfile.alpine \ - ./coins/monero/Dockerfile.monero \ - ./Dockerfile.parts/Dockerfile.alpine.start \ - ./coins/monero/Dockerfile.monero.end >> ./coins/monero/Dockerfile - -# Monero wallet rpc -rm -f ./coins/monero-wallet-rpc/Dockerfile -mkdir -p ./coins/monero-wallet-rpc/temp/ -cp ./coins/monero/temp/hashes-v* ./coins/monero-wallet-rpc/temp/ -cat \ - ./Dockerfile.parts/mimalloc/Dockerfile.debian \ - ./coins/monero/Dockerfile.monero \ - ./Dockerfile.parts/Dockerfile.debian.start \ - ./coins/monero-wallet-rpc/Dockerfile.monero-wallet-rpc.end >> ./coins/monero-wallet-rpc/Dockerfile - -# Message Queue -rm ./message-queue/Dockerfile -cat \ - ./Dockerfile.parts/mimalloc/Dockerfile.debian \ - ./Dockerfile.parts/Dockerfile.serai.build \ - ./message-queue/Dockerfile.message-queue \ - ./Dockerfile.parts/Dockerfile.debian.start \ - ./message-queue/Dockerfile.message-queue.end >> ./message-queue/Dockerfile - -# Bitcoin Processor -rm ./processor/bitcoin/Dockerfile -cat \ - ./Dockerfile.parts/mimalloc/Dockerfile.debian \ - ./Dockerfile.parts/Dockerfile.serai.build \ - ./processor/bitcoin/Dockerfile.processor.bitcoin \ - ./Dockerfile.parts/Dockerfile.debian.start \ - ./processor/Dockerfile.processor.end >> ./processor/bitcoin/Dockerfile - -# Monero Processor -rm ./processor/monero/Dockerfile -cat \ - ./Dockerfile.parts/mimalloc/Dockerfile.debian \ - ./Dockerfile.parts/Dockerfile.serai.build \ - ./processor/monero/Dockerfile.processor.monero \ - ./Dockerfile.parts/Dockerfile.debian.start \ - ./processor/Dockerfile.processor.end >> ./processor/monero/Dockerfile - -# Coordinator -rm ./coordinator/Dockerfile -cat \ - ./Dockerfile.parts/mimalloc/Dockerfile.debian \ - ./Dockerfile.parts/Dockerfile.serai.build \ - ./coordinator/Dockerfile.coordinator \ - ./Dockerfile.parts/Dockerfile.debian.start \ - ./coordinator/Dockerfile.coordinator.end >> ./coordinator/Dockerfile - -# Node -rm ./serai/Dockerfile -cat \ - ./Dockerfile.parts/mimalloc/Dockerfile.debian \ - ./Dockerfile.parts/Dockerfile.serai.build \ - ./serai/Dockerfile.serai \ - ./Dockerfile.parts/Dockerfile.debian.start \ - ./serai/Dockerfile.serai.end >> ./serai/Dockerfile diff --git a/orchestration/kubernetes/Makefile b/orchestration/kubernetes/Makefile deleted file mode 100644 index 9889b0717..000000000 --- a/orchestration/kubernetes/Makefile +++ /dev/null @@ -1,103 +0,0 @@ -SHELL := /bin/bash - -check-helm: - @helm version || $(MAKE) install-helm - -check-kubectl: - @kubectl version || $(MAKE) install-kubectl - -install-helm: - @curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 - @chmod 700 get_helm.sh - @./get_helm.sh - @rm get_helm.sh - -install-kubectl: - @curl -LO 'https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl' - @sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - @rm kubectl - -deploy-base: - @docker compose -f ../docker-compose.yml --profile base build --quiet - @(cat ../serai/scripts/entry-dev.sh | base64 -w 0 -) | xargs -I % helm upgrade --install serai-base charts/serai/\ - --values charts/serai/values.yaml --set image.envVariablesfullnameOverride=serai-base,nameOverride=serai-base,\ - image.envVariables[1].value=base,configMapFile=% - -deploy-bitcoin: - @docker compose -f ../docker-compose.yml --profile bitcoin build --quiet - @(cat ../coins/bitcoin/scripts/entry-dev.sh | base64 -w 0 -) | xargs -I % helm upgrade --install bitcoin-daemon\ - charts/bitcoin/ --values charts/bitcoin/values.yaml --set configMapFile=% - -deploy-ethereum: - @docker compose -f ../docker-compose.yml --profile ethereum build --quiet - @(cat ../coins/ethereum/scripts/entry-dev.sh | base64 -w 0 -) | xargs -I % helm upgrade --install ethereum-daemon\ - charts/ethereum/ --values charts/ethereum/values.yaml --set configMapFile=% - -deploy-monero: - @docker compose -f ../docker-compose.yml --profile monero build --quiet - @(cat ../coins/monero/scripts/entry-dev.sh | base64 -w 0 -) | xargs -I % helm upgrade --install monero-daemon\ - charts/monero/ --values charts/monero/values.yaml --set configMapFile=% - -deploy-cluster-sm: - @docker compose -f ../docker-compose.yml --profile cluster-sm build --quiet - @(cat ../serai/scripts/entry-dev.sh | base64 -w 0 -) | xargs -I % helm upgrade --install serai-alice charts/serai/\ - --values charts/serai/values.yaml --set image.envVariablesfullnameOverride=serai-alice,nameOverride=serai-alice,\ - image.envVariables[1].value=Alice,image.envVariables[2].value="'1'",configMapFile=% - @(cat ../serai/scripts/entry-dev.sh | base64 -w 0 -) | xargs -I % helm upgrade --install serai-charlie charts/serai/\ - --values charts/serai/values.yaml --set image.envVariablesfullnameOverride=serai-charlie,nameOverride=serai-charlie,\ - image.envVariables[1].value=Charlie,configMapFile=% - @(cat ../serai/scripts/entry-dev.sh | base64 -w 0 -) | xargs -I % helm upgrade --install serai-bob charts/serai/\ - --values charts/serai/values.yaml --set image.envVariablesfullnameOverride=serai-bob,nameOverride=serai-bob,\ - image.envVariables[1].value=Bob,configMapFile=% - -deploy-cluster-lg: deploy-cluster-sm - @docker compose -f ../docker-compose.yml --profile cluster-lg build --quiet - @(cat ../serai/scripts/entry-dev.sh | base64 -w 0 -) | xargs -I % helm upgrade --install serai-dave charts/serai/\ - --values charts/serai/values.yaml --set image.envVariablesfullnameOverride=serai-dave,nameOverride=serai-dave,\ - image.envVariables[1].value=Dave,configMapFile=% - @(cat ../serai/scripts/entry-dev.sh | base64 -w 0 -) | xargs -I % helm upgrade --install serai-eve charts/serai/\ - --values charts/serai/values.yaml --set image.envVariablesfullnameOverride=serai-eve,nameOverride=serai-eve,\ - image.envVariables[1].value=Eve,configMapFile=% - @(cat ../serai/scripts/entry-dev.sh | base64 -w 0 -) | xargs -I % helm upgrade --install serai-ferdie charts/serai/\ - --values charts/serai/values.yaml --set image.envVariablesfullnameOverride=serai-ferdie,nameOverride=serai-ferdie,\ - image.envVariables[1].value=Ferdie,configMapFile=% - -deploy-coins: deploy-bitcoin deploy-ethereum deploy-monero - -deploy-cluster-coins-sm: deploy-cluster-sm deploy-coins - -deploy-cluster-coins-lg: deploy-cluster-lg deploy-coins - -deploy-all: deploy-cluster-coins-lg - -delete-base: - @helm delete serai-base - -delete-bitcoin: - @helm delete bitcoin-daemon - -delete-ethereum: - @helm delete ethereum-daemon - -delete-monero: - @helm delete monero-daemon - -delete-cluster-lg: delete-cluster-sm - @helm delete serai-dave - @helm delete serai-eve - @helm delete serai-ferdie - -delete-cluster-sm: - @helm delete serai-alice - @helm delete serai-charlie - @helm delete serai-bob - -delete-coins: delete-bitcoin delete-ethereum delete-monero - -delete-cluster-coins-sm: delete-cluster-sm delete-coins - -delete-cluster-coins-lg: delete-cluster-lg delete-coins - -delete-all: delete-cluster-coins-lg - -check-dependencies: check-helm check-kubectl diff --git a/orchestration/kubernetes/README.md b/orchestration/kubernetes/README.md deleted file mode 100644 index 7bf68d9e9..000000000 --- a/orchestration/kubernetes/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Kubernetes -## Run with Kubernetes -Running the Serai infrastructure is easy with Kubernetes. - -We utilize Makefile to easily orchestrate various pieces of the infrastructure on kubernetes. - -**Example to deploy:** `make deploy-` -```bash -make deploy-cluster-sm -``` -**Example to delete:** `make -i delete-` -```bash -make delete-cluster-sm -``` - -All commands are assumed to be ran from the kubernetes folder, not the serai root folder. - -### Profiles: -* deploy-base - single node, named base -* deploy-coins - node clients for coins only (BTC, ETH, XMR) -* deploy-cluster-sm - Alice (Validator), Bob, Charlie -* deploy-cluster-coins-sm - cluster-sm with coins -* deploy-cluster-lg - Alice (Validator), Bob, Charlie, Dave, Eve, Ferdie -* deploy-cluster-coins-lg - cluster-lg with coins -* deploy-monero - full node monero only -* deploy-bitcoin - full node bitcoin only -* deploy-ethereum - full node ethereum only - -## Requirements for Linux -* Local built images of serai and coins, please follow the Instructions [here](../README.md) -* Running kubernetes cluster (version >= 1.19) -* Curl tool -* Make tool -* Kubectl, check if not installed -```bash -make check-kubectl -``` -* Helm, check if not installed -```bash -make check-helm -``` diff --git a/orchestration/kubernetes/charts/bitcoin/Chart.yaml b/orchestration/kubernetes/charts/bitcoin/Chart.yaml deleted file mode 100644 index 3824b0e46..000000000 --- a/orchestration/kubernetes/charts/bitcoin/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v2 -name: bitcoin -description: A Helm chart for bitcoin-daemon -type: application -version: 0.1.0 diff --git a/orchestration/kubernetes/charts/bitcoin/templates/_helpers.tpl b/orchestration/kubernetes/charts/bitcoin/templates/_helpers.tpl deleted file mode 100644 index 17b464a42..000000000 --- a/orchestration/kubernetes/charts/bitcoin/templates/_helpers.tpl +++ /dev/null @@ -1,42 +0,0 @@ -{{- define "bitcoin.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 253 | trimSuffix "-" }} -{{- end }} - -{{- define "bitcoin.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 253 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 253 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 253 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{- define "bitcoin.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 253 | trimSuffix "-" }} -{{- end }} - -{{- define "bitcoin.labels" -}} -helm.sh/chart: {{ include "bitcoin.chart" . }} -{{ include "bitcoin.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{- define "bitcoin.selectorLabels" -}} -app.kubernetes.io/name: {{ include "bitcoin.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{- define "bitcoin.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "bitcoin.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/orchestration/kubernetes/charts/bitcoin/templates/configmap.yaml b/orchestration/kubernetes/charts/bitcoin/templates/configmap.yaml deleted file mode 100644 index bf5502005..000000000 --- a/orchestration/kubernetes/charts/bitcoin/templates/configmap.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-configmap -data: - entry-dev.sh: | -{{ .Values.configMapFile | b64dec | indent 4}} diff --git a/orchestration/kubernetes/charts/bitcoin/templates/deployment.yaml b/orchestration/kubernetes/charts/bitcoin/templates/deployment.yaml deleted file mode 100644 index f852a96e6..000000000 --- a/orchestration/kubernetes/charts/bitcoin/templates/deployment.yaml +++ /dev/null @@ -1,88 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "bitcoin.fullname" . }} - labels: - {{- include "bitcoin.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "bitcoin.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "bitcoin.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "bitcoin.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- if hasKey .Values.image "ports" }} - ports: - {{- range .Values.image.ports }} - - name: {{ .name }} - containerPort: {{ .containerPort }} - {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "command" }} - command: - {{- toYaml .Values.image.command | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "args" }} - args: - {{- toYaml .Values.image.args | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "envVariables" }} - env: - {{- toYaml .Values.image.envVariables | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "volumeMounts" }} - volumeMounts: - {{- range .Values.image.volumeMounts }} - - mountPath: {{ .mountPath }} - name: {{ $.Release.Name}}-{{ .name }} - {{- end }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- if hasKey .Values "volumes" }} - volumes: - {{- range .Values.volumes }} - - configMap: - defaultMode: {{ .configMap.defaultMode }} - name: {{ $.Release.Name}}-{{ .configMap.name }} - name: {{ $.Release.Name}}-{{ .name }} - {{- end }} - {{- end }} diff --git a/orchestration/kubernetes/charts/bitcoin/templates/ingress.yaml b/orchestration/kubernetes/charts/bitcoin/templates/ingress.yaml deleted file mode 100644 index 39dff21ce..000000000 --- a/orchestration/kubernetes/charts/bitcoin/templates/ingress.yaml +++ /dev/null @@ -1,50 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "bitcoin.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "bitcoin.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if .Values.ingress.className }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if .pathType }} - pathType: {{ .pathType }} - {{- end }} - backend: - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- end }} - {{- end }} -{{- end }} diff --git a/orchestration/kubernetes/charts/bitcoin/templates/service.yaml b/orchestration/kubernetes/charts/bitcoin/templates/service.yaml deleted file mode 100644 index e5fc600fd..000000000 --- a/orchestration/kubernetes/charts/bitcoin/templates/service.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "bitcoin.fullname" . }} - labels: - {{- include "bitcoin.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - {{- if hasKey .Values.service "ports" }} - {{- range .Values.service.ports }} - - port: {{ .port }} - name: {{ .name }} - targetPort: {{ .targetPort }} - protocol: {{ .protocol }} - {{- end }} - {{- else }} - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - {{- end }} - selector: - {{- include "bitcoin.selectorLabels" . | nindent 4 }} diff --git a/orchestration/kubernetes/charts/bitcoin/values.yaml b/orchestration/kubernetes/charts/bitcoin/values.yaml deleted file mode 100644 index 54bb04623..000000000 --- a/orchestration/kubernetes/charts/bitcoin/values.yaml +++ /dev/null @@ -1,74 +0,0 @@ -replicaCount: 1 - -net: mainnet - -image: - repository: serai-dev-bitcoin - pullPolicy: IfNotPresent - tag: "latest" - ports: - - name: p2p - containerPort: 18444 - protocol: TCP - - name: rpc - containerPort: 18443 - protocol: TCP - volumeMounts: - - mountPath: /scripts - name: configmap-volume - args: - - bash - - /scripts/entry-dev.sh - -volumes: - - configMap: - defaultMode: 420 - name: configmap - name: configmap-volume - -configMapFile: "entry-dev.sh" - -imagePullSecrets: [] - -serviceAccount: - create: false - name: "" - -podAnnotations: {} - -podSecurityContext: {} - -securityContext: {} - -service: - type: ClusterIP - ports: - - name: p2p - port: 18444 - targetPort: p2p - protocol: TCP - - name: rpc - port: 18443 - targetPort: rpc - protocol: TCP - -ingress: - enabled: false - className: "" - annotations: {} - hosts: [] - tls: [] - -resources: {} - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/orchestration/kubernetes/charts/ethereum/Chart.yaml b/orchestration/kubernetes/charts/ethereum/Chart.yaml deleted file mode 100644 index 315940449..000000000 --- a/orchestration/kubernetes/charts/ethereum/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v2 -name: ethereum -description: A Helm chart for ethereum-daemon -type: application -version: 0.1.0 diff --git a/orchestration/kubernetes/charts/ethereum/templates/_helpers.tpl b/orchestration/kubernetes/charts/ethereum/templates/_helpers.tpl deleted file mode 100644 index ab9b2fc9d..000000000 --- a/orchestration/kubernetes/charts/ethereum/templates/_helpers.tpl +++ /dev/null @@ -1,42 +0,0 @@ -{{- define "ethereum.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 253 | trimSuffix "-" }} -{{- end }} - -{{- define "ethereum.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 253 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 253 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 253 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{- define "ethereum.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 253 | trimSuffix "-" }} -{{- end }} - -{{- define "ethereum.labels" -}} -helm.sh/chart: {{ include "ethereum.chart" . }} -{{ include "ethereum.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{- define "ethereum.selectorLabels" -}} -app.kubernetes.io/name: {{ include "ethereum.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{- define "ethereum.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "ethereum.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/orchestration/kubernetes/charts/ethereum/templates/configmap.yaml b/orchestration/kubernetes/charts/ethereum/templates/configmap.yaml deleted file mode 100644 index bf5502005..000000000 --- a/orchestration/kubernetes/charts/ethereum/templates/configmap.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-configmap -data: - entry-dev.sh: | -{{ .Values.configMapFile | b64dec | indent 4}} diff --git a/orchestration/kubernetes/charts/ethereum/templates/deployment.yaml b/orchestration/kubernetes/charts/ethereum/templates/deployment.yaml deleted file mode 100644 index b673ba127..000000000 --- a/orchestration/kubernetes/charts/ethereum/templates/deployment.yaml +++ /dev/null @@ -1,89 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "ethereum.fullname" . }} - labels: - {{- include "ethereum.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "ethereum.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "ethereum.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "ethereum.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- if hasKey .Values.image "ports" }} - ports: - {{- range .Values.image.ports }} - - name: {{ .name }} - containerPort: {{ .containerPort }} - protocol: {{ .protocol }} - {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "command" }} - command: - {{- toYaml .Values.image.command | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "args" }} - args: - {{- toYaml .Values.image.args | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "envVariables" }} - env: - {{- toYaml .Values.image.envVariables | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "volumeMounts" }} - volumeMounts: - {{- range .Values.image.volumeMounts }} - - mountPath: {{ .mountPath }} - name: {{ $.Release.Name}}-{{ .name }} - {{- end }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- if hasKey .Values "volumes" }} - volumes: - {{- range .Values.volumes }} - - configMap: - defaultMode: {{ .configMap.defaultMode }} - name: {{ $.Release.Name}}-{{ .configMap.name }} - name: {{ $.Release.Name}}-{{ .name }} - {{- end }} - {{- end }} diff --git a/orchestration/kubernetes/charts/ethereum/templates/ingress.yaml b/orchestration/kubernetes/charts/ethereum/templates/ingress.yaml deleted file mode 100644 index 18dcd1852..000000000 --- a/orchestration/kubernetes/charts/ethereum/templates/ingress.yaml +++ /dev/null @@ -1,50 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "ethereum.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "ethereum.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if .Values.ingress.className }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if .pathType }} - pathType: {{ .pathType }} - {{- end }} - backend: - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- end }} - {{- end }} -{{- end }} diff --git a/orchestration/kubernetes/charts/ethereum/templates/service.yaml b/orchestration/kubernetes/charts/ethereum/templates/service.yaml deleted file mode 100644 index df83df170..000000000 --- a/orchestration/kubernetes/charts/ethereum/templates/service.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "ethereum.fullname" . }} - labels: - {{- include "ethereum.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - {{- if hasKey .Values.service "ports" }} - {{- range .Values.service.ports }} - - port: {{ .port }} - name: {{ .name }} - targetPort: {{ .targetPort }} - protocol: {{ .protocol }} - {{- end }} - {{- else }} - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - {{- end }} - selector: - {{- include "ethereum.selectorLabels" . | nindent 4 }} diff --git a/orchestration/kubernetes/charts/ethereum/values.yaml b/orchestration/kubernetes/charts/ethereum/values.yaml deleted file mode 100644 index cef45a09f..000000000 --- a/orchestration/kubernetes/charts/ethereum/values.yaml +++ /dev/null @@ -1,60 +0,0 @@ -replicaCount: 1 - -image: - repository: serai-dev-ethereum - pullPolicy: IfNotPresent - tag: "latest" - - ports: - - name: rpc - containerPort: 8545 - protocol: TCP - volumeMounts: - - mountPath: /scripts - name: configmap-volume - args: - - bash - - /scripts/entry-dev.sh - -volumes: - - configMap: - defaultMode: 420 - name: configmap - name: configmap-volume - -configMapFile: "entry-dev.sh" - -imagePullSecrets: [] - -serviceAccount: - create: false - name: "" - -podAnnotations: {} - -podSecurityContext: {} - -securityContext: {} - -service: - type: ClusterIP - port: 8545 - -ingress: - enabled: false - className: "" - annotations: {} - hosts: [] - tls: [] - -resources: {} - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - -nodeSelector: {} -tolerations: [] -affinity: {} diff --git a/orchestration/kubernetes/charts/monero/Chart.yaml b/orchestration/kubernetes/charts/monero/Chart.yaml deleted file mode 100644 index b9c0d84ce..000000000 --- a/orchestration/kubernetes/charts/monero/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v2 -name: monero -description: A Helm chart for monero-daemon -type: application -version: 0.1.0 diff --git a/orchestration/kubernetes/charts/monero/templates/_helpers.tpl b/orchestration/kubernetes/charts/monero/templates/_helpers.tpl deleted file mode 100644 index 2829e6037..000000000 --- a/orchestration/kubernetes/charts/monero/templates/_helpers.tpl +++ /dev/null @@ -1,42 +0,0 @@ -{{- define "monero.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 253 | trimSuffix "-" }} -{{- end }} - -{{- define "monero.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 253 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 253 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 253 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{- define "monero.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 253 | trimSuffix "-" }} -{{- end }} - -{{- define "monero.labels" -}} -helm.sh/chart: {{ include "monero.chart" . }} -{{ include "monero.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{- define "monero.selectorLabels" -}} -app.kubernetes.io/name: {{ include "monero.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{- define "monero.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "monero.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/orchestration/kubernetes/charts/monero/templates/configmap.yaml b/orchestration/kubernetes/charts/monero/templates/configmap.yaml deleted file mode 100644 index bf5502005..000000000 --- a/orchestration/kubernetes/charts/monero/templates/configmap.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-configmap -data: - entry-dev.sh: | -{{ .Values.configMapFile | b64dec | indent 4}} diff --git a/orchestration/kubernetes/charts/monero/templates/deployment.yaml b/orchestration/kubernetes/charts/monero/templates/deployment.yaml deleted file mode 100644 index fdd32c05d..000000000 --- a/orchestration/kubernetes/charts/monero/templates/deployment.yaml +++ /dev/null @@ -1,88 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "monero.fullname" . }} - labels: - {{- include "monero.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "monero.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "monero.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "monero.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- if hasKey .Values.image "ports" }} - ports: - {{- range .Values.image.ports }} - - name: {{ .name }} - containerPort: {{ .containerPort }} - {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "command" }} - command: - {{- toYaml .Values.image.command | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "args" }} - args: - {{- toYaml .Values.image.args | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "envVariables" }} - env: - {{- toYaml .Values.image.envVariables | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "volumeMounts" }} - volumeMounts: - {{- range .Values.image.volumeMounts }} - - mountPath: {{ .mountPath }} - name: {{ $.Release.Name}}-{{ .name }} - {{- end }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- if hasKey .Values "volumes" }} - volumes: - {{- range .Values.volumes }} - - configMap: - defaultMode: {{ .configMap.defaultMode }} - name: {{ $.Release.Name}}-{{ .configMap.name }} - name: {{ $.Release.Name}}-{{ .name }} - {{- end }} - {{- end }} diff --git a/orchestration/kubernetes/charts/monero/templates/ingress.yaml b/orchestration/kubernetes/charts/monero/templates/ingress.yaml deleted file mode 100644 index 2ae3d4fa5..000000000 --- a/orchestration/kubernetes/charts/monero/templates/ingress.yaml +++ /dev/null @@ -1,50 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "monero.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "monero.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if .Values.ingress.className }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if .pathType }} - pathType: {{ .pathType }} - {{- end }} - backend: - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- end }} - {{- end }} -{{- end }} diff --git a/orchestration/kubernetes/charts/monero/templates/service.yaml b/orchestration/kubernetes/charts/monero/templates/service.yaml deleted file mode 100644 index 1a4f90d23..000000000 --- a/orchestration/kubernetes/charts/monero/templates/service.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "monero.fullname" . }} - labels: - {{- include "monero.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - {{- if hasKey .Values.service "ports" }} - {{- range .Values.service.ports }} - - port: {{ .port }} - name: {{ .name }} - targetPort: {{ .targetPort }} - protocol: {{ .protocol }} - {{- end }} - {{- else }} - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - {{- end }} - selector: - {{- include "monero.selectorLabels" . | nindent 4 }} diff --git a/orchestration/kubernetes/charts/monero/values.yaml b/orchestration/kubernetes/charts/monero/values.yaml deleted file mode 100644 index 8105e003c..000000000 --- a/orchestration/kubernetes/charts/monero/values.yaml +++ /dev/null @@ -1,72 +0,0 @@ -replicaCount: 1 - -image: - repository: serai-dev-monero - pullPolicy: IfNotPresent - tag: "latest" - ports: - - name: p2p - containerPort: 18080 - protocol: TCP - - name: rpc - containerPort: 18081 - protocol: TCP - volumeMounts: - - mountPath: /scripts - name: configmap-volume - args: - - bash - - /scripts/entry-dev.sh - -volumes: - - configMap: - defaultMode: 420 - name: configmap - name: configmap-volume - -configMapFile: "entry-dev.sh" - -imagePullSecrets: [] - -serviceAccount: - create: false - name: "" - -podAnnotations: {} - -podSecurityContext: {} - -securityContext: {} - -service: - type: ClusterIP - ports: - - name: p2p - port: 18080 - targetPort: p2p - protocol: TCP - - name: rpc - port: 18081 - targetPort: rpc - protocol: TCP - -ingress: - enabled: false - className: "" - annotations: {} - hosts: [] - tls: [] - -resources: {} - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/orchestration/kubernetes/charts/serai/Chart.yaml b/orchestration/kubernetes/charts/serai/Chart.yaml deleted file mode 100644 index 69249e1de..000000000 --- a/orchestration/kubernetes/charts/serai/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v2 -name: serai -description: A Helm chart for serai -type: application -version: 0.1.0 diff --git a/orchestration/kubernetes/charts/serai/templates/_helpers.tpl b/orchestration/kubernetes/charts/serai/templates/_helpers.tpl deleted file mode 100644 index 962c2f802..000000000 --- a/orchestration/kubernetes/charts/serai/templates/_helpers.tpl +++ /dev/null @@ -1,42 +0,0 @@ -{{- define "serai-base.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 253 | trimSuffix "-" }} -{{- end }} - -{{- define "serai-base.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 253 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 253 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 253 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{- define "serai-base.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 253 | trimSuffix "-" }} -{{- end }} - -{{- define "serai-base.labels" -}} -helm.sh/chart: {{ include "serai-base.chart" . }} -{{ include "serai-base.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{- define "serai-base.selectorLabels" -}} -app.kubernetes.io/name: {{ include "serai-base.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{- define "serai-base.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "serai-base.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/orchestration/kubernetes/charts/serai/templates/configmap.yaml b/orchestration/kubernetes/charts/serai/templates/configmap.yaml deleted file mode 100644 index bf5502005..000000000 --- a/orchestration/kubernetes/charts/serai/templates/configmap.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ .Release.Name }}-configmap -data: - entry-dev.sh: | -{{ .Values.configMapFile | b64dec | indent 4}} diff --git a/orchestration/kubernetes/charts/serai/templates/deployment.yaml b/orchestration/kubernetes/charts/serai/templates/deployment.yaml deleted file mode 100644 index 35d0d7c07..000000000 --- a/orchestration/kubernetes/charts/serai/templates/deployment.yaml +++ /dev/null @@ -1,88 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "serai-base.fullname" . }} - labels: - {{- include "serai-base.labels" . | nindent 4 }} -spec: - {{- if not .Values.autoscaling.enabled }} - replicas: {{ .Values.replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "serai-base.selectorLabels" . | nindent 6 }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "serai-base.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "serai-base.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- if hasKey .Values.image "ports" }} - ports: - {{- range .Values.image.ports }} - - name: {{ .name }} - containerPort: {{ .containerPort }} - {{- end }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "command" }} - command: - {{- toYaml .Values.image.command | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "args" }} - args: - {{- toYaml .Values.image.args | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "envVariables" }} - env: - {{- toYaml .Values.image.envVariables | nindent 12 }} - {{- end }} - - {{- if hasKey .Values.image "volumeMounts" }} - volumeMounts: - {{- range .Values.image.volumeMounts }} - - mountPath: {{ .mountPath }} - name: {{ $.Release.Name}}-{{ .name }} - {{- end }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- if hasKey .Values "volumes" }} - volumes: - {{- range .Values.volumes }} - - configMap: - defaultMode: {{ .configMap.defaultMode }} - name: {{ $.Release.Name}}-{{ .configMap.name }} - name: {{ $.Release.Name}}-{{ .name }} - {{- end }} - {{- end }} diff --git a/orchestration/kubernetes/charts/serai/templates/ingress.yaml b/orchestration/kubernetes/charts/serai/templates/ingress.yaml deleted file mode 100644 index b020fc02b..000000000 --- a/orchestration/kubernetes/charts/serai/templates/ingress.yaml +++ /dev/null @@ -1,50 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "serai-base.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "serai-base.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if .Values.ingress.className }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if .pathType }} - pathType: {{ .pathType }} - {{- end }} - backend: - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- end }} - {{- end }} -{{- end }} diff --git a/orchestration/kubernetes/charts/serai/templates/service.yaml b/orchestration/kubernetes/charts/serai/templates/service.yaml deleted file mode 100644 index f45d73ef5..000000000 --- a/orchestration/kubernetes/charts/serai/templates/service.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ include "serai-base.fullname" . }} - labels: - {{- include "serai-base.labels" . | nindent 4 }} -spec: - type: {{ .Values.service.type }} - ports: - {{- if hasKey .Values.service "ports" }} - {{- range .Values.service.ports }} - - port: {{ .port }} - name: {{ .name }} - targetPort: {{ .targetPort }} - protocol: {{ .protocol }} - {{- end }} - {{- else }} - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http - {{- end }} - selector: - {{- include "serai-base.selectorLabels" . | nindent 4 }} diff --git a/orchestration/kubernetes/charts/serai/values.yaml b/orchestration/kubernetes/charts/serai/values.yaml deleted file mode 100644 index 9c702dafa..000000000 --- a/orchestration/kubernetes/charts/serai/values.yaml +++ /dev/null @@ -1,92 +0,0 @@ -replicaCount: 1 - -image: - repository: serai - pullPolicy: IfNotPresent - tag: "dev" - ports: - - name: p2p - containerPort: 30333 - protocol: TCP - - name: prometheus - containerPort: 9615 - protocol: TCP - - name: rpc - containerPort: 9933 - protocol: TCP - - name: ws - containerPort: 9944 - protocol: TCP - - volumeMounts: - - mountPath: /scripts - name: configmap-volume - envVariables: - - name: CHAIN - value: dev - - name: NAME - value: base - - name: VALIDATOR - value: - args: - - bash - - /scripts/entry-dev.sh - -volumes: - - configMap: - defaultMode: 420 - name: configmap - name: configmap-volume - -configMapFile: "entry-dev.sh" - -imagePullSecrets: [] - -serviceAccount: - create: false - name: "" - -podAnnotations: {} - -podSecurityContext: {} - -securityContext: {} - -service: - type: ClusterIP - ports: - - name: p2p - port: 30333 - targetPort: p2p - protocol: TCP - - name: prometheus - port: 9615 - targetPort: prometheus - protocol: TCP - - name: rpc - port: 9933 - targetPort: rpc - protocol: TCP - - name: ws - port: 9944 - targetPort: ws - protocol: TCP - -ingress: - enabled: false - className: "" - annotations: {} - hosts: [] - tls: [] - -resources: {} - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - -nodeSelector: {} -tolerations: [] -affinity: {} diff --git a/orchestration/message-queue/Dockerfile b/orchestration/message-queue/Dockerfile deleted file mode 100644 index 63ee9f72e..000000000 --- a/orchestration/message-queue/Dockerfile +++ /dev/null @@ -1,71 +0,0 @@ -FROM debian:bookworm-slim as mimalloc - -RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git -RUN git clone https://github.com/microsoft/mimalloc && \ - cd mimalloc && \ - git checkout 43ce4bd7fd34bcc730c1c7471c99995597415488 && \ - mkdir -p out/secure && \ - cd out/secure && \ - cmake -DMI_SECURE=ON ../.. && \ - make && \ - cp ./libmimalloc-secure.so ../../../libmimalloc.so -FROM rust:1.75-slim-bookworm as builder - -COPY --from=mimalloc libmimalloc.so /usr/lib -RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload - -RUN apt update && apt upgrade -y && apt autoremove -y && apt clean - -# Add dev dependencies -RUN apt install -y pkg-config clang - -# Dependencies for the Serai node -RUN apt install -y make protobuf-compiler - -# Add the wasm toolchain -RUN rustup target add wasm32-unknown-unknown - -# Add files for build -ADD common /serai/common -ADD crypto /serai/crypto -ADD coins /serai/coins -ADD message-queue /serai/message-queue -ADD processor /serai/processor -ADD coordinator /serai/coordinator -ADD substrate /serai/substrate -ADD mini /serai/mini -ADD tests /serai/tests -ADD patches /serai/patches -ADD Cargo.toml /serai -ADD Cargo.lock /serai -ADD AGPL-3.0 /serai - -WORKDIR /serai - -# Mount the caches and build -RUN --mount=type=cache,target=/root/.cargo \ - --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/usr/local/cargo/git \ - --mount=type=cache,target=/serai/target \ - mkdir /serai/bin && \ - cargo build --features parity-db -p serai-message-queue && \ - mv /serai/target/debug/serai-message-queue /serai/bin -FROM debian:bookworm-slim as image - -COPY --from=mimalloc libmimalloc.so /usr/lib -RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload - -RUN apt update && apt upgrade -y && apt autoremove -y && apt clean -# Switch to a non-root user -RUN useradd --system --home /home/message-queue --create-home --shell /sbin/nologin messagequeue -USER messagequeue - -WORKDIR /home/message-queue - -# Copy the Message Queue binary and relevant license -COPY --from=builder --chown=messagequeue /serai/bin/serai-message-queue /bin -COPY --from=builder --chown=messagequeue /serai/AGPL-3.0 . - -# Run message-queue -EXPOSE 2287 -CMD ["serai-message-queue"] diff --git a/orchestration/message-queue/Dockerfile.message-queue b/orchestration/message-queue/Dockerfile.message-queue deleted file mode 100644 index a40fad494..000000000 --- a/orchestration/message-queue/Dockerfile.message-queue +++ /dev/null @@ -1,2 +0,0 @@ - cargo build --features parity-db -p serai-message-queue && \ - mv /serai/target/debug/serai-message-queue /serai/bin diff --git a/orchestration/message-queue/Dockerfile.message-queue.end b/orchestration/message-queue/Dockerfile.message-queue.end deleted file mode 100644 index 2b3a39717..000000000 --- a/orchestration/message-queue/Dockerfile.message-queue.end +++ /dev/null @@ -1,13 +0,0 @@ -# Switch to a non-root user -RUN useradd --system --home /home/message-queue --create-home --shell /sbin/nologin messagequeue -USER messagequeue - -WORKDIR /home/message-queue - -# Copy the Message Queue binary and relevant license -COPY --from=builder --chown=messagequeue /serai/bin/serai-message-queue /bin -COPY --from=builder --chown=messagequeue /serai/AGPL-3.0 . - -# Run message-queue -EXPOSE 2287 -CMD ["serai-message-queue"] diff --git a/orchestration/message-queue/scripts/entry-dev.sh b/orchestration/message-queue/scripts/entry-dev.sh deleted file mode 100755 index 514f1aaf7..000000000 --- a/orchestration/message-queue/scripts/entry-dev.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -export BITCOIN_KEY="0000000000000000000000000000000000000000000000000000000000000000" -export ETHEREUM_KEY="0000000000000000000000000000000000000000000000000000000000000000" -export MONERO_KEY="0000000000000000000000000000000000000000000000000000000000000000" -export COORDINATOR_KEY="0000000000000000000000000000000000000000000000000000000000000000" - -export DB_PATH="./message-queue-db" - -serai-message-queue diff --git a/orchestration/processor/Dockerfile.processor.end b/orchestration/processor/Dockerfile.processor.end deleted file mode 100644 index 410ba5e80..000000000 --- a/orchestration/processor/Dockerfile.processor.end +++ /dev/null @@ -1,15 +0,0 @@ -# Install ca-certificates -RUN apt install -y ca-certificates - -# Switch to a non-root user -RUN useradd --system --create-home --shell /sbin/nologin processor -USER processor - -WORKDIR /home/processor - -# Copy the Processor binary and relevant license -COPY --from=builder --chown=processor /serai/bin/serai-processor /bin/ -COPY --from=builder --chown=processor /serai/AGPL-3.0 . - -# Run processor -CMD ["serai-processor"] diff --git a/orchestration/processor/bitcoin/Dockerfile b/orchestration/processor/bitcoin/Dockerfile deleted file mode 100644 index e5f287e76..000000000 --- a/orchestration/processor/bitcoin/Dockerfile +++ /dev/null @@ -1,73 +0,0 @@ -FROM debian:bookworm-slim as mimalloc - -RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git -RUN git clone https://github.com/microsoft/mimalloc && \ - cd mimalloc && \ - git checkout 43ce4bd7fd34bcc730c1c7471c99995597415488 && \ - mkdir -p out/secure && \ - cd out/secure && \ - cmake -DMI_SECURE=ON ../.. && \ - make && \ - cp ./libmimalloc-secure.so ../../../libmimalloc.so -FROM rust:1.75-slim-bookworm as builder - -COPY --from=mimalloc libmimalloc.so /usr/lib -RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload - -RUN apt update && apt upgrade -y && apt autoremove -y && apt clean - -# Add dev dependencies -RUN apt install -y pkg-config clang - -# Dependencies for the Serai node -RUN apt install -y make protobuf-compiler - -# Add the wasm toolchain -RUN rustup target add wasm32-unknown-unknown - -# Add files for build -ADD common /serai/common -ADD crypto /serai/crypto -ADD coins /serai/coins -ADD message-queue /serai/message-queue -ADD processor /serai/processor -ADD coordinator /serai/coordinator -ADD substrate /serai/substrate -ADD mini /serai/mini -ADD tests /serai/tests -ADD patches /serai/patches -ADD Cargo.toml /serai -ADD Cargo.lock /serai -ADD AGPL-3.0 /serai - -WORKDIR /serai - -# Mount the caches and build -RUN --mount=type=cache,target=/root/.cargo \ - --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/usr/local/cargo/git \ - --mount=type=cache,target=/serai/target \ - mkdir /serai/bin && \ - cargo build --features "binaries parity-db bitcoin" -p serai-processor && \ - mv /serai/target/debug/serai-processor /serai/bin -FROM debian:bookworm-slim as image - -COPY --from=mimalloc libmimalloc.so /usr/lib -RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload - -RUN apt update && apt upgrade -y && apt autoremove -y && apt clean -# Install ca-certificates -RUN apt install -y ca-certificates - -# Switch to a non-root user -RUN useradd --system --create-home --shell /sbin/nologin processor -USER processor - -WORKDIR /home/processor - -# Copy the Processor binary and relevant license -COPY --from=builder --chown=processor /serai/bin/serai-processor /bin/ -COPY --from=builder --chown=processor /serai/AGPL-3.0 . - -# Run processor -CMD ["serai-processor"] diff --git a/orchestration/processor/bitcoin/Dockerfile.processor.bitcoin b/orchestration/processor/bitcoin/Dockerfile.processor.bitcoin deleted file mode 100644 index 685eaf7f9..000000000 --- a/orchestration/processor/bitcoin/Dockerfile.processor.bitcoin +++ /dev/null @@ -1,2 +0,0 @@ - cargo build --features "binaries parity-db bitcoin" -p serai-processor && \ - mv /serai/target/debug/serai-processor /serai/bin diff --git a/orchestration/processor/monero/Dockerfile b/orchestration/processor/monero/Dockerfile deleted file mode 100644 index 4cd7ed51f..000000000 --- a/orchestration/processor/monero/Dockerfile +++ /dev/null @@ -1,73 +0,0 @@ -FROM debian:bookworm-slim as mimalloc - -RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git -RUN git clone https://github.com/microsoft/mimalloc && \ - cd mimalloc && \ - git checkout 43ce4bd7fd34bcc730c1c7471c99995597415488 && \ - mkdir -p out/secure && \ - cd out/secure && \ - cmake -DMI_SECURE=ON ../.. && \ - make && \ - cp ./libmimalloc-secure.so ../../../libmimalloc.so -FROM rust:1.75-slim-bookworm as builder - -COPY --from=mimalloc libmimalloc.so /usr/lib -RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload - -RUN apt update && apt upgrade -y && apt autoremove -y && apt clean - -# Add dev dependencies -RUN apt install -y pkg-config clang - -# Dependencies for the Serai node -RUN apt install -y make protobuf-compiler - -# Add the wasm toolchain -RUN rustup target add wasm32-unknown-unknown - -# Add files for build -ADD common /serai/common -ADD crypto /serai/crypto -ADD coins /serai/coins -ADD message-queue /serai/message-queue -ADD processor /serai/processor -ADD coordinator /serai/coordinator -ADD substrate /serai/substrate -ADD mini /serai/mini -ADD tests /serai/tests -ADD patches /serai/patches -ADD Cargo.toml /serai -ADD Cargo.lock /serai -ADD AGPL-3.0 /serai - -WORKDIR /serai - -# Mount the caches and build -RUN --mount=type=cache,target=/root/.cargo \ - --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/usr/local/cargo/git \ - --mount=type=cache,target=/serai/target \ - mkdir /serai/bin && \ - cargo build --features "binaries parity-db monero" -p serai-processor && \ - mv /serai/target/debug/serai-processor /serai/bin -FROM debian:bookworm-slim as image - -COPY --from=mimalloc libmimalloc.so /usr/lib -RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload - -RUN apt update && apt upgrade -y && apt autoremove -y && apt clean -# Install ca-certificates -RUN apt install -y ca-certificates - -# Switch to a non-root user -RUN useradd --system --create-home --shell /sbin/nologin processor -USER processor - -WORKDIR /home/processor - -# Copy the Processor binary and relevant license -COPY --from=builder --chown=processor /serai/bin/serai-processor /bin/ -COPY --from=builder --chown=processor /serai/AGPL-3.0 . - -# Run processor -CMD ["serai-processor"] diff --git a/orchestration/processor/monero/Dockerfile.processor.monero b/orchestration/processor/monero/Dockerfile.processor.monero deleted file mode 100644 index 920097f12..000000000 --- a/orchestration/processor/monero/Dockerfile.processor.monero +++ /dev/null @@ -1,2 +0,0 @@ - cargo build --features "binaries parity-db monero" -p serai-processor && \ - mv /serai/target/debug/serai-processor /serai/bin diff --git a/orchestration/processor/scripts/entry-dev.sh b/orchestration/processor/scripts/entry-dev.sh deleted file mode 100755 index 545457d83..000000000 --- a/orchestration/processor/scripts/entry-dev.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -export MESSAGE_QUEUE_KEY="0000000000000000000000000000000000000000000000000000000000000000" -export MESSAGE_QUEUE_RPC="http://127.0.0.1:2287" - -export DB_PATH="./processor-bitcoin-db" -export ENTROPY="0001020304050607080910111213141516171819202122232425262728293031" -export NETWORK="bitcoin" -export NETWORK_RPC_LOGIN="serai:seraidex" -export NETWORK_RPC_HOSTNAME="127.0.0.1" -export NETWORK_RPC_PORT="18443" - -serai-processor diff --git a/orchestration/runtime/Dockerfile b/orchestration/runtime/Dockerfile index fd8ecf077..c91731f24 100644 --- a/orchestration/runtime/Dockerfile +++ b/orchestration/runtime/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.75.0-slim-bookworm as builder +FROM --platform=linux/amd64 rust:1.75.0-slim-bookworm as builder # Move to a Debian package snapshot RUN rm -rf /etc/apt/sources.list.d/debian.sources && \ @@ -13,6 +13,7 @@ RUN apt install clang -y RUN rustup target add wasm32-unknown-unknown # Add files for build +ADD patches /serai/patches ADD common /serai/common ADD crypto /serai/crypto ADD coins /serai/coins @@ -20,9 +21,10 @@ ADD message-queue /serai/message-queue ADD processor /serai/processor ADD coordinator /serai/coordinator ADD substrate /serai/substrate +ADD orchestration/Cargo.toml /serai/orchestration/Cargo.toml +ADD orchestration/src /serai/orchestration/src ADD mini /serai/mini ADD tests /serai/tests -ADD patches /serai/patches ADD Cargo.toml /serai ADD Cargo.lock /serai ADD AGPL-3.0 /serai diff --git a/orchestration/serai/Dockerfile b/orchestration/serai/Dockerfile deleted file mode 100644 index c06205f46..000000000 --- a/orchestration/serai/Dockerfile +++ /dev/null @@ -1,71 +0,0 @@ -FROM debian:bookworm-slim as mimalloc - -RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git -RUN git clone https://github.com/microsoft/mimalloc && \ - cd mimalloc && \ - git checkout 43ce4bd7fd34bcc730c1c7471c99995597415488 && \ - mkdir -p out/secure && \ - cd out/secure && \ - cmake -DMI_SECURE=ON ../.. && \ - make && \ - cp ./libmimalloc-secure.so ../../../libmimalloc.so -FROM rust:1.75-slim-bookworm as builder - -COPY --from=mimalloc libmimalloc.so /usr/lib -RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload - -RUN apt update && apt upgrade -y && apt autoremove -y && apt clean - -# Add dev dependencies -RUN apt install -y pkg-config clang - -# Dependencies for the Serai node -RUN apt install -y make protobuf-compiler - -# Add the wasm toolchain -RUN rustup target add wasm32-unknown-unknown - -# Add files for build -ADD common /serai/common -ADD crypto /serai/crypto -ADD coins /serai/coins -ADD message-queue /serai/message-queue -ADD processor /serai/processor -ADD coordinator /serai/coordinator -ADD substrate /serai/substrate -ADD mini /serai/mini -ADD tests /serai/tests -ADD patches /serai/patches -ADD Cargo.toml /serai -ADD Cargo.lock /serai -ADD AGPL-3.0 /serai - -WORKDIR /serai - -# Mount the caches and build -RUN --mount=type=cache,target=/root/.cargo \ - --mount=type=cache,target=/usr/local/cargo/registry \ - --mount=type=cache,target=/usr/local/cargo/git \ - --mount=type=cache,target=/serai/target \ - mkdir /serai/bin && \ - cargo build --release -p serai-node && \ - mv /serai/target/release/serai-node /serai/bin -FROM debian:bookworm-slim as image - -COPY --from=mimalloc libmimalloc.so /usr/lib -RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload - -RUN apt update && apt upgrade -y && apt autoremove -y && apt clean -# Switch to a non-root user -RUN useradd --system --home /home/serai --shell /sbin/nologin serai -USER serai - -WORKDIR /home/serai - -# Copy the Serai binary and relevant license -COPY --from=builder --chown=serai /serai/bin/serai-node /bin/ -COPY --from=builder --chown=serai /serai/AGPL-3.0 . - -# Run node -EXPOSE 30333 9615 9933 9944 -CMD ["serai-node"] diff --git a/orchestration/serai/Dockerfile.serai b/orchestration/serai/Dockerfile.serai deleted file mode 100644 index 839ec923a..000000000 --- a/orchestration/serai/Dockerfile.serai +++ /dev/null @@ -1,2 +0,0 @@ - cargo build --release -p serai-node && \ - mv /serai/target/release/serai-node /serai/bin diff --git a/orchestration/serai/Dockerfile.serai.end b/orchestration/serai/Dockerfile.serai.end deleted file mode 100644 index 52c16ad37..000000000 --- a/orchestration/serai/Dockerfile.serai.end +++ /dev/null @@ -1,13 +0,0 @@ -# Switch to a non-root user -RUN useradd --system --home /home/serai --shell /sbin/nologin serai -USER serai - -WORKDIR /home/serai - -# Copy the Serai binary and relevant license -COPY --from=builder --chown=serai /serai/bin/serai-node /bin/ -COPY --from=builder --chown=serai /serai/AGPL-3.0 . - -# Run node -EXPOSE 30333 9615 9933 9944 -CMD ["serai-node"] diff --git a/orchestration/serai/scripts/entry-dev.sh b/orchestration/serai/scripts/entry-dev.sh deleted file mode 100755 index 4ae4afe03..000000000 --- a/orchestration/serai/scripts/entry-dev.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -if [[ -z $VALIDATOR ]]; then - serai-node --tmp --chain $CHAIN --name $NAME -else - serai-node --tmp --chain $CHAIN --$NAME -fi diff --git a/orchestration/coins/bitcoin/Dockerfile b/orchestration/src/coins/bitcoin.rs similarity index 56% rename from orchestration/coins/bitcoin/Dockerfile rename to orchestration/src/coins/bitcoin.rs index e2bd81d16..13c86bad7 100644 --- a/orchestration/coins/bitcoin/Dockerfile +++ b/orchestration/src/coins/bitcoin.rs @@ -1,14 +1,10 @@ -FROM debian:bookworm-slim as mimalloc - -RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git -RUN git clone https://github.com/microsoft/mimalloc && \ - cd mimalloc && \ - git checkout 43ce4bd7fd34bcc730c1c7471c99995597415488 && \ - mkdir -p out/secure && \ - cd out/secure && \ - cmake -DMI_SECURE=ON ../.. && \ - make && \ - cp ./libmimalloc-secure.so ../../../libmimalloc.so +use std::{path::Path}; + +use crate::{Network, Os, mimalloc, os, write_dockerfile}; + +pub fn bitcoin(orchestration_path: &Path, network: Network) { + #[rustfmt::skip] + const DOWNLOAD_BITCOIN: &str = r#" FROM alpine:latest as bitcoin ENV BITCOIN_VERSION=26.0 @@ -31,19 +27,30 @@ RUN grep bitcoin-${BITCOIN_VERSION}-$(uname -m)-linux-gnu.tar.gz SHA256SUMS | sh # Prepare Image RUN tar xzvf bitcoin-${BITCOIN_VERSION}-$(uname -m)-linux-gnu.tar.gz RUN mv bitcoin-${BITCOIN_VERSION}/bin/bitcoind . -FROM debian:bookworm-slim as image +"#; -COPY --from=mimalloc libmimalloc.so /usr/lib -RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload - -RUN apt update && apt upgrade -y && apt autoremove -y && apt clean -# Switch to a non-root user -RUN useradd --system --create-home --shell /sbin/nologin bitcoin -USER bitcoin -WORKDIR /home/bitcoin + let setup = mimalloc(Os::Debian).to_string() + DOWNLOAD_BITCOIN; + let run_bitcoin = format!( + r#" COPY --from=bitcoin --chown=bitcoin bitcoind /bin -COPY ./scripts /scripts -EXPOSE 8332 8333 18332 18333 18443 18444 -# VOLUME ["/home/bitcoin/.bitcoin"] +EXPOSE 8332 8333 + +ADD /orchestration/{}/coins/bitcoin/run.sh / +CMD ["/run.sh"] +"#, + network.label() + ); + + let run = + os(Os::Debian, "RUN mkdir /volume && chown bitcoin:bitcoin /volume", "bitcoin") + &run_bitcoin; + let res = setup + &run; + + let mut bitcoin_path = orchestration_path.to_path_buf(); + bitcoin_path.push("coins"); + bitcoin_path.push("bitcoin"); + bitcoin_path.push("Dockerfile"); + + write_dockerfile(bitcoin_path, &res); +} diff --git a/orchestration/src/coins/ethereum.rs b/orchestration/src/coins/ethereum.rs new file mode 100644 index 000000000..2e15d3709 --- /dev/null +++ b/orchestration/src/coins/ethereum.rs @@ -0,0 +1,5 @@ +use std::path::Path; + +pub fn ethereum(_orchestration_path: &Path) { + // TODO +} diff --git a/orchestration/src/coins/mod.rs b/orchestration/src/coins/mod.rs new file mode 100644 index 000000000..cab1eb86e --- /dev/null +++ b/orchestration/src/coins/mod.rs @@ -0,0 +1,8 @@ +mod bitcoin; +pub use bitcoin::*; + +mod ethereum; +pub use ethereum::*; + +mod monero; +pub use monero::*; diff --git a/orchestration/src/coins/monero.rs b/orchestration/src/coins/monero.rs new file mode 100644 index 000000000..1113fcf64 --- /dev/null +++ b/orchestration/src/coins/monero.rs @@ -0,0 +1,87 @@ +use std::{path::Path}; + +use crate::{Network, Os, mimalloc, write_dockerfile}; + +fn monero_internal( + network: Network, + os: Os, + orchestration_path: &Path, + folder: &str, + monero_binary: &str, + ports: &str, +) { + const MONERO_VERSION: &str = "0.18.3.1"; + + let arch = match std::env::consts::ARCH { + // We probably would run this without issues yet it's not worth needing to provide support for + "x86" | "arm" => panic!("unsupported architecture, please download a 64-bit OS"), + "x86_64" => "x64", + "aarch64" => "armv8", + _ => panic!("unsupported architecture"), + }; + + #[rustfmt::skip] + let download_monero = format!(r#" +FROM alpine:latest as monero + +RUN apk --no-cache add gnupg + +# Download Monero +RUN wget https://downloads.getmonero.org/cli/monero-linux-{arch}-v{MONERO_VERSION}.tar.bz2 + +# Verify Binary -- fingerprint from https://github.com/monero-project/monero-site/issues/1949 +ADD orchestration/{}/coins/monero/hashes-v{MONERO_VERSION}.txt . +RUN gpg --keyserver hkp://keyserver.ubuntu.com:80 --keyserver-options no-self-sigs-only --receive-keys 81AC591FE9C4B65C5806AFC3F0AF4D462A0BDF92 && \ + gpg --verify hashes-v{MONERO_VERSION}.txt && \ + grep "$(sha256sum monero-linux-{arch}-v{MONERO_VERSION}.tar.bz2 | cut -c 1-64)" hashes-v{MONERO_VERSION}.txt + +# Extract it +RUN tar -xvjf monero-linux-{arch}-v{MONERO_VERSION}.tar.bz2 --strip-components=1 +"#, + network.label(), + ); + + let setup = mimalloc(os).to_string() + &download_monero; + + let run_monero = format!( + r#" +COPY --from=monero --chown=monero {monero_binary} /bin + +EXPOSE {ports} + +ADD /orchestration/{}/coins/{folder}/run.sh / +CMD ["/run.sh"] +"#, + network.label(), + ); + + let run = crate::os( + os, + &("RUN mkdir /volume && chown monero /volume\r\n".to_string() + + if os == Os::Alpine { "RUN apk --no-cache add gcompat" } else { "" }), + "monero", + ) + &run_monero; + let res = setup + &run; + + let mut monero_path = orchestration_path.to_path_buf(); + monero_path.push("coins"); + monero_path.push(folder); + monero_path.push("Dockerfile"); + + write_dockerfile(monero_path, &res); +} + +pub fn monero(orchestration_path: &Path, network: Network) { + monero_internal(network, Os::Alpine, orchestration_path, "monero", "monerod", "18080 18081") +} + +pub fn monero_wallet_rpc(orchestration_path: &Path) { + monero_internal( + Network::Dev, + Os::Debian, + orchestration_path, + "monero-wallet-rpc", + "monero-wallet-rpc", + "18082", + ) +} diff --git a/orchestration/src/coordinator.rs b/orchestration/src/coordinator.rs new file mode 100644 index 000000000..a8556a009 --- /dev/null +++ b/orchestration/src/coordinator.rs @@ -0,0 +1,62 @@ +use std::{path::Path}; + +use zeroize::Zeroizing; + +use ciphersuite::{group::ff::PrimeField, Ciphersuite, Ristretto}; + +use crate::{Network, Os, mimalloc, os, build_serai_service, write_dockerfile}; + +#[allow(clippy::needless_pass_by_value)] +pub fn coordinator( + orchestration_path: &Path, + network: Network, + coordinator_key: Zeroizing<::F>, + serai_key: Zeroizing<::F>, +) { + let db = network.db(); + let longer_reattempts = if network == Network::Dev { "longer-reattempts" } else { "" }; + let setup = mimalloc(Os::Debian).to_string() + + &build_serai_service( + network.release(), + &format!("{db} {longer_reattempts}"), + "serai-coordinator", + ); + + const ADDITIONAL_ROOT: &str = r#" +# Install ca-certificates +RUN apt install -y ca-certificates +"#; + + let env_vars = [ + ("MESSAGE_QUEUE_RPC", format!("serai-{}-message-queue", network.label())), + ("MESSAGE_QUEUE_KEY", hex::encode(coordinator_key.to_repr())), + ("DB_PATH", "./coordinator-db".to_string()), + ("SERAI_KEY", hex::encode(serai_key.to_repr())), + ("SERAI_HOSTNAME", format!("serai-{}-serai", network.label())), + ("RUST_LOG", "serai_coordinator=debug,tributary_chain=debug,tendermint=debug".to_string()), + ]; + let mut env_vars_str = String::new(); + for (env_var, value) in env_vars { + env_vars_str += &format!(r#"{env_var}=${{{env_var}:="{value}"}} "#); + } + + let run_coordinator = format!( + r#" +# Copy the Coordinator binary and relevant license +COPY --from=builder --chown=coordinator /serai/bin/serai-coordinator /bin/ +COPY --from=builder --chown=coordinator /serai/AGPL-3.0 . + +# Run coordinator +CMD {env_vars_str} serai-coordinator +"# + ); + + let run = os(Os::Debian, ADDITIONAL_ROOT, "coordinator") + &run_coordinator; + let res = setup + &run; + + let mut coordinator_path = orchestration_path.to_path_buf(); + coordinator_path.push("coordinator"); + coordinator_path.push("Dockerfile"); + + write_dockerfile(coordinator_path, &res); +} diff --git a/orchestration/src/docker.rs b/orchestration/src/docker.rs new file mode 100644 index 000000000..2523bfaa0 --- /dev/null +++ b/orchestration/src/docker.rs @@ -0,0 +1,47 @@ +use std::{collections::HashSet, path::Path, env, process::Command}; + +use crate::Network; + +pub fn build(orchestration_path: &Path, network: Network, name: &str) { + let mut repo_path = env::current_exe().unwrap(); + repo_path.pop(); + if repo_path.as_path().ends_with("deps") { + repo_path.pop(); + } + assert!(repo_path.as_path().ends_with("debug") || repo_path.as_path().ends_with("release")); + repo_path.pop(); + assert!(repo_path.as_path().ends_with("target")); + repo_path.pop(); + + let mut dockerfile_path = orchestration_path.to_path_buf(); + if HashSet::from(["bitcoin", "ethereum", "monero", "monero-wallet-rpc"]).contains(name) { + dockerfile_path = dockerfile_path.join("coins"); + } + if name.contains("-processor") { + dockerfile_path = + dockerfile_path.join("processor").join(name.split('-').next().unwrap()).join("Dockerfile"); + } else { + dockerfile_path = dockerfile_path.join(name).join("Dockerfile"); + } + + println!("Building {}...", &name); + + if !Command::new("docker") + .current_dir(&repo_path) + .arg("build") + .arg("-f") + .arg(dockerfile_path) + .arg(".") + .arg("-t") + .arg(format!("serai-{}-{name}-img", network.label())) + .spawn() + .unwrap() + .wait() + .unwrap() + .success() + { + panic!("failed to build {name}"); + } + + println!("Built!"); +} diff --git a/orchestration/src/main.rs b/orchestration/src/main.rs new file mode 100644 index 000000000..60e4ee740 --- /dev/null +++ b/orchestration/src/main.rs @@ -0,0 +1,445 @@ +// TODO: Generate randomized RPC credentials for all services +// TODO: Generate keys for a validator and the infra + +use core::ops::Deref; +use std::{collections::HashSet, env, path::PathBuf, io::Write, fs, process::Command}; + +use zeroize::Zeroizing; + +use rand_core::{RngCore, SeedableRng, OsRng}; +use rand_chacha::ChaCha20Rng; + +use transcript::{Transcript, RecommendedTranscript}; + +use ciphersuite::{ + group::{ + ff::{Field, PrimeField}, + GroupEncoding, + }, + Ciphersuite, Ristretto, +}; + +mod mimalloc; +use mimalloc::mimalloc; + +mod coins; +use coins::*; + +mod message_queue; +use message_queue::message_queue; + +mod processor; +use processor::processor; + +mod coordinator; +use coordinator::coordinator; + +mod serai; +use serai::serai; + +mod docker; + +#[global_allocator] +static ALLOCATOR: zalloc::ZeroizingAlloc = + zalloc::ZeroizingAlloc(std::alloc::System); + +#[derive(Clone, Copy, PartialEq, Eq, Debug, PartialOrd, Ord, Hash)] +pub enum Network { + Dev, + Testnet, +} + +impl Network { + pub fn db(&self) -> &'static str { + match self { + Network::Dev => "parity-db", + Network::Testnet => "rocksdb", + } + } + + pub fn release(&self) -> bool { + match self { + Network::Dev => false, + Network::Testnet => true, + } + } + + pub fn label(&self) -> &'static str { + match self { + Network::Dev => "dev", + Network::Testnet => "testnet", + } + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug, PartialOrd, Ord, Hash)] +enum Os { + Alpine, + Debian, +} + +fn os(os: Os, additional_root: &str, user: &str) -> String { + match os { + Os::Alpine => format!( + r#" +FROM alpine:latest as image + +COPY --from=mimalloc-alpine libmimalloc.so /usr/lib +ENV LD_PRELOAD=libmimalloc.so + +RUN apk update && apk upgrade + +# System user (not a human), shell of nologin, no password assigned +RUN adduser -S -s /sbin/nologin -D {user} + +{additional_root} + +# Switch to a non-root user +USER {user} + +WORKDIR /home/{user} +"# + ), + + Os::Debian => format!( + r#" +FROM debian:bookworm-slim as image + +COPY --from=mimalloc-debian libmimalloc.so /usr/lib +RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload + +RUN apt update && apt upgrade -y && apt autoremove -y && apt clean + +RUN useradd --system --create-home --shell /sbin/nologin {user} + +{additional_root} + +# Switch to a non-root user +USER {user} + +WORKDIR /home/{user} +"# + ), + } +} + +fn build_serai_service(release: bool, features: &str, package: &str) -> String { + let profile = if release { "release" } else { "debug" }; + let profile_flag = if release { "--release" } else { "" }; + + format!( + r#" +FROM rust:1.75-slim-bookworm as builder + +COPY --from=mimalloc-debian libmimalloc.so /usr/lib +RUN echo "/usr/lib/libmimalloc.so" >> /etc/ld.so.preload + +RUN apt update && apt upgrade -y && apt autoremove -y && apt clean + +# Add dev dependencies +RUN apt install -y pkg-config clang + +# Dependencies for the Serai node +RUN apt install -y make protobuf-compiler + +# Add the wasm toolchain +RUN rustup target add wasm32-unknown-unknown + +# Add files for build +ADD patches /serai/patches +ADD common /serai/common +ADD crypto /serai/crypto +ADD coins /serai/coins +ADD message-queue /serai/message-queue +ADD processor /serai/processor +ADD coordinator /serai/coordinator +ADD substrate /serai/substrate +ADD orchestration/Cargo.toml /serai/orchestration/Cargo.toml +ADD orchestration/src /serai/orchestration/src +ADD mini /serai/mini +ADD tests /serai/tests +ADD Cargo.toml /serai +ADD Cargo.lock /serai +ADD AGPL-3.0 /serai + +WORKDIR /serai + +# Mount the caches and build +RUN --mount=type=cache,target=/root/.cargo \ + --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git \ + --mount=type=cache,target=/serai/target \ + mkdir /serai/bin && \ + cargo build {profile_flag} --features "{features}" -p {package} && \ + mv /serai/target/{profile}/{package} /serai/bin +"# + ) +} + +pub fn write_dockerfile(path: PathBuf, dockerfile: &str) { + if let Ok(existing) = fs::read_to_string(&path).as_ref() { + if existing == dockerfile { + return; + } + } + fs::File::create(path).unwrap().write_all(dockerfile.as_bytes()).unwrap(); +} + +fn orchestration_path(network: Network) -> PathBuf { + let mut repo_path = env::current_exe().unwrap(); + repo_path.pop(); + assert!(repo_path.as_path().ends_with("debug")); + repo_path.pop(); + assert!(repo_path.as_path().ends_with("target")); + repo_path.pop(); + + let mut orchestration_path = repo_path.clone(); + orchestration_path.push("orchestration"); + orchestration_path.push(network.label()); + orchestration_path +} + +fn dockerfiles(network: Network) { + let orchestration_path = orchestration_path(network); + + bitcoin(&orchestration_path, network); + ethereum(&orchestration_path); + monero(&orchestration_path, network); + if network == Network::Dev { + monero_wallet_rpc(&orchestration_path); + } + + // TODO: Generate infra keys in key_gen, yet service entropy here? + + // Generate entropy for the infrastructure keys + let mut entropy = Zeroizing::new([0; 32]); + // Only use actual entropy if this isn't a development environment + if network != Network::Dev { + OsRng.fill_bytes(entropy.as_mut()); + } + let mut transcript = RecommendedTranscript::new(b"Serai Orchestrator Transcript"); + transcript.append_message(b"entropy", entropy); + let mut new_rng = |label| ChaCha20Rng::from_seed(transcript.rng_seed(label)); + + let mut message_queue_keys_rng = new_rng(b"message_queue_keys"); + let mut key_pair = || { + let key = Zeroizing::new(::F::random(&mut message_queue_keys_rng)); + let public = Ristretto::generator() * key.deref(); + (key, public) + }; + let coordinator_key = key_pair(); + let bitcoin_key = key_pair(); + let ethereum_key = key_pair(); + let monero_key = key_pair(); + + message_queue( + &orchestration_path, + network, + coordinator_key.1, + bitcoin_key.1, + ethereum_key.1, + monero_key.1, + ); + + let mut processor_entropy_rng = new_rng(b"processor_entropy"); + let mut new_entropy = || { + let mut res = Zeroizing::new([0; 32]); + processor_entropy_rng.fill_bytes(res.as_mut()); + res + }; + processor( + &orchestration_path, + network, + "bitcoin", + coordinator_key.1, + bitcoin_key.0, + new_entropy(), + ); + processor( + &orchestration_path, + network, + "ethereum", + coordinator_key.1, + ethereum_key.0, + new_entropy(), + ); + processor(&orchestration_path, network, "monero", coordinator_key.1, monero_key.0, new_entropy()); + + let serai_key = { + let serai_key = Zeroizing::new( + fs::read(home::home_dir().unwrap().join(".serai").join(network.label()).join("key")) + .expect("couldn't read key for this network"), + ); + let mut serai_key_repr = + Zeroizing::new(<::F as PrimeField>::Repr::default()); + serai_key_repr.as_mut().copy_from_slice(serai_key.as_ref()); + Zeroizing::new(::F::from_repr(*serai_key_repr).unwrap()) + }; + + coordinator(&orchestration_path, network, coordinator_key.0, serai_key); + + serai(&orchestration_path, network); +} + +fn key_gen(network: Network) { + let serai_dir = home::home_dir().unwrap().join(".serai").join(network.label()); + let key_file = serai_dir.join("key"); + if fs::File::open(&key_file).is_ok() { + println!("already created key"); + return; + } + + let key = ::F::random(&mut OsRng); + + let _ = fs::create_dir_all(&serai_dir); + fs::write(key_file, key.to_repr()).expect("couldn't write key"); + + println!( + "Public Key: {}", + hex::encode((::generator() * key).to_bytes()) + ); +} + +fn start(network: Network, services: HashSet) { + // Create the serai network + Command::new("docker") + .arg("network") + .arg("create") + .arg("--driver") + .arg("bridge") + .arg("serai") + .output() + .unwrap(); + + for service in services { + println!("Starting {service}"); + let name = match service.as_ref() { + "serai" => "serai", + "coordinator" => "coordinator", + "message-queue" => "message-queue", + "bitcoin-daemon" => "bitcoin", + "bitcoin-processor" => "bitcoin-processor", + "monero-daemon" => "monero", + "monero-processor" => "monero-processor", + "monero-wallet-rpc" => "monero-wallet-rpc", + _ => panic!("starting unrecognized service"), + }; + + // Build it + println!("Building {service}"); + docker::build(&orchestration_path(network), network, name); + + let docker_name = format!("serai-{}-{name}", network.label()); + let docker_image = format!("{docker_name}-img"); + if !Command::new("docker") + .arg("container") + .arg("inspect") + .arg(&docker_name) + .status() + .unwrap() + .success() + { + // Create the docker container + println!("Creating new container for {service}"); + let volume = format!("serai-{}-{name}-volume:/volume", network.label()); + let mut command = Command::new("docker"); + let command = command.arg("create").arg("--name").arg(&docker_name); + let command = command.arg("--network").arg("serai"); + let command = match name { + "bitcoin" => { + if network == Network::Dev { + command.arg("-p").arg("8332:8332") + } else { + command.arg("--volume").arg(volume) + } + } + "monero" => { + if network == Network::Dev { + command.arg("-p").arg("18081:18081") + } else { + command.arg("--volume").arg(volume) + } + } + "monero-wallet-rpc" => { + assert_eq!(network, Network::Dev, "monero-wallet-rpc is only for dev"); + command.arg("-p").arg("18082:18082") + } + _ => command, + }; + assert!( + command.arg(docker_image).status().unwrap().success(), + "couldn't create the container" + ); + } + + // Start it + // TODO: Check it successfully started + println!("Starting existing container for {service}"); + let _ = Command::new("docker").arg("start").arg(docker_name).output(); + } +} + +fn main() { + let help = || -> ! { + println!( + r#" +Serai Orchestrator v0.0.1 + +Commands: + key_gen *network* + Generates a key for the validator. + + setup *network* + Generate infrastructure keys and the Dockerfiles for every Serai service. + + start *network* [service1, service2...] + Start the specified services for the specified network ("dev" or "testnet"). + + - `serai` + - `coordinator` + - `message-queue` + - `bitcoin-daemon` + - `bitcoin-processor` + - `monero-daemon` + - `monero-processor` + - `monero-wallet-rpc` (if "dev") + + are valid services. + + `*network*-processor` will automatically start `*network*-daemon`. +"# + ); + std::process::exit(1); + }; + + let mut args = env::args(); + args.next(); + let command = args.next(); + let network = match args.next().as_ref().map(AsRef::as_ref) { + Some("dev") => Network::Dev, + Some("testnet") => Network::Testnet, + Some(_) => panic!(r#"unrecognized network. only "dev" and "testnet" are recognized"#), + None => help(), + }; + + match command.as_ref().map(AsRef::as_ref) { + Some("key_gen") => { + key_gen(network); + } + Some("setup") => { + dockerfiles(network); + } + Some("start") => { + let mut services = HashSet::new(); + for arg in args { + if let Some(ext_network) = arg.strip_suffix("-processor") { + services.insert(ext_network.to_string() + "-daemon"); + } + services.insert(arg); + } + + start(network, services); + } + _ => help(), + } +} diff --git a/orchestration/src/message_queue.rs b/orchestration/src/message_queue.rs new file mode 100644 index 000000000..ef6bdcbfc --- /dev/null +++ b/orchestration/src/message_queue.rs @@ -0,0 +1,51 @@ +use std::{path::Path}; + +use ciphersuite::{group::GroupEncoding, Ciphersuite, Ristretto}; + +use crate::{Network, Os, mimalloc, os, build_serai_service, write_dockerfile}; + +pub fn message_queue( + orchestration_path: &Path, + network: Network, + coordinator_key: ::G, + bitcoin_key: ::G, + ethereum_key: ::G, + monero_key: ::G, +) { + let setup = mimalloc(Os::Debian).to_string() + + &build_serai_service(network.release(), network.db(), "serai-message-queue"); + + let env_vars = [ + ("COORDINATOR_KEY", hex::encode(coordinator_key.to_bytes())), + ("BITCOIN_KEY", hex::encode(bitcoin_key.to_bytes())), + ("ETHEREUM_KEY", hex::encode(ethereum_key.to_bytes())), + ("MONERO_KEY", hex::encode(monero_key.to_bytes())), + ("DB_PATH", "./message-queue-db".to_string()), + ("RUST_LOG", "serai_message_queue=trace".to_string()), + ]; + let mut env_vars_str = String::new(); + for (env_var, value) in env_vars { + env_vars_str += &format!(r#"{env_var}=${{{env_var}:="{value}"}} "#); + } + + let run_message_queue = format!( + r#" +# Copy the Message Queue binary and relevant license +COPY --from=builder --chown=messagequeue /serai/bin/serai-message-queue /bin +COPY --from=builder --chown=messagequeue /serai/AGPL-3.0 . + +# Run message-queue +EXPOSE 2287 +CMD {env_vars_str} serai-message-queue +"# + ); + + let run = os(Os::Debian, "", "messagequeue") + &run_message_queue; + let res = setup + &run; + + let mut message_queue_path = orchestration_path.to_path_buf(); + message_queue_path.push("message-queue"); + message_queue_path.push("Dockerfile"); + + write_dockerfile(message_queue_path, &res); +} diff --git a/orchestration/src/mimalloc.rs b/orchestration/src/mimalloc.rs new file mode 100644 index 000000000..cb400ca60 --- /dev/null +++ b/orchestration/src/mimalloc.rs @@ -0,0 +1,36 @@ +use crate::Os; + +pub fn mimalloc(os: Os) -> &'static str { + const ALPINE_MIMALLOC: &str = r#" +FROM alpine:latest as mimalloc-alpine + +RUN apk update && apk upgrade && apk --no-cache add gcc g++ libc-dev make cmake git +RUN git clone https://github.com/microsoft/mimalloc && \ + cd mimalloc && \ + git checkout 43ce4bd7fd34bcc730c1c7471c99995597415488 && \ + mkdir -p out/secure && \ + cd out/secure && \ + cmake -DMI_SECURE=ON ../.. && \ + make && \ + cp ./libmimalloc-secure.so ../../../libmimalloc.so +"#; + + const DEBIAN_MIMALLOC: &str = r#" +FROM debian:bookworm-slim as mimalloc-debian + +RUN apt update && apt upgrade -y && apt install -y gcc g++ make cmake git +RUN git clone https://github.com/microsoft/mimalloc && \ + cd mimalloc && \ + git checkout 43ce4bd7fd34bcc730c1c7471c99995597415488 && \ + mkdir -p out/secure && \ + cd out/secure && \ + cmake -DMI_SECURE=ON ../.. && \ + make && \ + cp ./libmimalloc-secure.so ../../../libmimalloc.so +"#; + + match os { + Os::Alpine => ALPINE_MIMALLOC, + Os::Debian => DEBIAN_MIMALLOC, + } +} diff --git a/orchestration/src/processor.rs b/orchestration/src/processor.rs new file mode 100644 index 000000000..e2afde09d --- /dev/null +++ b/orchestration/src/processor.rs @@ -0,0 +1,78 @@ +use std::{path::Path}; + +use zeroize::Zeroizing; + +use ciphersuite::{group::ff::PrimeField, Ciphersuite, Ristretto}; + +use crate::{Network, Os, mimalloc, os, build_serai_service, write_dockerfile}; + +#[allow(clippy::needless_pass_by_value)] +pub fn processor( + orchestration_path: &Path, + network: Network, + coin: &'static str, + _coordinator_key: ::G, + coin_key: Zeroizing<::F>, + entropy: Zeroizing<[u8; 32]>, +) { + let setup = mimalloc(Os::Debian).to_string() + + &build_serai_service( + network.release(), + &format!("binaries {} {coin}", network.db()), + "serai-processor", + ); + + const ADDITIONAL_ROOT: &str = r#" +# Install ca-certificates +RUN apt install -y ca-certificates +"#; + + // TODO: Randomly generate these + const RPC_USER: &str = "serai"; + const RPC_PASS: &str = "seraidex"; + // TODO: Isolate networks + let hostname = format!("serai-{}-{coin}", network.label()); + let port = match coin { + "bitcoin" => 8332, + "ethereum" => return, // TODO + "monero" => 18081, + _ => panic!("unrecognized external network"), + }; + + let env_vars = [ + ("MESSAGE_QUEUE_RPC", format!("serai-{}-message_queue", network.label())), + ("MESSAGE_QUEUE_KEY", hex::encode(coin_key.to_repr())), + ("ENTROPY", hex::encode(entropy.as_ref())), + ("NETWORK", coin.to_string()), + ("NETWORK_RPC_LOGIN", format!("{RPC_USER}:{RPC_PASS}")), + ("NETWORK_RPC_HOSTNAME", hostname), + ("NETWORK_RPC_PORT", format!("{port}")), + ("DB_PATH", "./processor-db".to_string()), + ("RUST_LOG", "serai_processor=debug".to_string()), + ]; + let mut env_vars_str = String::new(); + for (env_var, value) in env_vars { + env_vars_str += &format!(r#"{env_var}=${{{env_var}:="{value}"}} "#); + } + + let run_processor = format!( + r#" +# Copy the Processor binary and relevant license +COPY --from=builder --chown=processor /serai/bin/serai-processor /bin/ +COPY --from=builder --chown=processor /serai/AGPL-3.0 . + +# Run processor +CMD {env_vars_str} serai-processor +"# + ); + + let run = os(Os::Debian, ADDITIONAL_ROOT, "processor") + &run_processor; + let res = setup + &run; + + let mut processor_path = orchestration_path.to_path_buf(); + processor_path.push("processor"); + processor_path.push(coin); + processor_path.push("Dockerfile"); + + write_dockerfile(processor_path, &res); +} diff --git a/orchestration/src/serai.rs b/orchestration/src/serai.rs new file mode 100644 index 000000000..ac677dd58 --- /dev/null +++ b/orchestration/src/serai.rs @@ -0,0 +1,33 @@ +use std::{path::Path}; + +use crate::{Network, Os, mimalloc, os, build_serai_service, write_dockerfile}; + +pub fn serai(orchestration_path: &Path, network: Network) { + // Always builds in release for performance reasons + let setup = mimalloc(Os::Debian).to_string() + &build_serai_service(true, "", "serai-node"); + + // TODO: Review the ports exposed here + let run_serai = format!( + r#" +# Copy the Serai binary and relevant license +COPY --from=builder --chown=serai /serai/bin/serai-node /bin/ +COPY --from=builder --chown=serai /serai/AGPL-3.0 . + +# Run the Serai node +EXPOSE 30333 9615 9933 9944 + +ADD /orchestration/{}/serai/run.sh / +CMD ["/run.sh"] +"#, + network.label() + ); + + let run = os(Os::Debian, "", "serai") + &run_serai; + let res = setup + &run; + + let mut serai_path = orchestration_path.to_path_buf(); + serai_path.push("serai"); + serai_path.push("Dockerfile"); + + write_dockerfile(serai_path, &res); +} diff --git a/orchestration/testnet/coins/bitcoin/run.sh b/orchestration/testnet/coins/bitcoin/run.sh new file mode 100755 index 000000000..dbec375ac --- /dev/null +++ b/orchestration/testnet/coins/bitcoin/run.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +RPC_USER="${RPC_USER:=serai}" +RPC_PASS="${RPC_PASS:=seraidex}" + +bitcoind -txindex -testnet -port=8333 \ + -rpcuser=$RPC_USER -rpcpassword=$RPC_PASS \ + -rpcbind=0.0.0.0 -rpcallowip=0.0.0.0/0 -rpcport=8332 \ + --datadir=/volume diff --git a/orchestration/testnet/coins/ethereum/run.sh b/orchestration/testnet/coins/ethereum/run.sh new file mode 100755 index 000000000..2bb8d868b --- /dev/null +++ b/orchestration/testnet/coins/ethereum/run.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exit 1 diff --git a/orchestration/testnet/coins/monero/hashes-v0.18.3.1.txt b/orchestration/testnet/coins/monero/hashes-v0.18.3.1.txt new file mode 100644 index 000000000..ff23e4a83 --- /dev/null +++ b/orchestration/testnet/coins/monero/hashes-v0.18.3.1.txt @@ -0,0 +1,49 @@ +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA256 + +# This GPG-signed message exists to confirm the SHA256 sums of Monero binaries. +# +# Please verify the signature against the key for binaryFate in the +# source code repository (/utils/gpg_keys). +# +# +## CLI +fc6a93eabc3fd524ff1ceedbf502b8d43c61a7805728b7ed5f9e7204e26b91f5 monero-android-armv7-v0.18.3.1.tar.bz2 +6d9c7d31942dde86ce39757fd55027448ceb260b60b3c8d32ed018211eb4f1e4 monero-android-armv8-v0.18.3.1.tar.bz2 +3e2d9964a9e52c146b4d26b5eb53e691b3ba88e2468dc4fbfee4c318a367a90e monero-freebsd-x64-v0.18.3.1.tar.bz2 +2ea2c8898cbab88f49423f4f6c15f2a94046cb4bbe827493dd061edc0fd5f1ca monero-linux-armv7-v0.18.3.1.tar.bz2 +445032e88dc07e51ac5fff7034752be530d1c4117d8d605100017bcd87c7b21f monero-linux-armv8-v0.18.3.1.tar.bz2 +23af572fdfe3459b9ab97e2e9aa7e3c11021c955d6064b801a27d7e8c21ae09d monero-linux-x64-v0.18.3.1.tar.bz2 +c8553558dece79a4c23e1114fdf638b15e46899d7cf0af41457f18bbbee83986 monero-linux-x86-v0.18.3.1.tar.bz2 +915288b023cb5811e626e10052adc6ac5323dd283c5a25b91059b0fb86a21fb6 monero-mac-armv8-v0.18.3.1.tar.bz2 +7f8bd9364ef16482b418aa802a65be0e4cc660c794bb5d77b2d17bc84427883a monero-mac-x64-v0.18.3.1.tar.bz2 +35dcc4bee4caad3442659d37837e0119e4649a77f2e3b5e80dd6d9b8fc4fb6ad monero-win-x64-v0.18.3.1.zip +5bcbeddce32b50ebe18289d0560ebf779441526ec84d73b6a83094f092365271 monero-win-x86-v0.18.3.1.zip +4d217e2aa61a6f105054dddbab52c0301f52766e88783de2480316c5a8661e0c monero-source-v0.18.3.1.tar.bz2 +# +## GUI +792271147ad71a2eaa02fc37d61d72cd92f2f9857dcc09ea032f48481f87e279 monero-gui-install-win-x64-v0.18.3.1.exe +06f6e600db51205116d52522964cf9b96337d7b5cb1e101730ccb0039b30e15b monero-gui-linux-x64-v0.18.3.1.tar.bz2 +b0c8d07f8d8ade49d08419b196ddb9f691717ef05cae066e220db707e4dfedc4 monero-gui-mac-armv8-v0.18.3.1.dmg +8ae53f0908f9bc03452f23d5092bf1eb1d2ad9f1224580486b486cf0a2020401 monero-gui-mac-x64-v0.18.3.1.dmg +f263ce5863fd87ea959f79420e28ef0002649fa02bd57ae34efda926bdcf1a70 monero-gui-win-x64-v0.18.3.1.zip +045a84e343423a62ed617f200465b290267ff0a071375fdfc49ea02dcdb1a785 monero-gui-source-v0.18.3.1.tar.bz2 +# +# +# ~binaryFate +-----BEGIN PGP SIGNATURE----- + +iQIzBAEBCAAdFiEEgaxZH+nEtlxYBq/D8K9NRioL35IFAmUljRIACgkQ8K9NRioL +35IJjBAAqzoh4saUkTPWPAXOdLMLEcpJ5TvzQpImBNKzegjVMV4VYm+0llZw5uMP +/9OvkW7Ho5nLfslsF+C/qkfTc+EXm4h7J7iOyIpj8sr52RttfXjecEla5Ah75qZk +X9puVFd18nEDMktrNp4tkx/WQvzxpPAnsIRwsrX912rOc0jPwqCZ1DFn5JsB4KsK +dSjyWdRjKuxbMv+97GEpxiG6wAkN5lnEzj9LFZcaOLHAtBhxfZhwDBWTWFdPp2cM +TL3dNkMgpONGBEpX/7PJTFbmfba8gRZy7jXFVI0KqLLJC+6vpfGGr+NSX1zdIqrR +Z0Dvl3AA43E/Cjl5ma4L381wEul+7qFB2HN+fB1S6nNHzn/zWVepjD4bvgPvQiVI +d7PK5jhrX9c0XkR4kQrtPoONJW6blhoGiM2CWCfrifXzGA51WvZ1Vc5s8yuUG2p7 +e5+7c6AWFqOIP/8RexPx4ViYmFqE59P9/JCs+JRNgo7A2/JHGCyjdZalmt3/79Bf +aBmfv5mcPe/zPbngU9W6DfKbysYozv2/IQ5nUknU8Qgnaq3PADN2Xx5GlAsC69e0 +tZid955OAmtVzMjNO0KPiGEea2t/a8f3lSir2Irdz/LwIv8RID5/VeyafnUoOvGl +kv15IYnJAQ7vjlskoE/Tzaym/LSaILOHzU5CskI/HjG+7P50mo4= +=6gw4 +-----END PGP SIGNATURE----- diff --git a/orchestration/testnet/coins/monero/run.sh b/orchestration/testnet/coins/monero/run.sh new file mode 100755 index 000000000..baf287857 --- /dev/null +++ b/orchestration/testnet/coins/monero/run.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +RPC_USER="${RPC_USER:=serai}" +RPC_PASS="${RPC_PASS:=seraidex}" + +# Run Monero +monerod --non-interactive --stagenet \ + --no-zmq --rpc-bind-ip=0.0.0.0 --rpc-bind-port=18081 --confirm-external-bind \ + --rpc-access-control-origins "*" --disable-rpc-ban \ + --rpc-login=$RPC_USER:$RPC_PASS \ + --data-dir=/volume diff --git a/orchestration/testnet/coordinator/.folder b/orchestration/testnet/coordinator/.folder new file mode 100644 index 000000000..e69de29bb diff --git a/orchestration/testnet/message-queue/.folder b/orchestration/testnet/message-queue/.folder new file mode 100644 index 000000000..e69de29bb diff --git a/orchestration/testnet/processor/bitcoin/.folder b/orchestration/testnet/processor/bitcoin/.folder new file mode 100644 index 000000000..e69de29bb diff --git a/orchestration/testnet/processor/ethereum/.folder b/orchestration/testnet/processor/ethereum/.folder new file mode 100644 index 000000000..e69de29bb diff --git a/orchestration/testnet/processor/monero/.folder b/orchestration/testnet/processor/monero/.folder new file mode 100644 index 000000000..e69de29bb diff --git a/orchestration/testnet/serai/run.sh b/orchestration/testnet/serai/run.sh new file mode 100755 index 000000000..2bb8d868b --- /dev/null +++ b/orchestration/testnet/serai/run.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +exit 1 diff --git a/patches/zstd/Cargo.toml b/patches/zstd/Cargo.toml new file mode 100644 index 000000000..f7bf11d6b --- /dev/null +++ b/patches/zstd/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +description = "zstd which patches to the latest update" +license = "MIT" +repository = "https://github.com/serai-dex/serai/tree/develop/patches/zstd" +authors = ["Luke Parker "] +keywords = [] +edition = "2021" +rust-version = "1.70" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[dependencies] +zstd = "0.12" diff --git a/patches/zstd/src/lib.rs b/patches/zstd/src/lib.rs new file mode 100644 index 000000000..b698a9b25 --- /dev/null +++ b/patches/zstd/src/lib.rs @@ -0,0 +1 @@ +pub use zstd::*; diff --git a/processor/src/tests/literal/mod.rs b/processor/src/tests/literal/mod.rs index f7cd72fb7..2974d040f 100644 --- a/processor/src/tests/literal/mod.rs +++ b/processor/src/tests/literal/mod.rs @@ -25,16 +25,6 @@ mod bitcoin { let composition = TestBodySpecification::with_image( Image::with_repository("serai-dev-bitcoin").pull_policy(PullPolicy::Never), ) - .replace_cmd(vec![ - "bitcoind".to_string(), - "-txindex".to_string(), - "-regtest".to_string(), - format!("-rpcuser=serai"), - format!("-rpcpassword=seraidex"), - "-rpcbind=0.0.0.0".to_string(), - "-rpcallowip=0.0.0.0/0".to_string(), - "-rpcport=8332".to_string(), - ]) .set_start_policy(StartPolicy::Strict) .set_log_options(Some(LogOptions { action: LogAction::Forward, @@ -79,19 +69,6 @@ mod monero { let composition = TestBodySpecification::with_image( Image::with_repository("serai-dev-monero").pull_policy(PullPolicy::Never), ) - .replace_cmd(vec![ - "monerod".to_string(), - "--regtest".to_string(), - "--offline".to_string(), - "--fixed-difficulty=1".to_string(), - "--no-zmq".to_string(), - "--disable-rpc-ban".to_string(), - "--rpc-bind-ip=0.0.0.0".to_string(), - "--rpc-login=serai:seraidex".to_string(), - "--rpc-access-control-origins=*".to_string(), - "--confirm-external-bind".to_string(), - "--non-interactive".to_string(), - ]) .set_start_policy(StartPolicy::Strict) .set_log_options(Some(LogOptions { action: LogAction::Forward, diff --git a/substrate/client/tests/dht.rs b/substrate/client/tests/dht.rs index deed79031..2fd40b126 100644 --- a/substrate/client/tests/dht.rs +++ b/substrate/client/tests/dht.rs @@ -9,20 +9,12 @@ async fn dht() { serai_docker_tests::build("serai".to_string()); - let handle = |name| format!("serai_client-serai_node-{name}"); - let composition = |name| { + let handle = |name: &str| format!("serai_client-serai_node-{name}"); + let composition = |name: &str| { TestBodySpecification::with_image( Image::with_repository("serai-dev-serai").pull_policy(PullPolicy::Never), ) - .replace_cmd(vec![ - "serai-node".to_string(), - "--unsafe-rpc-external".to_string(), - "--rpc-cors".to_string(), - "all".to_string(), - "--chain".to_string(), - "local".to_string(), - format!("--{name}"), - ]) + .replace_env([("SERAI_NAME".to_string(), name.to_string())].into()) .set_publish_all_ports(true) .set_handle(handle(name)) .set_start_policy(StartPolicy::Strict) diff --git a/substrate/node/Cargo.toml b/substrate/node/Cargo.toml index c34f70445..1a0a21395 100644 --- a/substrate/node/Cargo.toml +++ b/substrate/node/Cargo.toml @@ -20,7 +20,14 @@ workspace = true name = "serai-node" [dependencies] +zeroize = "1" +rand_core = "0.6" +hex = "0.4" + +schnorrkel = "0.11" + sp-core = { git = "https://github.com/serai-dex/substrate" } +sp-keystore = { git = "https://github.com/serai-dex/substrate" } sp-timestamp = { git = "https://github.com/serai-dex/substrate" } sp-io = { git = "https://github.com/serai-dex/substrate" } sp-blockchain = { git = "https://github.com/serai-dex/substrate" } @@ -61,6 +68,8 @@ sc-rpc-api = { git = "https://github.com/serai-dex/substrate" } substrate-frame-rpc-system = { git = "https://github.com/serai-dex/substrate" } pallet-transaction-payment-rpc = { git = "https://github.com/serai-dex/substrate" } +serai-env = { path = "../../common/env" } + [build-dependencies] substrate-build-script-utils = { git = "https://github.com/serai-dex/substrate" } diff --git a/substrate/node/src/command.rs b/substrate/node/src/command.rs index 91df761b5..3588f95f0 100644 --- a/substrate/node/src/command.rs +++ b/substrate/node/src/command.rs @@ -58,23 +58,23 @@ pub fn run() -> sc_cli::Result<()> { Some(Subcommand::CheckBlock(cmd)) => cli.create_runner(cmd)?.async_run(|config| { let PartialComponents { client, task_manager, import_queue, .. } = - service::new_partial(&config)?; + service::new_partial(&config)?.0; Ok((cmd.run(client, import_queue), task_manager)) }), Some(Subcommand::ExportBlocks(cmd)) => cli.create_runner(cmd)?.async_run(|config| { - let PartialComponents { client, task_manager, .. } = service::new_partial(&config)?; + let PartialComponents { client, task_manager, .. } = service::new_partial(&config)?.0; Ok((cmd.run(client, config.database), task_manager)) }), Some(Subcommand::ExportState(cmd)) => cli.create_runner(cmd)?.async_run(|config| { - let PartialComponents { client, task_manager, .. } = service::new_partial(&config)?; + let PartialComponents { client, task_manager, .. } = service::new_partial(&config)?.0; Ok((cmd.run(client, config.chain_spec), task_manager)) }), Some(Subcommand::ImportBlocks(cmd)) => cli.create_runner(cmd)?.async_run(|config| { let PartialComponents { client, task_manager, import_queue, .. } = - service::new_partial(&config)?; + service::new_partial(&config)?.0; Ok((cmd.run(client, import_queue), task_manager)) }), @@ -83,7 +83,8 @@ pub fn run() -> sc_cli::Result<()> { } Some(Subcommand::Revert(cmd)) => cli.create_runner(cmd)?.async_run(|config| { - let PartialComponents { client, task_manager, backend, .. } = service::new_partial(&config)?; + let PartialComponents { client, task_manager, backend, .. } = + service::new_partial(&config)?.0; let aux_revert = Box::new(|client: Arc, backend, blocks| { sc_consensus_babe::revert(client.clone(), backend, blocks)?; sc_consensus_grandpa::revert(client, blocks)?; diff --git a/substrate/node/src/keystore.rs b/substrate/node/src/keystore.rs new file mode 100644 index 000000000..333cd9c3a --- /dev/null +++ b/substrate/node/src/keystore.rs @@ -0,0 +1,107 @@ +use zeroize::Zeroize; +use rand_core::RngCore; + +use sp_core::{crypto::*, ed25519, sr25519}; +use sp_keystore::*; + +pub struct Keystore(sr25519::Pair); + +impl Keystore { + pub fn from_env() -> Option { + let mut key_hex = serai_env::var("KEY")?; + let mut key = hex::decode(&key_hex).expect("KEY from environment wasn't hex"); + key_hex.zeroize(); + + assert_eq!(key.len(), 32, "KEY from environment wasn't 32 bytes"); + key.extend([0; 32]); + rand_core::OsRng.fill_bytes(&mut key[32 ..]); + + let res = Self(sr25519::Pair::from(schnorrkel::SecretKey::from_bytes(&key).unwrap())); + key.zeroize(); + Some(res) + } +} + +impl sp_keystore::Keystore for Keystore { + fn sr25519_public_keys(&self, _: KeyTypeId) -> Vec { + vec![self.0.public()] + } + + fn sr25519_generate_new(&self, _: KeyTypeId, _: Option<&str>) -> Result { + panic!("asked to generate an sr25519 key"); + } + + fn sr25519_sign( + &self, + _: KeyTypeId, + public: &sr25519::Public, + msg: &[u8], + ) -> Result, Error> { + if public == &self.0.public() { + Ok(Some(self.0.sign(msg))) + } else { + Ok(None) + } + } + + fn sr25519_vrf_sign( + &self, + _: KeyTypeId, + public: &sr25519::Public, + data: &sr25519::vrf::VrfSignData, + ) -> Result, Error> { + if public == &self.0.public() { + Ok(Some(self.0.vrf_sign(data))) + } else { + Ok(None) + } + } + + fn sr25519_vrf_output( + &self, + _: KeyTypeId, + public: &sr25519::Public, + input: &sr25519::vrf::VrfInput, + ) -> Result, Error> { + if public == &self.0.public() { + Ok(Some(self.0.vrf_output(input))) + } else { + Ok(None) + } + } + + fn ed25519_public_keys(&self, _: KeyTypeId) -> Vec { + panic!("asked for ed25519 keys"); + } + + fn ed25519_generate_new(&self, _: KeyTypeId, _: Option<&str>) -> Result { + panic!("asked to generate an ed25519 key"); + } + + fn ed25519_sign( + &self, + _: KeyTypeId, + _: &ed25519::Public, + _: &[u8], + ) -> Result, Error> { + panic!("asked to produce an ed25519 signature"); + } + + fn insert(&self, _: KeyTypeId, _: &str, _: &[u8]) -> Result<(), ()> { + panic!("asked to insert a key"); + } + + fn keys(&self, _: KeyTypeId) -> Result>, Error> { + Ok(vec![self.0.public().0.to_vec()]) + } + + fn has_keys(&self, public_keys: &[(Vec, KeyTypeId)]) -> bool { + let our_key = self.0.public().0; + for (public_key, _) in public_keys { + if our_key != public_key.as_slice() { + return false; + } + } + true + } +} diff --git a/substrate/node/src/main.rs b/substrate/node/src/main.rs index b44f83d80..5f33b51bf 100644 --- a/substrate/node/src/main.rs +++ b/substrate/node/src/main.rs @@ -1,3 +1,5 @@ +mod keystore; + mod chain_spec; mod service; diff --git a/substrate/node/src/service.rs b/substrate/node/src/service.rs index 5a124eeba..686e4c39a 100644 --- a/substrate/node/src/service.rs +++ b/substrate/node/src/service.rs @@ -58,7 +58,9 @@ fn create_inherent_data_providers( (BabeInherent::from_timestamp_and_slot_duration(*timestamp, slot_duration), timestamp) } -pub fn new_partial(config: &Configuration) -> Result { +pub fn new_partial( + config: &Configuration, +) -> Result<(PartialComponents, Arc), ServiceError> { let telemetry = config .telemetry_endpoints .clone() @@ -87,6 +89,13 @@ pub fn new_partial(config: &Configuration) -> Result = + if let Some(keystore) = crate::keystore::Keystore::from_env() { + Arc::new(keystore) + } else { + keystore_container.keystore() + }; + let telemetry = telemetry.map(|(worker, telemetry)| { task_manager.spawn_handle().spawn("telemetry", None, worker.run()); telemetry @@ -137,29 +146,35 @@ pub fn new_partial(config: &Configuration) -> Result Result { - let sc_service::PartialComponents { - client, - backend, - mut task_manager, - import_queue, + let ( + sc_service::PartialComponents { + client, + backend, + mut task_manager, + keystore_container: _, + import_queue, + select_chain, + transaction_pool, + other: (block_import, babe_link, grandpa_link, shared_voter_state, mut telemetry), + }, keystore_container, - select_chain, - transaction_pool, - other: (block_import, babe_link, grandpa_link, shared_voter_state, mut telemetry), - } = new_partial(&config)?; + ) = new_partial(&config)?; let mut net_config = sc_network::config::FullNetworkConfiguration::new(&config.network); let grandpa_protocol_name = @@ -195,7 +210,7 @@ pub fn new_full(config: Configuration) -> Result { sc_offchain::OffchainWorkers::new(sc_offchain::OffchainWorkerOptions { runtime_api_provider: client.clone(), is_validator: config.role.is_authority(), - keystore: Some(keystore_container.keystore()), + keystore: Some(keystore_container.clone()), offchain_db: backend.offchain_storage(), transaction_pool: Some(OffchainTransactionPoolFactory::new(transaction_pool.clone())), network_provider: network.clone(), @@ -207,7 +222,7 @@ pub fn new_full(config: Configuration) -> Result { } let role = config.role.clone(); - let keystore = keystore_container.keystore(); + let keystore = keystore_container; let prometheus_registry = config.prometheus_registry().cloned(); // TODO: Ensure we're considered as an authority is a validator of an external network diff --git a/tests/coordinator/src/lib.rs b/tests/coordinator/src/lib.rs index 82fad2f26..0f010456a 100644 --- a/tests/coordinator/src/lib.rs +++ b/tests/coordinator/src/lib.rs @@ -63,15 +63,7 @@ pub fn serai_composition(name: &str) -> TestBodySpecification { TestBodySpecification::with_image( Image::with_repository("serai-dev-serai").pull_policy(PullPolicy::Never), ) - .replace_cmd(vec![ - "serai-node".to_string(), - "--unsafe-rpc-external".to_string(), - "--rpc-cors".to_string(), - "all".to_string(), - "--chain".to_string(), - "local".to_string(), - format!("--{}", name.to_lowercase()), - ]) + .replace_env([("SERAI_NAME".to_string(), name.to_lowercase())].into()) .set_publish_all_ports(true) } diff --git a/tests/docker/src/lib.rs b/tests/docker/src/lib.rs index 2ae847e3b..6df010441 100644 --- a/tests/docker/src/lib.rs +++ b/tests/docker/src/lib.rs @@ -22,6 +22,7 @@ pub fn fresh_logs_folder(first: bool, label: &str) -> String { logs_path.to_str().unwrap().to_string() } +// TODO: Merge this with what's in serai-orchestrator/have serai-orchestrator perform building static BUILT: OnceLock>> = OnceLock::new(); pub fn build(name: String) { let built = BUILT.get_or_init(|| Mutex::new(HashMap::new())); @@ -42,8 +43,57 @@ pub fn build(name: String) { assert!(repo_path.as_path().ends_with("target")); repo_path.pop(); + // Run the orchestrator to ensure the most recent files exist + if !Command::new("cargo") + .current_dir(&repo_path) + .arg("run") + .arg("-p") + .arg("serai-orchestrator") + .arg("--") + .arg("key_gen") + .arg("dev") + .spawn() + .unwrap() + .wait() + .unwrap() + .success() + { + panic!("failed to run the orchestrator"); + } + + if !Command::new("cargo") + .current_dir(&repo_path) + .arg("run") + .arg("-p") + .arg("serai-orchestrator") + .arg("--") + .arg("setup") + .arg("dev") + .spawn() + .unwrap() + .wait() + .unwrap() + .success() + { + panic!("failed to run the orchestrator"); + } + let mut orchestration_path = repo_path.clone(); orchestration_path.push("orchestration"); + if name != "runtime" { + orchestration_path.push("dev"); + } + + let mut dockerfile_path = orchestration_path.clone(); + if HashSet::from(["bitcoin", "ethereum", "monero"]).contains(name.as_str()) { + dockerfile_path = dockerfile_path.join("coins"); + } + if name.contains("-processor") { + dockerfile_path = + dockerfile_path.join("processor").join(name.split('-').next().unwrap()).join("Dockerfile"); + } else { + dockerfile_path = dockerfile_path.join(&name).join("Dockerfile"); + } // If this Docker image was created after this repo was last edited, return here // This should have better performance than Docker and allows running while offline @@ -65,22 +115,9 @@ pub fn build(name: String) { .0, ); - let mut dockerfile_path = orchestration_path.clone(); - if HashSet::from(["bitcoin", "ethereum", "monero"]).contains(name.as_str()) { - dockerfile_path = dockerfile_path.join("coins"); - } - if name.contains("-processor") { - dockerfile_path = dockerfile_path - .join("processor") - .join(name.split('-').next().unwrap()) - .join("Dockerfile"); - } else { - dockerfile_path = dockerfile_path.join(&name).join("Dockerfile"); - } - // For all services, if the Dockerfile was edited after the image was built we should rebuild let mut last_modified = - fs::metadata(dockerfile_path).ok().and_then(|meta| meta.modified().ok()); + fs::metadata(&dockerfile_path).ok().and_then(|meta| meta.modified().ok()); // Check any additionally specified paths let meta = |path: PathBuf| (path.clone(), fs::metadata(path)); @@ -153,10 +190,13 @@ pub fn build(name: String) { // Version which always prints if !Command::new("docker") - .current_dir(orchestration_path) - .arg("compose") + .current_dir(&repo_path) .arg("build") - .arg(&name) + .arg("-f") + .arg(dockerfile_path) + .arg(".") + .arg("-t") + .arg(format!("serai-dev-{name}")) .spawn() .unwrap() .wait() @@ -169,10 +209,11 @@ pub fn build(name: String) { // Version which only prints on error /* let res = Command::new("docker") - .current_dir(orchestration_path) - .arg("compose") + .current_dir(dockerfile_path) .arg("build") - .arg(&name) + .arg(".") + .arg("-t") + .arg(format!("serai-dev-{name}")) .output() .unwrap(); if !res.status.success() { diff --git a/tests/processor/src/networks.rs b/tests/processor/src/networks.rs index dc234476b..fc75db4a7 100644 --- a/tests/processor/src/networks.rs +++ b/tests/processor/src/networks.rs @@ -27,16 +27,6 @@ pub fn bitcoin_instance() -> (TestBodySpecification, u32) { let composition = TestBodySpecification::with_image( Image::with_repository("serai-dev-bitcoin").pull_policy(PullPolicy::Never), ) - .replace_cmd(vec![ - "bitcoind".to_string(), - "-txindex".to_string(), - "-regtest".to_string(), - format!("-rpcuser={RPC_USER}"), - format!("-rpcpassword={RPC_PASS}"), - "-rpcbind=0.0.0.0".to_string(), - "-rpcallowip=0.0.0.0/0".to_string(), - "-rpcport=8332".to_string(), - ]) .set_publish_all_ports(true); (composition, BTC_PORT) } @@ -47,17 +37,6 @@ pub fn monero_instance() -> (TestBodySpecification, u32) { let composition = TestBodySpecification::with_image( Image::with_repository("serai-dev-monero").pull_policy(PullPolicy::Never), ) - .replace_cmd(vec![ - "monerod".to_string(), - "--regtest".to_string(), - "--offline".to_string(), - "--fixed-difficulty=1".to_string(), - "--rpc-bind-ip=0.0.0.0".to_string(), - format!("--rpc-login={RPC_USER}:{RPC_PASS}"), - "--rpc-access-control-origins=*".to_string(), - "--confirm-external-bind".to_string(), - "--non-interactive".to_string(), - ]) .set_start_policy(StartPolicy::Strict) .set_publish_all_ports(true); (composition, XMR_PORT)