From c99773a5f694cbd59066e17904b9cd8cf76a79c1 Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Fri, 4 Aug 2023 22:00:48 +0200 Subject: [PATCH 01/10] Revert "Revert "feat: ic binary dependencies are included in tarball"" --- .gitignore | 6 + Cargo.lock | 276 +++++++++--------- Cargo.toml | 1 + extensions-utils/Cargo.toml | 4 + extensions-utils/src/dfx.rs | 59 +++- extensions-utils/src/download_dependencies.rs | 44 +++ extensions-utils/src/lib.rs | 6 +- extensions-utils/src/project/import.rs | 2 - extensions/nns/Cargo.toml | 7 +- extensions/nns/build.rs | 18 ++ extensions/nns/e2e/tests/nns.bash | 14 +- extensions/nns/src/commands/install.rs | 3 - extensions/nns/src/install_nns.rs | 211 +++++-------- extensions/sns/Cargo.toml | 7 +- extensions/sns/build.rs | 16 + extensions/sns/e2e/tests/sns.bash | 5 + extensions/sns/src/commands/config/create.rs | 6 +- extensions/sns/src/commands/config/mod.rs | 8 +- .../sns/src/commands/config/validate.rs | 6 +- extensions/sns/src/commands/deploy.rs | 6 +- extensions/sns/src/create_config.rs | 6 +- extensions/sns/src/deploy.rs | 6 +- extensions/sns/src/main.rs | 4 +- extensions/sns/src/validate_config.rs | 8 +- 24 files changed, 414 insertions(+), 315 deletions(-) create mode 100644 extensions-utils/src/download_dependencies.rs create mode 100644 extensions/nns/build.rs create mode 100644 extensions/sns/build.rs diff --git a/.gitignore b/.gitignore index c537d56..7377465 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,9 @@ target/ ## MSVC Windows builds of rustc generate these, which store debugging information *.pdb + +## don't include binaries that are downloaded in build.rs scripts +extensions/sns/sns-cli +extensions/nns/ic-admin +extensions/nns/ic-nns-init +extensions/nns/sns-cli diff --git a/Cargo.lock b/Cargo.lock index e9a4c26..0b24574 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -174,9 +174,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" [[package]] name = "arbitrary" @@ -247,13 +247,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.71" +version = "0.1.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf" +checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -572,9 +572,9 @@ checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "candid" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4df671c37a9c6168db0334f2b289dd4e02dea1bbefe1fb22c5d43b12d865aacd" +checksum = "73f6dd83439943085a6d03503f55c3c9329052a2ad815d24a0c1a0ce6147690e" dependencies = [ "anyhow", "arbitrary", @@ -613,14 +613,17 @@ dependencies = [ "lazy_static", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] name = "cc" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "6c6b2562119bf28c3439f7f02db99faf0aa1a8cdfe5772a2ee155d32227239f0" +dependencies = [ + "libc", +] [[package]] name = "cfg-if" @@ -639,9 +642,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.11" +version = "4.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1640e5cc7fb47dbb8338fd471b105e7ed6c3cb2aeb00c2e067127ffd3764a05d" +checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d" dependencies = [ "clap_builder", "clap_derive", @@ -650,9 +653,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.11" +version = "4.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c59138d527eeaf9b53f35a77fcc1fad9d883116070c63d5de1c7dc7b00c72b" +checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1" dependencies = [ "anstream", "anstyle", @@ -662,14 +665,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.3.2" +version = "4.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" +checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -871,6 +874,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8810e7e2cf385b1e9b50d68264908ec367ba642c96d02edfe61c39e88e2a3c01" + [[package]] name = "derivative" version = "2.2.0" @@ -1053,9 +1062,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "dyn-clone" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30" +checksum = "304e6508efa593091e97a9abbc10f90aa7ca635b6d2784feff3c89d41dd12272" [[package]] name = "ecdsa" @@ -1071,9 +1080,9 @@ dependencies = [ [[package]] name = "ecdsa" -version = "0.16.7" +version = "0.16.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0997c976637b606099b9985693efa3581e84e41f5c11ba5255f88711058ad428" +checksum = "a4b1e0c257a9e9f25f90ff76d7a68360ed497ee519c8e428d1825ef0000799d4" dependencies = [ "der 0.7.7", "digest 0.10.7", @@ -1085,9 +1094,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "elliptic-curve" @@ -1125,7 +1134,7 @@ dependencies = [ "pem-rfc7468 0.7.0", "pkcs8 0.10.2", "rand_core", - "sec1 0.7.2", + "sec1 0.7.3", "subtle", "zeroize", ] @@ -1192,9 +1201,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", @@ -1236,6 +1245,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + [[package]] name = "ff" version = "0.12.1" @@ -1293,7 +1308,7 @@ checksum = "2cd66269887534af4b0c3e3337404591daa8dc8b9b2b3db71f9523beb4bafb41" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -1386,7 +1401,7 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" dependencies = [ - "fastrand", + "fastrand 1.9.0", "futures-core", "futures-io", "memchr", @@ -1403,7 +1418,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -1712,7 +1727,7 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls 0.21.5", + "rustls 0.21.6", "tokio", "tokio-rustls 0.24.1", ] @@ -1751,7 +1766,7 @@ dependencies = [ "reqwest", "ring", "rustls 0.20.8", - "sec1 0.7.2", + "sec1 0.7.3", "serde", "serde_bytes", "serde_cbor", @@ -1882,7 +1897,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi 0.3.2", - "rustix 0.38.4", + "rustix 0.38.6", "windows-sys 0.48.0", ] @@ -1906,9 +1921,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b02a5381cc465bd3041d84623d0fa3b66738b52b8e2fc3bab8ad63ab032f4a" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "js-sys" @@ -1939,7 +1954,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cadb76004ed8e97623117f3df85b17aaa6626ab0b0831e6573f104df16cd1bcc" dependencies = [ "cfg-if", - "ecdsa 0.16.7", + "ecdsa 0.16.8", "elliptic-curve 0.13.5", "once_cell", "sha2 0.10.7", @@ -2029,9 +2044,9 @@ dependencies = [ [[package]] name = "libz-ng-sys" -version = "1.1.9" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2468756f34903b582fe7154dc1ffdebd89d0562c4a43b53c621bb0f1b1043ccb" +checksum = "3dd9f43e75536a46ee0f92b758f6b63846e594e86638c61a9251338a65baea63" dependencies = [ "cmake", "libc", @@ -2045,9 +2060,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" +checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" [[package]] name = "lock_api" @@ -2085,7 +2100,7 @@ dependencies = [ "proc-macro2", "quote", "regex-syntax 0.6.29", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -2307,9 +2322,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", ] @@ -2342,7 +2357,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -2398,7 +2413,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -2409,9 +2424,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "111.26.0+1.1.1u" +version = "111.27.0+1.1.1v" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efc62c9f12b22b8f5208c23a7200a442b2e5999f8bdf80233852122b5a4f6f37" +checksum = "06e8f197c82d7511c5b014030c9b1efeda40d7d5f99d23b4ceed3524a5e63f02" dependencies = [ "cc", ] @@ -2480,9 +2495,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b27ab7be369122c218afc2079489cdcb4b517c0a3fc386ff11e1fedfcc2b35" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "pbkdf2" @@ -2529,9 +2544,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" -version = "2.7.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f73935e4d55e2abf7f130186537b19e7a4abc886a0252380b59248af473a3fc9" +checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a" dependencies = [ "thiserror", "ucd-trie", @@ -2561,9 +2576,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aef623c9bbfa0eedf5a0efba11a5ee83209c326653ca31ff019bec3a95bfff2b" +checksum = "666d00490d4ac815001da55838c500eafb0320019bbaa44444137c48b443a853" dependencies = [ "pest", "pest_generator", @@ -2571,22 +2586,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e8cba4ec22bada7fc55ffe51e2deb6a0e0db2d0b7ab0b103acc80d2510c190" +checksum = "68ca01446f50dbda87c1786af8770d535423fa8a53aec03b8f4e3d7eb10e0929" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] name = "pest_meta" -version = "2.7.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a01f71cb40bd8bb94232df14b946909e14660e33fc05db3e50ae2a82d7ea0ca0" +checksum = "56af0a30af74d0445c0bf6d9d051c979b516a1a5af790d251daee76005420a48" dependencies = [ "once_cell", "pest", @@ -2750,9 +2765,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.64" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -2788,9 +2803,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.29" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2", ] @@ -2874,9 +2889,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83d3daa6976cffb758ec878f108ba0e062a45b2d6ca3a2cca965338855476caf" +checksum = "b7b6d6190b7594385f61bd3911cd1be99dfddcfc365a4160cc2ab5bff4aed294" dependencies = [ "aho-corasick", "memchr", @@ -2929,7 +2944,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.5", + "rustls 0.21.6", "rustls-pemfile", "serde", "serde_json", @@ -3023,13 +3038,12 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.30.0" +version = "1.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0446843641c69436765a35a5a77088e28c2e6a12da93e84aa3ab1cd4aa5a042" +checksum = "4a2ab0025103a60ecaaf3abf24db1db240a4e1c15837090d2c32f625ac98abea" dependencies = [ "arrayvec 0.7.4", "borsh", - "bytecheck", "byteorder", "bytes", "num-traits", @@ -3067,14 +3081,14 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.4" +version = "0.38.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +checksum = "1ee020b1716f0a80e2ace9b03441a749e402e86712f15f16fe8a8f75afac732f" dependencies = [ "bitflags 2.3.3", "errno", "libc", - "linux-raw-sys 0.4.3", + "linux-raw-sys 0.4.5", "windows-sys 0.48.0", ] @@ -3092,9 +3106,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.5" +version = "0.21.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36" +checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" dependencies = [ "log", "ring", @@ -3125,9 +3139,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.1" +version = "0.101.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e" +checksum = "513722fd73ad80a71f72b61009ea1b584bcfa1483ca93949c8f290298837fa59" dependencies = [ "ring", "untrusted", @@ -3135,15 +3149,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc31bd9b61a32c31f9650d18add92aa83a49ba979c143eefd27fe7177b05bd5f" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe232bdf6be8c8de797b22184ee71118d63780ea42ac85b61d1baa6d3b782ae9" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "schannel" @@ -3186,9 +3200,9 @@ checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" @@ -3222,9 +3236,9 @@ dependencies = [ [[package]] name = "sec1" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0aec48e813d6b90b15f0b8948af3c63483992dee44c03e9930b3eebdabe046e" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct 0.2.0", "der 0.7.7", @@ -3256,9 +3270,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.1" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" dependencies = [ "bitflags 1.3.2", "core-foundation", @@ -3269,9 +3283,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" dependencies = [ "core-foundation-sys", "libc", @@ -3279,24 +3293,24 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "serde" -version = "1.0.171" +version = "1.0.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" +checksum = "0ea67f183f058fe88a4e3ec6e2788e003840893b91bac4559cabedd00863b3ed" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.11" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a16be4fe5320ade08736447e3198294a5ea9a6d44dde6f35f0a5e06859c427a" +checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" dependencies = [ "serde", ] @@ -3313,13 +3327,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.171" +version = "1.0.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" +checksum = "24e744d7782b686ab3b73267ef05697159cc0e5abbed3f47f9933165e5219036" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -3348,9 +3362,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.102" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5062a995d481b2308b6064e9af76011f2921c35f97b0468811ed9f6cd91dfed" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" dependencies = [ "itoa", "ryu", @@ -3359,13 +3373,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.14" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d89a8107374290037607734c0b73a85db7ed80cae314b3c5791f192a496e731" +checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -3642,9 +3656,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.25" +version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ "proc-macro2", "quote", @@ -3665,9 +3679,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +checksum = "ec96d2ffad078296368d46ff1cb309be1c23c513b4ab0e22a45de0185275ac96" dependencies = [ "filetime", "libc", @@ -3676,15 +3690,14 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.6.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" dependencies = [ - "autocfg", "cfg-if", - "fastrand", + "fastrand 2.0.0", "redox_syscall 0.3.5", - "rustix 0.37.23", + "rustix 0.38.6", "windows-sys 0.48.0", ] @@ -3710,22 +3723,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.43" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.43" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] @@ -3740,10 +3753,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.23" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59e399c068f43a5d116fedaf73b203fa4f9c519f17e2b34f63221d3792f81446" +checksum = "b79eabcd964882a646b3584543ccabeae7869e9ac32a46f6f22b7a5bd405308b" dependencies = [ + "deranged", "itoa", "libc", "num_threads", @@ -3760,9 +3774,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96ba15a897f3c86766b757e5ac7221554c6750054d74d5b28844fce5fb36a6c4" +checksum = "eb71511c991639bb078fd5bf97757e03914361c48100d52878b8e52b46fb92cd" dependencies = [ "time-core", ] @@ -3854,7 +3868,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.5", + "rustls 0.21.6", "tokio", ] @@ -3889,9 +3903,9 @@ checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" [[package]] name = "toml_edit" -version = "0.19.12" +version = "0.19.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c500344a19072298cd05a7224b3c0c629348b78692bf48466c5238656e315a78" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" dependencies = [ "indexmap 2.0.0", "toml_datetime", @@ -3956,9 +3970,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" @@ -4034,9 +4048,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d023da39d1fde5a8a3fe1f3e01ca9632ada0a63e9797de55a879d6e2236277be" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" [[package]] name = "vcpkg" @@ -4092,7 +4106,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", "wasm-bindgen-shared", ] @@ -4126,7 +4140,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4344,9 +4358,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.9" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81a2094c43cc94775293eaa0e499fbc30048a6d824ac82c0351a8c0bf9112529" +checksum = "f46aab759304e4d7b2075a9aecba26228bb073ee8c50db796b2c72c676b5d807" dependencies = [ "memchr", ] @@ -4388,7 +4402,7 @@ dependencies = [ "byteorder", "derivative", "enumflags2", - "fastrand", + "fastrand 1.9.0", "futures", "nb-connect", "nix", @@ -4430,7 +4444,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.28", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index bac0662..c89b820 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,6 +16,7 @@ dfx-extensions-utils.path = "./extensions-utils" anyhow = "1.0.70" clap = { version = "4.2.1", features = ["derive", "env"] } +flate2 = { version = "1.0.25", default-features = false, features = ["zlib-ng"] } fn-error-context = "0.2.1" futures-util = "0.3.28" ic-agent = "0.25.0" diff --git a/extensions-utils/Cargo.toml b/extensions-utils/Cargo.toml index 6f9bd81..aef537c 100644 --- a/extensions-utils/Cargo.toml +++ b/extensions-utils/Cargo.toml @@ -9,6 +9,10 @@ publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[build-dependencies] +flate2.workspace = true +reqwest.workspace = true + [dependencies] dfx-core.workspace = true diff --git a/extensions-utils/src/dfx.rs b/extensions-utils/src/dfx.rs index ba37aed..f5a509c 100644 --- a/extensions-utils/src/dfx.rs +++ b/extensions-utils/src/dfx.rs @@ -7,13 +7,17 @@ use std::ffi::OsStr; use std::path::Path; use std::process::{self, Command}; -/// Calls a bundled command line tool. +/// Calls a binary from dfx cache. /// /// # Returns /// - On success, returns stdout as a string. /// - On error, returns an error message including stdout and stderr. #[context("Calling {} CLI, or, it returned an error.", command)] -pub fn call_bundled(dfx_cache_path: &Path, command: &str, args: I) -> anyhow::Result +pub fn call_dfx_bundled_binary( + dfx_cache_path: &Path, + command: &str, + args: I, +) -> anyhow::Result where I: IntoIterator, S: AsRef, @@ -50,6 +54,57 @@ where }) } +/// Calls a binary that was delivered with an extension tarball. +/// +/// # Returns +/// - On success, returns stdout as a string. +/// - On error, returns an error message including stdout and stderr. +#[context("Calling {} CLI failed, or, it returned an error.", binary_name)] +pub fn call_extension_bundled_binary(binary_name: &str, args: I) -> anyhow::Result +where + I: IntoIterator, + S: AsRef, +{ + let extension_binary_path = + std::env::current_exe().map_err(|e| anyhow::anyhow!("Failed to get current exe: {}", e))?; + let extension_dir_path = extension_binary_path + .parent() + .unwrap_or_else(|| Path::new(".")); + let binary_to_call = extension_dir_path.join(binary_name); + + let mut command = Command::new(&binary_to_call); + command.args(args); + // The sns command line tool itself calls dfx; it should call this dfx. + // The sns command line tool should not rely on commands not packaged with dfx. + // The same applies to other bundled binaries. + command.env( + "PATH", + binary_to_call.parent().unwrap_or_else(|| Path::new(".")), + ); + command + .stdin(process::Stdio::null()) + .output() + .map_err(anyhow::Error::from) + .and_then(|output| { + if output.status.success() { + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + } else { + let args: Vec<_> = command + .get_args() + .into_iter() + .map(OsStr::to_string_lossy) + .collect(); + Err(anyhow!( + "Call failed:\n{:?} {}\nStdout:\n{}\n\nStderr:\n{}", + command.get_program(), + args.join(" "), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )) + } + }) +} + pub fn replica_rev(dfx_cache_path: &Path) -> Result { let args = ["info", "replica-rev"]; let rev = Command::new(dfx_cache_path.join("dfx")) diff --git a/extensions-utils/src/download_dependencies.rs b/extensions-utils/src/download_dependencies.rs new file mode 100644 index 0000000..deb8ceb --- /dev/null +++ b/extensions-utils/src/download_dependencies.rs @@ -0,0 +1,44 @@ +use flate2::read::GzDecoder; +use std::fs::File; +use std::path::Path; +use std::{env, fs, io::copy}; + +pub fn download_ic_binary(replica_rev: &str, binary_name: &str, renamed_binary_name: &str) { + let arch = match std::env::consts::ARCH { + "x86_64" => "x86_64", + "aarch64" => "x86_64", // let's rely on rosetta2 for now, since sns-cli is not available for arm64 + _ => panic!("Unsupported architecture"), + }; + let os = match std::env::consts::OS { + "macos" => "darwin", + "linux" => "linux", + // "windows" => "windows", // unsupported till dfx supports windows + _ => panic!("Unsupported OS"), + }; + + let url = format!( + "https://download.dfinity.systems/ic/{replica_rev}/openssl-static-binaries/{arch}-{os}/{binary_name}.gz", + arch = arch, + os = os, + binary_name = binary_name, + ); + println!("Downloading {}", url); + + let resp = reqwest::blocking::get(&url).expect("Request failed"); + let bytes = resp.bytes().expect("Failed to read response"); + + let mut d = GzDecoder::new(&*bytes); + let mut dest = File::create(binary_name).expect("Failed to create sns file"); + + copy(&mut d, &mut dest).expect("Failed to copy content"); + + let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); + let out_path = Path::new(&manifest_dir).join(renamed_binary_name); + fs::rename(binary_name, &out_path).expect("Failed to move extension"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + dfx_core::fs::set_permissions(&out_path, std::fs::Permissions::from_mode(0o777)) + .expect("Failed to set permissions"); + } +} diff --git a/extensions-utils/src/lib.rs b/extensions-utils/src/lib.rs index 7459aab..66dd22e 100644 --- a/extensions-utils/src/lib.rs +++ b/extensions-utils/src/lib.rs @@ -1,12 +1,16 @@ //! Library for calling bundled command line tools. mod dfx; +pub mod download_dependencies; mod download_wasms; mod error; mod logger; mod project; -pub use dfx::{call_bundled, dfx_version, replica_rev, webserver_port}; +pub use dfx::{ + call_dfx_bundled_binary, call_extension_bundled_binary, dfx_version, replica_rev, + webserver_port, +}; pub use download_wasms::download_ic_repo_wasm; pub use download_wasms::nns::download_nns_wasms; pub use download_wasms::nns::{ diff --git a/extensions-utils/src/project/import.rs b/extensions-utils/src/project/import.rs index 32dca31..8eea8dd 100644 --- a/extensions-utils/src/project/import.rs +++ b/extensions-utils/src/project/import.rs @@ -2,8 +2,6 @@ use crate::project::error::ProjectError; use dfx_core::config::model::canister_id_store; use dfx_core::config::model::canister_id_store::CanisterIds; use dfx_core::config::model::dfinity::Config; - -use hyper_rustls::ConfigBuilderExt; use reqwest::{Client, StatusCode}; use serde::Deserialize; use serde_json::{Map, Value}; diff --git a/extensions/nns/Cargo.toml b/extensions/nns/Cargo.toml index a07ea32..93d6490 100644 --- a/extensions/nns/Cargo.toml +++ b/extensions/nns/Cargo.toml @@ -8,6 +8,10 @@ repository.workspace = true # Temp hack until https://github.com/axodotdev/cargo-dist/issues/187 is resovled. # Nothing will actually be published (unless `cargo publish` is executed (and it shouldn't)). publish = true +build = "build.rs" + +[build-dependencies] +dfx-extensions-utils.workspace = true [dependencies] dfx-core.workspace = true @@ -46,5 +50,4 @@ pre-release-replacements = [ ] [package.metadata.dist] -include = ["extension.json"] - +include = ["extension.json", "sns-cli", "ic-admin", "ic-nns-init"] diff --git a/extensions/nns/build.rs b/extensions/nns/build.rs new file mode 100644 index 0000000..f9cff35 --- /dev/null +++ b/extensions/nns/build.rs @@ -0,0 +1,18 @@ +const REPLICA_REV: &str = "90e2799c255733409d0e61682685afcc2431c928"; + +const BINARY_DEPENDENCIES: &[(&str, &str)] = &[ + // (downloaded binary name, renamed binary name) + ("ic-nns-init", "ic-nns-init"), + ("ic-admin", "ic-admin"), + ("sns", "sns-cli"), +]; + +fn main() { + for (binary_name, renamed_binary_name) in BINARY_DEPENDENCIES { + dfx_extensions_utils::download_dependencies::download_ic_binary( + REPLICA_REV, + binary_name, + renamed_binary_name, + ); + } +} diff --git a/extensions/nns/e2e/tests/nns.bash b/extensions/nns/e2e/tests/nns.bash index 428b4ce..b9f3dfa 100755 --- a/extensions/nns/e2e/tests/nns.bash +++ b/extensions/nns/e2e/tests/nns.bash @@ -22,10 +22,8 @@ teardown() { } @test "ic-nns-init binary exists and is executable" { - dfx cache install - # it panics, but still shows help - run "$(dfx cache show)/ic-nns-init" --help + run "$(dfx cache show)/extensions/nns/ic-nns-init" --help assert_failure assert_output --partial "thread 'main' panicked at 'Illegal arguments:" assert_output --partial "ic-nns-init [OPTIONS]" @@ -33,22 +31,18 @@ teardown() { assert_output --regexp '--version.*Print version information' # --version fails too - run "$(dfx cache show)/ic-nns-init" --version + run "$(dfx cache show)/extensions/nns/ic-nns-init" --version assert_failure } @test "ic-admin binary exists and is executable" { - dfx cache install - - run "$(dfx cache show)/ic-admin" --help + run "$(dfx cache show)/extensions/nns/ic-admin" --help assert_success assert_output --partial 'Common command-line options for `ic-admin`' } @test "sns binary exists and is executable" { - dfx cache install - - run "$(dfx cache show)/sns" --help + run "$(dfx cache show)/extensions/nns/sns-cli" --help assert_failure assert_output --partial "Initialize, deploy and interact with an SNS." } diff --git a/extensions/nns/src/commands/install.rs b/extensions/nns/src/commands/install.rs index 789bb35..fcded86 100644 --- a/extensions/nns/src/commands/install.rs +++ b/extensions/nns/src/commands/install.rs @@ -69,14 +69,11 @@ pub async fn exec(opts: InstallOpts, dfx_cache_path: &Path) -> anyhow::Result<() fetch_root_key_when_local(&agent, &network_descriptor).await?; - let ic_nns_init_path = dfx_cache_path.join("ic-nns-init"); - install_nns( &agent, &network_descriptor, &networks_config, &dfx_cache_path, - &ic_nns_init_path, &opts.ledger_accounts, &logger, ) diff --git a/extensions/nns/src/install_nns.rs b/extensions/nns/src/install_nns.rs index 1ec6c36..dc9078f 100644 --- a/extensions/nns/src/install_nns.rs +++ b/extensions/nns/src/install_nns.rs @@ -11,9 +11,9 @@ use dfx_core::config::model::dfinity::{NetworksConfig, ReplicaSubnetType}; use dfx_core::config::model::network_descriptor::NetworkDescriptor; use dfx_core::identity::CallSender; use dfx_extensions_utils::{ - download_nns_wasms, nns_wasm_dir, IcNnsInitCanister, SnsCanisterInstallation, StandardCanister, - ED25519_TEST_ACCOUNT, NNS_CORE, NNS_FRONTEND, NNS_SNS_WASM, SECP256K1_TEST_ACCOUNT, - SNS_CANISTERS, + call_extension_bundled_binary, download_nns_wasms, nns_wasm_dir, IcNnsInitCanister, + SnsCanisterInstallation, StandardCanister, ED25519_TEST_ACCOUNT, NNS_CORE, NNS_FRONTEND, + NNS_SNS_WASM, SECP256K1_TEST_ACCOUNT, SNS_CANISTERS, }; use anyhow::{anyhow, bail, Context}; @@ -28,12 +28,10 @@ use ic_utils::interfaces::management_canister::builders::InstallMode; use ic_utils::interfaces::ManagementCanister; use reqwest::Url; use slog::Logger; -use std::ffi::OsStr; use std::fs; use std::io::Write; use std::path::Component; use std::path::{Path, PathBuf}; -use std::process::{self, Command}; /// Installs NNS canisters on a local dfx server. /// # Notes: @@ -57,7 +55,6 @@ pub async fn install_nns( network: &NetworkDescriptor, networks_config: &NetworksConfig, dfx_cache_path: &Path, - ic_nns_init_path: &Path, ledger_accounts: &[String], logger: &Logger, ) -> anyhow::Result<()> { @@ -67,7 +64,6 @@ pub async fn install_nns( let provider_url = get_and_check_provider(network)?; let nns_url = get_and_check_replica_url(network, logger)?; let subnet_id = get_subnet_id(agent).await?.to_text(); - let ic_admin_cli = bundled_binary(dfx_cache_path, "ic-admin")?; eprintln!("Installing the core backend wasm canisters..."); download_nns_wasms(dfx_cache_path).await?; @@ -82,7 +78,7 @@ pub async fn install_nns( test_accounts, sns_subnets: Some(subnet_id.to_string()), }; - ic_nns_init(ic_nns_init_path, &ic_nns_init_opts).await?; + ic_nns_init(&ic_nns_init_opts).await?; eprintln!("Uploading NNS configuration data..."); upload_nns_sns_wasms_canister_wasms(dfx_cache_path)?; @@ -116,8 +112,8 @@ pub async fn install_nns( } // ... and configure the backend NNS canisters: eprintln!("Configuring the NNS..."); - set_xdr_rate(1234567, &nns_url, &ic_admin_cli)?; - set_cmc_authorized_subnets(&nns_url, &subnet_id, &ic_admin_cli)?; + set_xdr_rate(1234567, &nns_url)?; + set_cmc_authorized_subnets(&nns_url, &subnet_id)?; print_nns_details(provider_url)?; Ok(()) @@ -416,38 +412,24 @@ pub struct IcNnsInitOpts { /// - This won't work with an HSM, because the agent holds a session open /// - The provider_url is what the agent connects to, and forwards to the replica. #[context("Failed to install NNS components.")] -pub async fn ic_nns_init(ic_nns_init_path: &Path, opts: &IcNnsInitOpts) -> anyhow::Result<()> { - let mut cmd = std::process::Command::new(ic_nns_init_path); - cmd.arg("--pass-specified-id"); - cmd.arg("--url"); - cmd.arg(&opts.nns_url); - cmd.arg("--wasm-dir"); - cmd.arg(&opts.wasm_dir); - opts.test_accounts.iter().for_each(|account| { - cmd.arg("--initialize-ledger-with-test-accounts"); - cmd.arg(account); - }); - opts.sns_subnets.iter().for_each(|subnet| { - cmd.arg("--sns-subnet"); - cmd.arg(subnet); - }); - let args: Vec<_> = cmd - .get_args() - .into_iter() - .map(OsStr::to_string_lossy) - .collect(); - dbg!(&cmd); - println!("ic-nns-init {}", args.join(" ")); - cmd.stdout(std::process::Stdio::inherit()); - cmd.stderr(std::process::Stdio::inherit()); - let output = cmd - .output() - .with_context(|| format!("Error executing {:#?}", cmd))?; - - if !output.status.success() { - return Err(anyhow!("ic-nns-init call failed")); +pub async fn ic_nns_init(opts: &IcNnsInitOpts) -> anyhow::Result { + let mut args = vec![ + "--pass-specified-id", + "--url", + &opts.nns_url, + "--wasm-dir", + opts.wasm_dir.to_str().unwrap(), + ]; + for account in &opts.test_accounts { + args.push("--initialize-ledger-with-test-accounts"); + args.push(account); } - Ok(()) + if let Some(subnets) = &opts.sns_subnets { + args.push("--sns-subnet"); + args.push(subnets); + } + call_extension_bundled_binary("ic-nns-init", &args) + .with_context(|| format!("Error executing `ic-admin` with args: {:?}", args)) } /// Sets the exchange rate between ICP and cycles. @@ -456,59 +438,45 @@ pub async fn ic_nns_init(ic_nns_init_path: &Path, opts: &IcNnsInitOpts) -> anyho /// This is done by proposal. Just after startung a test server, ic-admin /// proposals with a test user pass immediately, as the small test neuron is /// the only neuron and has absolute majority. -#[context("Failed to set an initial exchange rate between ICP and cycles. It may not be possible to create canisters or purchase cycles.")] -pub fn set_xdr_rate(rate: u64, nns_url: &Url, ic_admin: &PathBuf) -> anyhow::Result<()> { - std::process::Command::new(ic_admin) - .arg("--nns-url") - .arg(nns_url.as_str()) - .arg("propose-xdr-icp-conversion-rate") - .arg("--test-neuron-proposer") - .arg("--summary") - .arg(format!("Set the cycle exchange rate to {rate}.")) - .arg("--xdr-permyriad-per-icp") - .arg(format!("{}", rate)) - .stdin(process::Stdio::null()) - .output() - .map_err(anyhow::Error::from) - .and_then(|output| { - if output.status.success() { - Ok(()) - } else { - Err(anyhow!("Call to propose to set xdr rate failed")) - } - }) +#[context("Failed to set an initial exchange rate between ICP and cycles. It may not be possible to create canisters or purchase cycles.")] +pub fn set_xdr_rate(rate: u64, nns_url: &Url) -> anyhow::Result { + let summary = format!("Set the cycle exchange rate to {}.", rate.clone()); + let xdr_permyriad_per_icp = rate.to_string(); + let args = vec![ + "--nns-url", + nns_url.as_str(), + "propose-xdr-icp-conversion-rate", + "--test-neuron-proposer", + "--summary", + &summary, + "--xdr-permyriad-per-icp", + &xdr_permyriad_per_icp, + ]; + call_extension_bundled_binary("ic-admin", args) + .map_err(|e| anyhow!("Call to propose to set xdr rate failed: {e}")) } /// Sets the subnets the CMC is authorized to create canisters in. -#[context("Failed to authorize a subnet for use by the cycles management canister. The CMC may not be able to create canisters.")] -pub fn set_cmc_authorized_subnets( - nns_url: &Url, - subnet: &str, - ic_admin: &PathBuf, -) -> anyhow::Result<()> { - std::process::Command::new(ic_admin) - .arg("--nns-url") - .arg(nns_url.as_str()) - .arg("propose-to-set-authorized-subnetworks") - .arg("--test-neuron-proposer") - .arg("--proposal-title") - .arg("Set Cycles Minting Canister Authorized Subnets") - .arg("--summary") - .arg(format!( - "Authorize the Cycles Minting Canister to create canisters in the subnet '{subnet}'." - )) - .arg("--subnets") - .arg(subnet) - .stdin(process::Stdio::null()) - .output() - .map_err(anyhow::Error::from) - .and_then(|output| { - if output.status.success() { - Ok(()) - } else { - Err(anyhow!("Call to propose to set xdr rate failed")) - } - }) +#[context("Failed to authorize a subnet for use by the cycles management canister. The CMC may not be able to create canisters.")] +pub fn set_cmc_authorized_subnets(nns_url: &Url, subnet: &str) -> anyhow::Result { + let summary = format!( + "Authorize the Cycles Minting Canister to create canisters in the subnet '{}'.", + subnet.clone() + ); + let args = vec![ + "--nns-url", + nns_url.as_str(), + "propose-to-set-authorized-subnetworks", + "--test-neuron-proposer", + "--proposal-title", + "Set Cycles Minting Canister Authorized Subnets", + "--summary", + &summary, + "--subnets", + subnet, + ]; + call_extension_bundled_binary("ic-admin", args) + .map_err(|e| anyhow!("Call to propose to set authorized subnets failed: {e}")) } /// Uploads wasms to the nns-sns-wasm canister. @@ -520,37 +488,24 @@ pub fn upload_nns_sns_wasms_canister_wasms(dfx_cache_path: &Path) -> anyhow::Res .. } in SNS_CANISTERS { - let sns_cli = bundled_binary(dfx_cache_path, "sns")?; - let wasm_path = nns_wasm_dir(dfx_cache_path).join(wasm_name); - let mut command = Command::new(sns_cli); - command - .arg("add-sns-wasm-for-tests") - .arg("--network") - .arg("local") - .arg("--override-sns-wasm-canister-id-for-tests") - .arg(NNS_SNS_WASM.canister_id) - .arg("--wasm-file") - .arg(&wasm_path) - .arg(upload_name); - command - .stdin(process::Stdio::null()) - .output() - .map_err(anyhow::Error::from) - .and_then(|output| { - if output.status.success() { - Ok(()) - } else { - Err(anyhow!( - "Failed to upload {} from {} to the nns-sns-wasm canister:\n{:?} {:?}\nStdout:\n{:?}\n\nStderr:\n{:?}", - upload_name, - wasm_path.to_string_lossy(), - command.get_program(), - command.get_args(), - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - )) - } - })?; + let wasm_path = nns_wasm_dir(dfx_cache_path) + .join(wasm_name) + .display() + .to_string(); + let args = vec![ + "add-sns-wasm-for-tests", + "--network", + "local", + "--override-sns-wasm-canister-id-for-tests", + NNS_SNS_WASM.canister_id, + "--wasm-file", + &wasm_path, + upload_name, + ]; + call_extension_bundled_binary("sns-cli", &args) + .map_err(|e| anyhow!( + "Failed to upload {upload_name} from {wasm_path} to the nns-sns-wasm canister by calling `sns-cli` with args {args:?}: {e}" + ))?; } Ok(()) } @@ -604,15 +559,3 @@ pub async fn install_canister( Ok(canister_id) } - -/// Get the path to a bundled command line binary -fn bundled_binary(dfx_cache_path: &Path, cli_name: &str) -> anyhow::Result { - let bin_path = dfx_cache_path.join(cli_name); - if !bin_path.exists() { - return Err(anyhow::anyhow!(format!( - "Could not find bundled binary '{bin_path}'.", - bin_path = bin_path.display() - ))); - } - Ok(bin_path) -} diff --git a/extensions/sns/Cargo.toml b/extensions/sns/Cargo.toml index c38b967..d01c044 100644 --- a/extensions/sns/Cargo.toml +++ b/extensions/sns/Cargo.toml @@ -8,6 +8,10 @@ repository.workspace = true # Temp hack until https://github.com/axodotdev/cargo-dist/issues/187 is resovled. # Nothing will actually be published (unless `cargo publish` is executed (and it shouldn't)). publish = true +build = "build.rs" + +[build-dependencies] +dfx-extensions-utils.workspace = true [dependencies] dfx-core.workspace = true @@ -34,5 +38,4 @@ pre-release-replacements = [ ] [package.metadata.dist] -include = ["extension.json"] - +include = ["extension.json", "sns-cli"] diff --git a/extensions/sns/build.rs b/extensions/sns/build.rs new file mode 100644 index 0000000..8570d37 --- /dev/null +++ b/extensions/sns/build.rs @@ -0,0 +1,16 @@ +const REPLICA_REV: &str = "90e2799c255733409d0e61682685afcc2431c928"; + +const BINARY_DEPENDENCIES: &[(&str, &str)] = &[ + // (downloaded binary name, renamed binary name) + ("sns", "sns-cli"), +]; + +fn main() { + for (binary_name, renamed_binary_name) in BINARY_DEPENDENCIES { + dfx_extensions_utils::download_dependencies::download_ic_binary( + REPLICA_REV, + binary_name, + renamed_binary_name, + ); + } +} diff --git a/extensions/sns/e2e/tests/sns.bash b/extensions/sns/e2e/tests/sns.bash index fe10352..cd524e5 100755 --- a/extensions/sns/e2e/tests/sns.bash +++ b/extensions/sns/e2e/tests/sns.bash @@ -92,3 +92,8 @@ SNS_CONFIG_FILE_NAME="sns.yml" #dfx canister id sns_root #dfx canister id sns_swap } + +@test "sns-cli binary exists and is executable" { + run "$(dfx cache show)"/extensions/sns/sns-cli --help + assert_output --partial "Initialize, deploy and interact with an SNS" +} diff --git a/extensions/sns/src/commands/config/create.rs b/extensions/sns/src/commands/config/create.rs index e89d82d..9eb9059 100644 --- a/extensions/sns/src/commands/config/create.rs +++ b/extensions/sns/src/commands/config/create.rs @@ -1,6 +1,4 @@ //! Code for executing `dfx sns config create` -use std::path::Path; - use crate::create_config::create_config; use crate::CONFIG_FILE_NAME; use clap::Parser; @@ -11,14 +9,14 @@ use dfx_core::config::model::dfinity::Config; pub struct CreateOpts {} /// Executes `dfx sns config create` -pub fn exec(_opts: CreateOpts, dfx_cache_path: &Path) -> anyhow::Result<()> { +pub fn exec(_opts: CreateOpts) -> anyhow::Result<()> { let sns_config_path = if let Some(config) = Config::from_current_dir()? { config.get_project_root().join(CONFIG_FILE_NAME) } else { anyhow::bail!(crate::errors::DFXJSON_NOT_FOUND); }; - create_config(dfx_cache_path, &sns_config_path)?; + create_config(&sns_config_path)?; println!( "Created SNS configuration at: {}", sns_config_path.display() diff --git a/extensions/sns/src/commands/config/mod.rs b/extensions/sns/src/commands/config/mod.rs index 02dd306..9cf65e8 100644 --- a/extensions/sns/src/commands/config/mod.rs +++ b/extensions/sns/src/commands/config/mod.rs @@ -1,6 +1,4 @@ //! Code for the command line `dfx sns config`. -use std::path::Path; - use clap::Parser; mod create; @@ -29,9 +27,9 @@ enum SubCommand { } /// Executes `dfx sns config` and its subcommands. -pub fn exec(opts: SnsConfigOpts, dfx_cache_path: &Path) -> anyhow::Result<()> { +pub fn exec(opts: SnsConfigOpts) -> anyhow::Result<()> { match opts.subcmd { - SubCommand::Create(v) => create::exec(v, dfx_cache_path), - SubCommand::Validate(v) => validate::exec(v, dfx_cache_path), + SubCommand::Create(v) => create::exec(v), + SubCommand::Validate(v) => validate::exec(v), } } diff --git a/extensions/sns/src/commands/config/validate.rs b/extensions/sns/src/commands/config/validate.rs index a31a1fb..b02be30 100644 --- a/extensions/sns/src/commands/config/validate.rs +++ b/extensions/sns/src/commands/config/validate.rs @@ -1,6 +1,4 @@ //! Code for executing `dfx sns config validate` -use std::path::Path; - use crate::validate_config::validate_config; use crate::CONFIG_FILE_NAME; use clap::Parser; @@ -11,11 +9,11 @@ use dfx_core::config::model::dfinity::Config; pub struct ValidateOpts {} /// Executes `dfx sns config validate` -pub fn exec(_opts: ValidateOpts, dfx_cache_path: &Path) -> anyhow::Result<()> { +pub fn exec(_opts: ValidateOpts) -> anyhow::Result<()> { let sns_config_path = if let Some(config) = Config::from_current_dir()? { config.get_project_root().join(CONFIG_FILE_NAME) } else { anyhow::bail!(crate::errors::DFXJSON_NOT_FOUND); }; - validate_config(dfx_cache_path, &sns_config_path).map(|stdout| println!("{}", stdout)) + validate_config(&sns_config_path).map(|stdout| println!("{}", stdout)) } diff --git a/extensions/sns/src/commands/deploy.rs b/extensions/sns/src/commands/deploy.rs index 82a15c2..4568793 100644 --- a/extensions/sns/src/commands/deploy.rs +++ b/extensions/sns/src/commands/deploy.rs @@ -1,6 +1,4 @@ //! Code for the command line `dfx sns deploy`. -use std::path::Path; - use crate::deploy::deploy_sns; use crate::CONFIG_FILE_NAME; use clap::Parser; @@ -15,7 +13,7 @@ use dfx_core::config::model::dfinity::Config; pub struct DeployOpts {} /// Executes the command line `dfx sns deploy`. -pub fn exec(_opts: DeployOpts, dfx_cache_path: &Path) -> anyhow::Result<()> { +pub fn exec(_opts: DeployOpts) -> anyhow::Result<()> { println!("Creating SNS canisters. This typically takes about one minute..."); let sns_config_path = if let Some(config) = Config::from_current_dir()? { config.get_project_root().join(CONFIG_FILE_NAME) @@ -23,6 +21,6 @@ pub fn exec(_opts: DeployOpts, dfx_cache_path: &Path) -> anyhow::Result<()> { anyhow::bail!(crate::errors::DFXJSON_NOT_FOUND); }; - println!("{}", deploy_sns(dfx_cache_path, &sns_config_path)?); + println!("{}", deploy_sns(&sns_config_path)?); Ok(()) } diff --git a/extensions/sns/src/create_config.rs b/extensions/sns/src/create_config.rs index e8de87d..4d7111c 100644 --- a/extensions/sns/src/create_config.rs +++ b/extensions/sns/src/create_config.rs @@ -3,17 +3,17 @@ use fn_error_context::context; use std::ffi::OsString; use std::path::Path; -use dfx_extensions_utils::call_bundled; +use dfx_extensions_utils::call_extension_bundled_binary; /// Ceates an SNS configuration template. #[context("Failed to create sns config at {}.", path.display())] -pub fn create_config(dfx_cache_path: &Path, path: &Path) -> anyhow::Result<()> { +pub fn create_config(path: &Path) -> anyhow::Result<()> { let args = vec![ OsString::from("init-config-file"), OsString::from("--init-config-file-path"), OsString::from(path), OsString::from("new"), ]; - call_bundled(&dfx_cache_path, "sns", &args)?; + call_extension_bundled_binary("sns-cli", &args)?; Ok(()) } diff --git a/extensions/sns/src/deploy.rs b/extensions/sns/src/deploy.rs index bf3409e..8ac1e4f 100644 --- a/extensions/sns/src/deploy.rs +++ b/extensions/sns/src/deploy.rs @@ -4,11 +4,11 @@ use fn_error_context::context; use std::ffi::OsString; use std::path::Path; -use dfx_extensions_utils::call_bundled; +use dfx_extensions_utils::call_extension_bundled_binary; /// Creates an SNS. This requires funds but no proposal. #[context("Failed to deploy SNS with config: {}", path.display())] -pub fn deploy_sns(dfx_cache_path: &Path, path: &Path) -> anyhow::Result { +pub fn deploy_sns(path: &Path) -> anyhow::Result { // Note: It MAY be possible to get the did file location using existing sdk methods. let did_file = "candid/nns-sns-wasm.did"; if !Path::new(did_file).exists() { @@ -30,7 +30,7 @@ pub fn deploy_sns(dfx_cache_path: &Path, path: &Path) -> anyhow::Result OsString::from("--save-to"), OsString::from(canister_ids_file), ]; - call_bundled(dfx_cache_path, "sns", &args).map(|stdout| { + call_extension_bundled_binary("sns-cli", &args).map(|stdout| { format!( "Deployed SNS:\nSNS config: {}\nCanister ID file: {}\n\n{}", path.display(), diff --git a/extensions/sns/src/main.rs b/extensions/sns/src/main.rs index d011989..a335827 100644 --- a/extensions/sns/src/main.rs +++ b/extensions/sns/src/main.rs @@ -60,9 +60,9 @@ fn main() -> anyhow::Result<()> { })?; match opts.subcmd { - SubCommand::Config(v) => commands::config::exec(v, &dfx_cache_path), + SubCommand::Config(v) => commands::config::exec(v), SubCommand::Import(v) => commands::import::exec(v, &dfx_cache_path), - SubCommand::Deploy(v) => commands::deploy::exec(v, &dfx_cache_path), + SubCommand::Deploy(v) => commands::deploy::exec(v), SubCommand::Download(v) => commands::download::exec(v, &dfx_cache_path), }?; Ok(()) diff --git a/extensions/sns/src/validate_config.rs b/extensions/sns/src/validate_config.rs index 33a7cb5..4281d91 100644 --- a/extensions/sns/src/validate_config.rs +++ b/extensions/sns/src/validate_config.rs @@ -3,17 +3,19 @@ use fn_error_context::context; use std::ffi::OsString; use std::path::Path; -use dfx_extensions_utils::call_bundled; +use dfx_extensions_utils::call_extension_bundled_binary; /// Checks whether an SNS configuration file is valid. #[context("Failed to validate SNS config at {}.", path.display())] -pub fn validate_config(dfx_cache_path: &Path, path: &Path) -> anyhow::Result { +pub fn validate_config(path: &Path) -> anyhow::Result { let args = vec![ OsString::from("init-config-file"), OsString::from("--init-config-file-path"), OsString::from(path), OsString::from("validate"), ]; - call_bundled(dfx_cache_path, "sns", &args) + // current binary + + call_extension_bundled_binary("sns-cli", &args) .map(|_| format!("SNS config file is valid: {}", path.display())) } From 0a6de4feffa6fbbbe665fcd936bff7ae56681e8b Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Tue, 8 Aug 2023 00:38:24 +0200 Subject: [PATCH 02/10] applying reviewer's suggestions --- Cargo.lock | 5 ++ extensions-utils/Cargo.toml | 2 + extensions-utils/src/dependencies/call.rs | 63 +++++++++++++++++ .../src/{ => dependencies}/dfx.rs | 58 +-------------- .../src/dependencies/download_ic_binaries.rs | 64 +++++++++++++++++ .../{ => dependencies}/download_wasms/mod.rs | 0 .../{ => dependencies}/download_wasms/nns.rs | 0 .../{ => dependencies}/download_wasms/sns.rs | 0 extensions-utils/src/dependencies/mod.rs | 4 ++ extensions-utils/src/download_dependencies.rs | 44 ------------ extensions-utils/src/lib.rs | 27 ++++--- extensions/nns/build.rs | 10 +-- extensions/nns/src/install_nns.rs | 70 ++++++++++--------- extensions/sns/build.rs | 10 +-- extensions/sns/src/commands/config/create.rs | 6 +- extensions/sns/src/commands/config/mod.rs | 8 ++- .../sns/src/commands/config/validate.rs | 6 +- extensions/sns/src/commands/deploy.rs | 6 +- extensions/sns/src/create_config.rs | 4 +- extensions/sns/src/deploy.rs | 4 +- extensions/sns/src/main.rs | 4 +- extensions/sns/src/validate_config.rs | 4 +- 22 files changed, 224 insertions(+), 175 deletions(-) create mode 100644 extensions-utils/src/dependencies/call.rs rename extensions-utils/src/{ => dependencies}/dfx.rs (60%) create mode 100644 extensions-utils/src/dependencies/download_ic_binaries.rs rename extensions-utils/src/{ => dependencies}/download_wasms/mod.rs (100%) rename extensions-utils/src/{ => dependencies}/download_wasms/nns.rs (100%) rename extensions-utils/src/{ => dependencies}/download_wasms/sns.rs (100%) create mode 100644 extensions-utils/src/dependencies/mod.rs delete mode 100644 extensions-utils/src/download_dependencies.rs diff --git a/Cargo.lock b/Cargo.lock index 0b24574..0a48dc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -279,9 +279,12 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ + "futures-core", "getrandom", "instant", + "pin-project-lite", "rand", + "tokio", ] [[package]] @@ -930,6 +933,7 @@ name = "dfx-extensions-utils" version = "0.0.0" dependencies = [ "anyhow", + "backoff", "dfx-core", "flate2", "fn-error-context", @@ -945,6 +949,7 @@ dependencies = [ "slog-term", "tempfile", "thiserror", + "tokio", "url", ] diff --git a/extensions-utils/Cargo.toml b/extensions-utils/Cargo.toml index aef537c..c0a085d 100644 --- a/extensions-utils/Cargo.toml +++ b/extensions-utils/Cargo.toml @@ -17,6 +17,7 @@ reqwest.workspace = true dfx-core.workspace = true anyhow.workspace = true +backoff = { version = "0.4.0", features = [ "futures", "tokio" ] } flate2 = { version = "1.0.25", default-features = false, features = ["zlib-ng"] } fn-error-context.workspace = true futures-util.workspace = true @@ -31,4 +32,5 @@ slog-term = "2.9.0" slog.workspace = true tempfile.workspace = true thiserror = "1.0.40" +tokio.workspace = true url.workspace = true diff --git a/extensions-utils/src/dependencies/call.rs b/extensions-utils/src/dependencies/call.rs new file mode 100644 index 0000000..74ca9e7 --- /dev/null +++ b/extensions-utils/src/dependencies/call.rs @@ -0,0 +1,63 @@ +use anyhow::anyhow; +use fn_error_context::context; +use std::{ + ffi::OsStr, + path::Path, + process::{self, Command}, +}; + +/// Calls a binary that was delivered with an extension tarball. +/// +/// # Returns +/// - On success, returns stdout as a string. +/// - On error, returns an error message including stdout and stderr. +#[context("Calling {} CLI failed, or, it returned an error.", binary_name)] +pub fn call_extension_bundled_binary( + dfx_cache_path: &Path, + binary_name: &str, + args: I, +) -> anyhow::Result +where + I: IntoIterator, + S: AsRef, +{ + let extension_binary_path = + std::env::current_exe().map_err(|e| anyhow::anyhow!("Failed to get current exe: {}", e))?; + let extension_dir_path = extension_binary_path.parent().ok_or_else(|| { + anyhow::anyhow!( + "Failed to locate parent of dir of executable: {}", + extension_binary_path.display() + ) + })?; + let binary_to_call = extension_dir_path.join(binary_name); + // TODO + dbg!(&binary_to_call); + std::fs::remove_file(binary_to_call.clone()).unwrap(); + panic!("trying to prettify command output, but something is not working the way I expect."); + let mut command = Command::new(&binary_to_call); + // If extension's dependency calls dfx; it should call dfx in this dir. + command.env("PATH", dfx_cache_path.join("dfx")); + command.args(args); + command + .stdin(process::Stdio::null()) + .output() + .map_err(anyhow::Error::from) + .and_then(|output| -> Result { + if output.status.success() { + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + } else { + let args: Vec<_> = command + .get_args() + .into_iter() + .map(OsStr::to_string_lossy) + .collect(); + Err(anyhow!( + "Call failed:\n{:?} {}\nStdout:\n{}\n\nStderr:\n{}", + command.get_program(), + args.join(" "), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )) + } + }) +} diff --git a/extensions-utils/src/dfx.rs b/extensions-utils/src/dependencies/dfx.rs similarity index 60% rename from extensions-utils/src/dfx.rs rename to extensions-utils/src/dependencies/dfx.rs index f5a509c..992a576 100644 --- a/extensions-utils/src/dfx.rs +++ b/extensions-utils/src/dependencies/dfx.rs @@ -23,64 +23,10 @@ where S: AsRef, { let binary = dfx_cache_path.join(command); - let mut command = Command::new(&binary); + // If extension's dependency calls dfx; it should call dfx in this dir. + command.env("PATH", dfx_cache_path.join("dfx")); command.args(args); - // The sns command line tool itself calls dfx; it should call this dfx. - // The sns command line tool should not rely on commands not packaged with dfx. - // The same applies to other bundled binaries. - command.env("PATH", binary.parent().unwrap_or_else(|| Path::new("."))); - command - .stdin(process::Stdio::null()) - .output() - .map_err(anyhow::Error::from) - .and_then(|output| { - if output.status.success() { - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) - } else { - let args: Vec<_> = command - .get_args() - .into_iter() - .map(OsStr::to_string_lossy) - .collect(); - Err(anyhow!( - "Call failed:\n{:?} {}\nStdout:\n{}\n\nStderr:\n{}", - command.get_program(), - args.join(" "), - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - )) - } - }) -} - -/// Calls a binary that was delivered with an extension tarball. -/// -/// # Returns -/// - On success, returns stdout as a string. -/// - On error, returns an error message including stdout and stderr. -#[context("Calling {} CLI failed, or, it returned an error.", binary_name)] -pub fn call_extension_bundled_binary(binary_name: &str, args: I) -> anyhow::Result -where - I: IntoIterator, - S: AsRef, -{ - let extension_binary_path = - std::env::current_exe().map_err(|e| anyhow::anyhow!("Failed to get current exe: {}", e))?; - let extension_dir_path = extension_binary_path - .parent() - .unwrap_or_else(|| Path::new(".")); - let binary_to_call = extension_dir_path.join(binary_name); - - let mut command = Command::new(&binary_to_call); - command.args(args); - // The sns command line tool itself calls dfx; it should call this dfx. - // The sns command line tool should not rely on commands not packaged with dfx. - // The same applies to other bundled binaries. - command.env( - "PATH", - binary_to_call.parent().unwrap_or_else(|| Path::new(".")), - ); command .stdin(process::Stdio::null()) .output() diff --git a/extensions-utils/src/dependencies/download_ic_binaries.rs b/extensions-utils/src/dependencies/download_ic_binaries.rs new file mode 100644 index 0000000..5347323 --- /dev/null +++ b/extensions-utils/src/dependencies/download_ic_binaries.rs @@ -0,0 +1,64 @@ +use backoff::future::retry; +use backoff::ExponentialBackoffBuilder; +use flate2::read::GzDecoder; +use std::path::Path; +use std::time::Duration; +use std::{fs, io::copy}; +use tokio::runtime::Runtime; + +pub fn download_ic_binary(replica_rev: &str, binary_name: &str, destination_path: &Path) { + let arch = match std::env::consts::ARCH { + "x86_64" => "x86_64", + "aarch64" => "x86_64", // let's rely on rosetta2 for now, since ic binaiers are not available for arm64 + _ => panic!("Unsupported architecture"), + }; + let os = match std::env::consts::OS { + "macos" => "darwin", + "linux" => "linux", + // "windows" => "windows", // unsupported till dfx supports windows + _ => panic!("Unsupported OS"), + }; + + let url = format!( + "https://download.dfinity.systems/ic/{replica_rev}/openssl-static-binaries/{arch}-{os}/{binary_name}.gz", + arch = arch, + os = os, + binary_name = binary_name, + ); + println!("Downloading {}", url); + + let bytes = Runtime::new().unwrap().block_on(download_bytes(&url)); + let mut d = GzDecoder::new(&*bytes); + let tempdir = tempfile::tempdir().expect("Failed to create temp dir"); + let temp_file = tempdir.path().join(binary_name); + let mut temp = fs::File::create(&temp_file).expect("Failed to create the file"); + copy(&mut d, &mut temp).expect("Failed to copy content"); + + fs::rename(temp_file, &destination_path).expect("Failed to move extension"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + dfx_core::fs::set_permissions(&destination_path, std::fs::Permissions::from_mode(0o500)) + .expect("Failed to set permissions"); + } +} + +async fn download_bytes(url: &str) -> Vec { + let retry_policy = ExponentialBackoffBuilder::new() + .with_initial_interval(Duration::from_secs(1)) + .with_max_interval(Duration::from_secs(16)) + .with_multiplier(2.0) + .with_max_elapsed_time(Some(Duration::from_secs(300))) + .build(); + let resp = retry(retry_policy, || async { + match reqwest::get(url).await { + Ok(response) => Ok(response), + Err(err) => Err(backoff::Error::transient(err)), + } + }) + .await + .unwrap(); + + let bytes = resp.bytes().await.expect("Failed to read response"); + bytes.to_vec() +} diff --git a/extensions-utils/src/download_wasms/mod.rs b/extensions-utils/src/dependencies/download_wasms/mod.rs similarity index 100% rename from extensions-utils/src/download_wasms/mod.rs rename to extensions-utils/src/dependencies/download_wasms/mod.rs diff --git a/extensions-utils/src/download_wasms/nns.rs b/extensions-utils/src/dependencies/download_wasms/nns.rs similarity index 100% rename from extensions-utils/src/download_wasms/nns.rs rename to extensions-utils/src/dependencies/download_wasms/nns.rs diff --git a/extensions-utils/src/download_wasms/sns.rs b/extensions-utils/src/dependencies/download_wasms/sns.rs similarity index 100% rename from extensions-utils/src/download_wasms/sns.rs rename to extensions-utils/src/dependencies/download_wasms/sns.rs diff --git a/extensions-utils/src/dependencies/mod.rs b/extensions-utils/src/dependencies/mod.rs new file mode 100644 index 0000000..be50e04 --- /dev/null +++ b/extensions-utils/src/dependencies/mod.rs @@ -0,0 +1,4 @@ +pub mod call; +pub mod dfx; +pub mod download_ic_binaries; +pub mod download_wasms; diff --git a/extensions-utils/src/download_dependencies.rs b/extensions-utils/src/download_dependencies.rs deleted file mode 100644 index deb8ceb..0000000 --- a/extensions-utils/src/download_dependencies.rs +++ /dev/null @@ -1,44 +0,0 @@ -use flate2::read::GzDecoder; -use std::fs::File; -use std::path::Path; -use std::{env, fs, io::copy}; - -pub fn download_ic_binary(replica_rev: &str, binary_name: &str, renamed_binary_name: &str) { - let arch = match std::env::consts::ARCH { - "x86_64" => "x86_64", - "aarch64" => "x86_64", // let's rely on rosetta2 for now, since sns-cli is not available for arm64 - _ => panic!("Unsupported architecture"), - }; - let os = match std::env::consts::OS { - "macos" => "darwin", - "linux" => "linux", - // "windows" => "windows", // unsupported till dfx supports windows - _ => panic!("Unsupported OS"), - }; - - let url = format!( - "https://download.dfinity.systems/ic/{replica_rev}/openssl-static-binaries/{arch}-{os}/{binary_name}.gz", - arch = arch, - os = os, - binary_name = binary_name, - ); - println!("Downloading {}", url); - - let resp = reqwest::blocking::get(&url).expect("Request failed"); - let bytes = resp.bytes().expect("Failed to read response"); - - let mut d = GzDecoder::new(&*bytes); - let mut dest = File::create(binary_name).expect("Failed to create sns file"); - - copy(&mut d, &mut dest).expect("Failed to copy content"); - - let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); - let out_path = Path::new(&manifest_dir).join(renamed_binary_name); - fs::rename(binary_name, &out_path).expect("Failed to move extension"); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - dfx_core::fs::set_permissions(&out_path, std::fs::Permissions::from_mode(0o777)) - .expect("Failed to set permissions"); - } -} diff --git a/extensions-utils/src/lib.rs b/extensions-utils/src/lib.rs index 66dd22e..512cdaf 100644 --- a/extensions-utils/src/lib.rs +++ b/extensions-utils/src/lib.rs @@ -1,24 +1,23 @@ //! Library for calling bundled command line tools. -mod dfx; -pub mod download_dependencies; -mod download_wasms; +pub mod dependencies; mod error; mod logger; mod project; -pub use dfx::{ - call_dfx_bundled_binary, call_extension_bundled_binary, dfx_version, replica_rev, - webserver_port, +pub use dependencies::{ + call::call_extension_bundled_binary, + dfx::{call_dfx_bundled_binary, dfx_version, replica_rev, webserver_port}, + download_ic_binaries::download_ic_binary, + download_wasms::{ + download_ic_repo_wasm, + nns::{ + download_nns_wasms, nns_wasm_dir, IcNnsInitCanister, StandardCanister, + ED25519_TEST_ACCOUNT, NNS_CORE, NNS_FRONTEND, NNS_SNS_WASM, SECP256K1_TEST_ACCOUNT, + }, + sns::{download_sns_wasms, SnsCanisterInstallation, SNS_CANISTERS}, + }, }; -pub use download_wasms::download_ic_repo_wasm; -pub use download_wasms::nns::download_nns_wasms; -pub use download_wasms::nns::{ - nns_wasm_dir, IcNnsInitCanister, StandardCanister, ED25519_TEST_ACCOUNT, NNS_CORE, - NNS_FRONTEND, NNS_SNS_WASM, SECP256K1_TEST_ACCOUNT, -}; -pub use download_wasms::sns::SnsCanisterInstallation; -pub use download_wasms::sns::{download_sns_wasms, SNS_CANISTERS}; pub use logger::new_logger; pub use project::import::import_canister_definitions; pub use project::network_mappings::get_network_mappings; diff --git a/extensions/nns/build.rs b/extensions/nns/build.rs index f9cff35..43a2294 100644 --- a/extensions/nns/build.rs +++ b/extensions/nns/build.rs @@ -1,3 +1,5 @@ +use std::path::Path; + const REPLICA_REV: &str = "90e2799c255733409d0e61682685afcc2431c928"; const BINARY_DEPENDENCIES: &[(&str, &str)] = &[ @@ -8,11 +10,9 @@ const BINARY_DEPENDENCIES: &[(&str, &str)] = &[ ]; fn main() { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); for (binary_name, renamed_binary_name) in BINARY_DEPENDENCIES { - dfx_extensions_utils::download_dependencies::download_ic_binary( - REPLICA_REV, - binary_name, - renamed_binary_name, - ); + let destination_path = Path::new(&manifest_dir).join(renamed_binary_name); + dfx_extensions_utils::download_ic_binary(REPLICA_REV, binary_name, &destination_path); } } diff --git a/extensions/nns/src/install_nns.rs b/extensions/nns/src/install_nns.rs index dc9078f..a4ea1bd 100644 --- a/extensions/nns/src/install_nns.rs +++ b/extensions/nns/src/install_nns.rs @@ -28,6 +28,7 @@ use ic_utils::interfaces::management_canister::builders::InstallMode; use ic_utils::interfaces::ManagementCanister; use reqwest::Url; use slog::Logger; +use std::ffi::OsString; use std::fs; use std::io::Write; use std::path::Component; @@ -78,7 +79,7 @@ pub async fn install_nns( test_accounts, sns_subnets: Some(subnet_id.to_string()), }; - ic_nns_init(&ic_nns_init_opts).await?; + ic_nns_init(&ic_nns_init_opts, dfx_cache_path).await?; eprintln!("Uploading NNS configuration data..."); upload_nns_sns_wasms_canister_wasms(dfx_cache_path)?; @@ -112,8 +113,8 @@ pub async fn install_nns( } // ... and configure the backend NNS canisters: eprintln!("Configuring the NNS..."); - set_xdr_rate(1234567, &nns_url)?; - set_cmc_authorized_subnets(&nns_url, &subnet_id)?; + set_xdr_rate(1234567, &nns_url, dfx_cache_path)?; + set_cmc_authorized_subnets(&nns_url, &subnet_id, dfx_cache_path)?; print_nns_details(provider_url)?; Ok(()) @@ -412,24 +413,24 @@ pub struct IcNnsInitOpts { /// - This won't work with an HSM, because the agent holds a session open /// - The provider_url is what the agent connects to, and forwards to the replica. #[context("Failed to install NNS components.")] -pub async fn ic_nns_init(opts: &IcNnsInitOpts) -> anyhow::Result { - let mut args = vec![ - "--pass-specified-id", - "--url", - &opts.nns_url, - "--wasm-dir", - opts.wasm_dir.to_str().unwrap(), +pub async fn ic_nns_init(opts: &IcNnsInitOpts, dfx_cache_path: &Path) -> anyhow::Result { + let mut args: Vec = vec![ + "--pass-specified-id".into(), + "--url".into(), + opts.nns_url.clone().into(), + "--wasm-dir".into(), + opts.wasm_dir.as_os_str().into(), ]; for account in &opts.test_accounts { - args.push("--initialize-ledger-with-test-accounts"); - args.push(account); + args.push("--initialize-ledger-with-test-accounts".into()); + args.push(account.into()); } if let Some(subnets) = &opts.sns_subnets { - args.push("--sns-subnet"); - args.push(subnets); + args.push("--sns-subnet".into()); + args.push(subnets.into()); } - call_extension_bundled_binary("ic-nns-init", &args) - .with_context(|| format!("Error executing `ic-admin` with args: {:?}", args)) + call_extension_bundled_binary(dfx_cache_path, "ic-nns-init", &args) + .with_context(|| format!("Error executing `ic-nns-init` with args: {:?}", args)) } /// Sets the exchange rate between ICP and cycles. @@ -439,7 +440,7 @@ pub async fn ic_nns_init(opts: &IcNnsInitOpts) -> anyhow::Result { /// proposals with a test user pass immediately, as the small test neuron is /// the only neuron and has absolute majority. #[context("Failed to set an initial exchange rate between ICP and cycles. It may not be possible to create canisters or purchase cycles.")] -pub fn set_xdr_rate(rate: u64, nns_url: &Url) -> anyhow::Result { +pub fn set_xdr_rate(rate: u64, nns_url: &Url, dfx_cache_path: &Path) -> anyhow::Result { let summary = format!("Set the cycle exchange rate to {}.", rate.clone()); let xdr_permyriad_per_icp = rate.to_string(); let args = vec![ @@ -452,13 +453,17 @@ pub fn set_xdr_rate(rate: u64, nns_url: &Url) -> anyhow::Result { "--xdr-permyriad-per-icp", &xdr_permyriad_per_icp, ]; - call_extension_bundled_binary("ic-admin", args) + call_extension_bundled_binary(dfx_cache_path, "ic-admin", args) .map_err(|e| anyhow!("Call to propose to set xdr rate failed: {e}")) } /// Sets the subnets the CMC is authorized to create canisters in. #[context("Failed to authorize a subnet for use by the cycles management canister. The CMC may not be able to create canisters.")] -pub fn set_cmc_authorized_subnets(nns_url: &Url, subnet: &str) -> anyhow::Result { +pub fn set_cmc_authorized_subnets( + nns_url: &Url, + subnet: &str, + dfx_cache_path: &Path, +) -> anyhow::Result { let summary = format!( "Authorize the Cycles Minting Canister to create canisters in the subnet '{}'.", subnet.clone() @@ -475,7 +480,7 @@ pub fn set_cmc_authorized_subnets(nns_url: &Url, subnet: &str) -> anyhow::Result "--subnets", subnet, ]; - call_extension_bundled_binary("ic-admin", args) + call_extension_bundled_binary(dfx_cache_path, "ic-admin", args) .map_err(|e| anyhow!("Call to propose to set authorized subnets failed: {e}")) } @@ -488,23 +493,20 @@ pub fn upload_nns_sns_wasms_canister_wasms(dfx_cache_path: &Path) -> anyhow::Res .. } in SNS_CANISTERS { - let wasm_path = nns_wasm_dir(dfx_cache_path) - .join(wasm_name) - .display() - .to_string(); + let wasm_path = nns_wasm_dir(dfx_cache_path).join(wasm_name); let args = vec![ - "add-sns-wasm-for-tests", - "--network", - "local", - "--override-sns-wasm-canister-id-for-tests", - NNS_SNS_WASM.canister_id, - "--wasm-file", - &wasm_path, - upload_name, + "add-sns-wasm-for-tests".into(), + "--network".into(), + "local".into(), + "--override-sns-wasm-canister-id-for-tests".into(), + NNS_SNS_WASM.canister_id.into(), + "--wasm-file".into(), + wasm_path.clone().into_os_string(), + upload_name.into(), ]; - call_extension_bundled_binary("sns-cli", &args) + call_extension_bundled_binary(dfx_cache_path,"sns-cli", &args) .map_err(|e| anyhow!( - "Failed to upload {upload_name} from {wasm_path} to the nns-sns-wasm canister by calling `sns-cli` with args {args:?}: {e}" + "Failed to upload {upload_name} from {wasm_path:?} to the nns-sns-wasm canister by calling `sns-cli` with args {args:?}: {e}" ))?; } Ok(()) diff --git a/extensions/sns/build.rs b/extensions/sns/build.rs index 8570d37..5d8a598 100644 --- a/extensions/sns/build.rs +++ b/extensions/sns/build.rs @@ -1,3 +1,5 @@ +use std::path::Path; + const REPLICA_REV: &str = "90e2799c255733409d0e61682685afcc2431c928"; const BINARY_DEPENDENCIES: &[(&str, &str)] = &[ @@ -6,11 +8,9 @@ const BINARY_DEPENDENCIES: &[(&str, &str)] = &[ ]; fn main() { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); for (binary_name, renamed_binary_name) in BINARY_DEPENDENCIES { - dfx_extensions_utils::download_dependencies::download_ic_binary( - REPLICA_REV, - binary_name, - renamed_binary_name, - ); + let destination_path = Path::new(&manifest_dir).join(renamed_binary_name); + dfx_extensions_utils::download_ic_binary(REPLICA_REV, binary_name, &destination_path); } } diff --git a/extensions/sns/src/commands/config/create.rs b/extensions/sns/src/commands/config/create.rs index 9eb9059..927a5e3 100644 --- a/extensions/sns/src/commands/config/create.rs +++ b/extensions/sns/src/commands/config/create.rs @@ -1,4 +1,6 @@ //! Code for executing `dfx sns config create` +use std::path::Path; + use crate::create_config::create_config; use crate::CONFIG_FILE_NAME; use clap::Parser; @@ -9,14 +11,14 @@ use dfx_core::config::model::dfinity::Config; pub struct CreateOpts {} /// Executes `dfx sns config create` -pub fn exec(_opts: CreateOpts) -> anyhow::Result<()> { +pub fn exec(_opts: CreateOpts, dfx_cache_path: &Path) -> anyhow::Result<()> { let sns_config_path = if let Some(config) = Config::from_current_dir()? { config.get_project_root().join(CONFIG_FILE_NAME) } else { anyhow::bail!(crate::errors::DFXJSON_NOT_FOUND); }; - create_config(&sns_config_path)?; + create_config(&sns_config_path, dfx_cache_path)?; println!( "Created SNS configuration at: {}", sns_config_path.display() diff --git a/extensions/sns/src/commands/config/mod.rs b/extensions/sns/src/commands/config/mod.rs index 9cf65e8..02dd306 100644 --- a/extensions/sns/src/commands/config/mod.rs +++ b/extensions/sns/src/commands/config/mod.rs @@ -1,4 +1,6 @@ //! Code for the command line `dfx sns config`. +use std::path::Path; + use clap::Parser; mod create; @@ -27,9 +29,9 @@ enum SubCommand { } /// Executes `dfx sns config` and its subcommands. -pub fn exec(opts: SnsConfigOpts) -> anyhow::Result<()> { +pub fn exec(opts: SnsConfigOpts, dfx_cache_path: &Path) -> anyhow::Result<()> { match opts.subcmd { - SubCommand::Create(v) => create::exec(v), - SubCommand::Validate(v) => validate::exec(v), + SubCommand::Create(v) => create::exec(v, dfx_cache_path), + SubCommand::Validate(v) => validate::exec(v, dfx_cache_path), } } diff --git a/extensions/sns/src/commands/config/validate.rs b/extensions/sns/src/commands/config/validate.rs index b02be30..7c7859e 100644 --- a/extensions/sns/src/commands/config/validate.rs +++ b/extensions/sns/src/commands/config/validate.rs @@ -1,4 +1,6 @@ //! Code for executing `dfx sns config validate` +use std::path::Path; + use crate::validate_config::validate_config; use crate::CONFIG_FILE_NAME; use clap::Parser; @@ -9,11 +11,11 @@ use dfx_core::config::model::dfinity::Config; pub struct ValidateOpts {} /// Executes `dfx sns config validate` -pub fn exec(_opts: ValidateOpts) -> anyhow::Result<()> { +pub fn exec(_opts: ValidateOpts, dfx_cache_path: &Path) -> anyhow::Result<()> { let sns_config_path = if let Some(config) = Config::from_current_dir()? { config.get_project_root().join(CONFIG_FILE_NAME) } else { anyhow::bail!(crate::errors::DFXJSON_NOT_FOUND); }; - validate_config(&sns_config_path).map(|stdout| println!("{}", stdout)) + validate_config(&sns_config_path, dfx_cache_path).map(|stdout| println!("{}", stdout)) } diff --git a/extensions/sns/src/commands/deploy.rs b/extensions/sns/src/commands/deploy.rs index 4568793..efae4ae 100644 --- a/extensions/sns/src/commands/deploy.rs +++ b/extensions/sns/src/commands/deploy.rs @@ -1,4 +1,6 @@ //! Code for the command line `dfx sns deploy`. +use std::path::Path; + use crate::deploy::deploy_sns; use crate::CONFIG_FILE_NAME; use clap::Parser; @@ -13,7 +15,7 @@ use dfx_core::config::model::dfinity::Config; pub struct DeployOpts {} /// Executes the command line `dfx sns deploy`. -pub fn exec(_opts: DeployOpts) -> anyhow::Result<()> { +pub fn exec(_opts: DeployOpts, dfx_cache_path: &Path) -> anyhow::Result<()> { println!("Creating SNS canisters. This typically takes about one minute..."); let sns_config_path = if let Some(config) = Config::from_current_dir()? { config.get_project_root().join(CONFIG_FILE_NAME) @@ -21,6 +23,6 @@ pub fn exec(_opts: DeployOpts) -> anyhow::Result<()> { anyhow::bail!(crate::errors::DFXJSON_NOT_FOUND); }; - println!("{}", deploy_sns(&sns_config_path)?); + println!("{}", deploy_sns(&sns_config_path, dfx_cache_path)?); Ok(()) } diff --git a/extensions/sns/src/create_config.rs b/extensions/sns/src/create_config.rs index 4d7111c..67c4d2b 100644 --- a/extensions/sns/src/create_config.rs +++ b/extensions/sns/src/create_config.rs @@ -7,13 +7,13 @@ use dfx_extensions_utils::call_extension_bundled_binary; /// Ceates an SNS configuration template. #[context("Failed to create sns config at {}.", path.display())] -pub fn create_config(path: &Path) -> anyhow::Result<()> { +pub fn create_config(path: &Path, dfx_cache_path: &Path) -> anyhow::Result<()> { let args = vec![ OsString::from("init-config-file"), OsString::from("--init-config-file-path"), OsString::from(path), OsString::from("new"), ]; - call_extension_bundled_binary("sns-cli", &args)?; + call_extension_bundled_binary(dfx_cache_path, "sns-cli", &args)?; Ok(()) } diff --git a/extensions/sns/src/deploy.rs b/extensions/sns/src/deploy.rs index 8ac1e4f..2e95952 100644 --- a/extensions/sns/src/deploy.rs +++ b/extensions/sns/src/deploy.rs @@ -8,7 +8,7 @@ use dfx_extensions_utils::call_extension_bundled_binary; /// Creates an SNS. This requires funds but no proposal. #[context("Failed to deploy SNS with config: {}", path.display())] -pub fn deploy_sns(path: &Path) -> anyhow::Result { +pub fn deploy_sns(path: &Path, dfx_cache_path: &Path) -> anyhow::Result { // Note: It MAY be possible to get the did file location using existing sdk methods. let did_file = "candid/nns-sns-wasm.did"; if !Path::new(did_file).exists() { @@ -30,7 +30,7 @@ pub fn deploy_sns(path: &Path) -> anyhow::Result { OsString::from("--save-to"), OsString::from(canister_ids_file), ]; - call_extension_bundled_binary("sns-cli", &args).map(|stdout| { + call_extension_bundled_binary(dfx_cache_path, "sns-cli", &args).map(|stdout| { format!( "Deployed SNS:\nSNS config: {}\nCanister ID file: {}\n\n{}", path.display(), diff --git a/extensions/sns/src/main.rs b/extensions/sns/src/main.rs index a335827..d011989 100644 --- a/extensions/sns/src/main.rs +++ b/extensions/sns/src/main.rs @@ -60,9 +60,9 @@ fn main() -> anyhow::Result<()> { })?; match opts.subcmd { - SubCommand::Config(v) => commands::config::exec(v), + SubCommand::Config(v) => commands::config::exec(v, &dfx_cache_path), SubCommand::Import(v) => commands::import::exec(v, &dfx_cache_path), - SubCommand::Deploy(v) => commands::deploy::exec(v), + SubCommand::Deploy(v) => commands::deploy::exec(v, &dfx_cache_path), SubCommand::Download(v) => commands::download::exec(v, &dfx_cache_path), }?; Ok(()) diff --git a/extensions/sns/src/validate_config.rs b/extensions/sns/src/validate_config.rs index 4281d91..02c793c 100644 --- a/extensions/sns/src/validate_config.rs +++ b/extensions/sns/src/validate_config.rs @@ -7,7 +7,7 @@ use dfx_extensions_utils::call_extension_bundled_binary; /// Checks whether an SNS configuration file is valid. #[context("Failed to validate SNS config at {}.", path.display())] -pub fn validate_config(path: &Path) -> anyhow::Result { +pub fn validate_config(path: &Path, dfx_cache_path: &Path) -> anyhow::Result { let args = vec![ OsString::from("init-config-file"), OsString::from("--init-config-file-path"), @@ -16,6 +16,6 @@ pub fn validate_config(path: &Path) -> anyhow::Result { ]; // current binary - call_extension_bundled_binary("sns-cli", &args) + call_extension_bundled_binary(dfx_cache_path, "sns-cli", &args) .map(|_| format!("SNS config file is valid: {}", path.display())) } From 1d6bf2f03ae31903cac7092ed7e42a49b25e866b Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Tue, 8 Aug 2023 15:14:32 +0200 Subject: [PATCH 03/10] pretty command err output + fix build process --- extensions-utils/src/dependencies/call.rs | 30 ++++------------------- extensions-utils/src/dependencies/dfx.rs | 26 ++++---------------- extensions/nns/build.rs | 26 +++++++++++++++++--- extensions/sns/build.rs | 26 +++++++++++++++++--- 4 files changed, 54 insertions(+), 54 deletions(-) diff --git a/extensions-utils/src/dependencies/call.rs b/extensions-utils/src/dependencies/call.rs index 74ca9e7..6257422 100644 --- a/extensions-utils/src/dependencies/call.rs +++ b/extensions-utils/src/dependencies/call.rs @@ -1,4 +1,4 @@ -use anyhow::anyhow; +use anyhow::Context; use fn_error_context::context; use std::{ ffi::OsStr, @@ -30,34 +30,14 @@ where ) })?; let binary_to_call = extension_dir_path.join(binary_name); - // TODO - dbg!(&binary_to_call); - std::fs::remove_file(binary_to_call.clone()).unwrap(); - panic!("trying to prettify command output, but something is not working the way I expect."); let mut command = Command::new(&binary_to_call); // If extension's dependency calls dfx; it should call dfx in this dir. command.env("PATH", dfx_cache_path.join("dfx")); command.args(args); - command + let output = command .stdin(process::Stdio::null()) .output() - .map_err(anyhow::Error::from) - .and_then(|output| -> Result { - if output.status.success() { - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) - } else { - let args: Vec<_> = command - .get_args() - .into_iter() - .map(OsStr::to_string_lossy) - .collect(); - Err(anyhow!( - "Call failed:\n{:?} {}\nStdout:\n{}\n\nStderr:\n{}", - command.get_program(), - args.join(" "), - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - )) - } - }) + .with_context(|| format!("Error executing {:#?}", command))? + .stdout; + Ok(String::from_utf8_lossy(&output).to_string()) } diff --git a/extensions-utils/src/dependencies/dfx.rs b/extensions-utils/src/dependencies/dfx.rs index 992a576..da42b09 100644 --- a/extensions-utils/src/dependencies/dfx.rs +++ b/extensions-utils/src/dependencies/dfx.rs @@ -1,5 +1,5 @@ use crate::error::dfx_executable::DfxError; -use anyhow::anyhow; +use anyhow::Context; use fn_error_context::context; use semver::Version; @@ -27,28 +27,12 @@ where // If extension's dependency calls dfx; it should call dfx in this dir. command.env("PATH", dfx_cache_path.join("dfx")); command.args(args); - command + let output = command .stdin(process::Stdio::null()) .output() - .map_err(anyhow::Error::from) - .and_then(|output| { - if output.status.success() { - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) - } else { - let args: Vec<_> = command - .get_args() - .into_iter() - .map(OsStr::to_string_lossy) - .collect(); - Err(anyhow!( - "Call failed:\n{:?} {}\nStdout:\n{}\n\nStderr:\n{}", - command.get_program(), - args.join(" "), - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - )) - } - }) + .with_context(|| format!("Error executing {:#?}", command))? + .stdout; + Ok(String::from_utf8_lossy(&output).to_string()) } pub fn replica_rev(dfx_cache_path: &Path) -> Result { diff --git a/extensions/nns/build.rs b/extensions/nns/build.rs index 43a2294..40d1e11 100644 --- a/extensions/nns/build.rs +++ b/extensions/nns/build.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::path::PathBuf; const REPLICA_REV: &str = "90e2799c255733409d0e61682685afcc2431c928"; @@ -10,9 +10,27 @@ const BINARY_DEPENDENCIES: &[(&str, &str)] = &[ ]; fn main() { - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + // keep copy of the dependency in the root of the project, so that cargo-dist will be able to package it into a tarball + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + // and also in `target/debug` or `target/release` for development purposes (e.g. cargo run) + let target_dir = manifest_dir + .parent() + .unwrap() + .parent() + .unwrap() + .join("target") + .join(std::env::var("PROFILE").unwrap()); for (binary_name, renamed_binary_name) in BINARY_DEPENDENCIES { - let destination_path = Path::new(&manifest_dir).join(renamed_binary_name); - dfx_extensions_utils::download_ic_binary(REPLICA_REV, binary_name, &destination_path); + let destination_paths = ( + manifest_dir.join(renamed_binary_name), + target_dir.join(renamed_binary_name), + ); + dbg!(&destination_paths); + dfx_extensions_utils::download_ic_binary(REPLICA_REV, binary_name, &destination_paths.0); + if destination_paths.1.exists() { + std::fs::remove_file(&destination_paths.1).unwrap(); + } + std::fs::create_dir_all(destination_paths.1.parent().unwrap()).unwrap(); + std::fs::copy(destination_paths.0, destination_paths.1).unwrap(); } } diff --git a/extensions/sns/build.rs b/extensions/sns/build.rs index 5d8a598..430f351 100644 --- a/extensions/sns/build.rs +++ b/extensions/sns/build.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::path::PathBuf; const REPLICA_REV: &str = "90e2799c255733409d0e61682685afcc2431c928"; @@ -8,9 +8,27 @@ const BINARY_DEPENDENCIES: &[(&str, &str)] = &[ ]; fn main() { - let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + // keep copy of the dependency in the root of the project, so that cargo-dist will be able to package it into a tarball + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + // and also in `target/debug` or `target/release` for development purposes (e.g. cargo run) + let target_dir = manifest_dir + .parent() + .unwrap() + .parent() + .unwrap() + .join("target") + .join(std::env::var("PROFILE").unwrap()); for (binary_name, renamed_binary_name) in BINARY_DEPENDENCIES { - let destination_path = Path::new(&manifest_dir).join(renamed_binary_name); - dfx_extensions_utils::download_ic_binary(REPLICA_REV, binary_name, &destination_path); + let destination_paths = ( + manifest_dir.join(renamed_binary_name), + target_dir.join(renamed_binary_name), + ); + dbg!(&destination_paths); + dfx_extensions_utils::download_ic_binary(REPLICA_REV, binary_name, &destination_paths.0); + if destination_paths.1.exists() { + std::fs::remove_file(&destination_paths.1).unwrap(); + } + std::fs::create_dir_all(destination_paths.1.parent().unwrap()).unwrap(); + std::fs::copy(destination_paths.0, destination_paths.1).unwrap(); } } From 515c2f2861b65f5480668d0e796d5654f768d952 Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Tue, 8 Aug 2023 20:26:53 +0200 Subject: [PATCH 04/10] Update extensions/nns/build.rs Co-authored-by: Eric Swanson <64809312+ericswanson-dfinity@users.noreply.github.com> --- extensions/nns/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/nns/build.rs b/extensions/nns/build.rs index 40d1e11..fa87a96 100644 --- a/extensions/nns/build.rs +++ b/extensions/nns/build.rs @@ -30,7 +30,7 @@ fn main() { if destination_paths.1.exists() { std::fs::remove_file(&destination_paths.1).unwrap(); } - std::fs::create_dir_all(destination_paths.1.parent().unwrap()).unwrap(); + std::fs::create_dir_all(target_dir).unwrap(); std::fs::copy(destination_paths.0, destination_paths.1).unwrap(); } } From 283849becbae59b8dcccf7ea1fadb72401403983 Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Tue, 8 Aug 2023 21:36:48 +0200 Subject: [PATCH 05/10] apply reviewer's suggestions --- extensions-utils/src/dependencies/call.rs | 14 +++++++-- extensions-utils/src/dependencies/dfx.rs | 16 ++++++++-- .../src/dependencies/download_ic_binaries.rs | 4 +-- extensions/nns/build.rs | 30 +++++++++---------- extensions/nns/src/install_nns.rs | 3 +- extensions/sns/build.rs | 30 +++++++++---------- 6 files changed, 58 insertions(+), 39 deletions(-) diff --git a/extensions-utils/src/dependencies/call.rs b/extensions-utils/src/dependencies/call.rs index 6257422..e68abf6 100644 --- a/extensions-utils/src/dependencies/call.rs +++ b/extensions-utils/src/dependencies/call.rs @@ -1,6 +1,7 @@ use anyhow::Context; use fn_error_context::context; use std::{ + env, ffi::OsStr, path::Path, process::{self, Command}, @@ -11,6 +12,8 @@ use std::{ /// # Returns /// - On success, returns stdout as a string. /// - On error, returns an error message including stdout and stderr. +/// +/// Does not stream stdout/stderr, and instead returns it after the process has exited. #[context("Calling {} CLI failed, or, it returned an error.", binary_name)] pub fn call_extension_bundled_binary( dfx_cache_path: &Path, @@ -30,9 +33,16 @@ where ) })?; let binary_to_call = extension_dir_path.join(binary_name); - let mut command = Command::new(&binary_to_call); + let mut command = Command::new(binary_to_call); // If extension's dependency calls dfx; it should call dfx in this dir. - command.env("PATH", dfx_cache_path.join("dfx")); + if let Some(path) = env::var_os("PATH") { + let mut paths = env::split_paths(&path).collect::>(); + paths.push(dfx_cache_path.to_path_buf()); + let new_path = env::join_paths(paths)?; + command.env("PATH", new_path); + } else { + command.env("PATH", dfx_cache_path); + } command.args(args); let output = command .stdin(process::Stdio::null()) diff --git a/extensions-utils/src/dependencies/dfx.rs b/extensions-utils/src/dependencies/dfx.rs index da42b09..265a4c4 100644 --- a/extensions-utils/src/dependencies/dfx.rs +++ b/extensions-utils/src/dependencies/dfx.rs @@ -3,6 +3,7 @@ use anyhow::Context; use fn_error_context::context; use semver::Version; +use std::env; use std::ffi::OsStr; use std::path::Path; use std::process::{self, Command}; @@ -12,6 +13,8 @@ use std::process::{self, Command}; /// # Returns /// - On success, returns stdout as a string. /// - On error, returns an error message including stdout and stderr. +/// +/// Does not stream stdout/stderr, and instead returns it after the process has exited. #[context("Calling {} CLI, or, it returned an error.", command)] pub fn call_dfx_bundled_binary( dfx_cache_path: &Path, @@ -23,9 +26,16 @@ where S: AsRef, { let binary = dfx_cache_path.join(command); - let mut command = Command::new(&binary); + let mut command = Command::new(binary); // If extension's dependency calls dfx; it should call dfx in this dir. - command.env("PATH", dfx_cache_path.join("dfx")); + if let Some(path) = env::var_os("PATH") { + let mut paths = env::split_paths(&path).collect::>(); + paths.push(dfx_cache_path.to_path_buf()); + let new_path = env::join_paths(paths)?; + command.env("PATH", new_path); + } else { + command.env("PATH", dfx_cache_path); + } command.args(args); let output = command .stdin(process::Stdio::null()) @@ -87,7 +97,7 @@ pub fn dfx_version(dfx_cache_path: &Path) -> Result { .map(|c| *c as char) .collect::(); if let Some(version) = version_cmd_output.split_whitespace().last() { - Version::parse(&version) // make sure the output is really a version + Version::parse(version) // make sure the output is really a version .map_err(DfxError::DfxVersionMalformed) .map(|v| v.to_string()) } else { diff --git a/extensions-utils/src/dependencies/download_ic_binaries.rs b/extensions-utils/src/dependencies/download_ic_binaries.rs index 5347323..4b1aa02 100644 --- a/extensions-utils/src/dependencies/download_ic_binaries.rs +++ b/extensions-utils/src/dependencies/download_ic_binaries.rs @@ -34,13 +34,13 @@ pub fn download_ic_binary(replica_rev: &str, binary_name: &str, destination_path let mut temp = fs::File::create(&temp_file).expect("Failed to create the file"); copy(&mut d, &mut temp).expect("Failed to copy content"); - fs::rename(temp_file, &destination_path).expect("Failed to move extension"); #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - dfx_core::fs::set_permissions(&destination_path, std::fs::Permissions::from_mode(0o500)) + dfx_core::fs::set_permissions(&temp_file, std::fs::Permissions::from_mode(0o500)) .expect("Failed to set permissions"); } + fs::rename(temp_file, destination_path).expect("Failed to move extension"); } async fn download_bytes(url: &str) -> Vec { diff --git a/extensions/nns/build.rs b/extensions/nns/build.rs index fa87a96..1e9118c 100644 --- a/extensions/nns/build.rs +++ b/extensions/nns/build.rs @@ -1,3 +1,4 @@ +use std::env; use std::path::PathBuf; const REPLICA_REV: &str = "90e2799c255733409d0e61682685afcc2431c928"; @@ -11,26 +12,25 @@ const BINARY_DEPENDENCIES: &[(&str, &str)] = &[ fn main() { // keep copy of the dependency in the root of the project, so that cargo-dist will be able to package it into a tarball - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - // and also in `target/debug` or `target/release` for development purposes (e.g. cargo run) - let target_dir = manifest_dir + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + // and also in `target/debug` or `target/release` for development purposes (e.g. cargo run), this is a bit hacky: https://github.com/rust-lang/cargo/issues/9661 + let target_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()) .parent() .unwrap() .parent() .unwrap() - .join("target") - .join(std::env::var("PROFILE").unwrap()); + .parent() + .unwrap() + .to_path_buf(); for (binary_name, renamed_binary_name) in BINARY_DEPENDENCIES { - let destination_paths = ( - manifest_dir.join(renamed_binary_name), - target_dir.join(renamed_binary_name), - ); - dbg!(&destination_paths); - dfx_extensions_utils::download_ic_binary(REPLICA_REV, binary_name, &destination_paths.0); - if destination_paths.1.exists() { - std::fs::remove_file(&destination_paths.1).unwrap(); + let bin_in_manifest_dir = manifest_dir.join(renamed_binary_name); + let bin_in_target_dir = target_dir.join(renamed_binary_name); + dbg!(&bin_in_manifest_dir, &bin_in_target_dir); + dfx_extensions_utils::download_ic_binary(REPLICA_REV, binary_name, &bin_in_manifest_dir); + if bin_in_target_dir.exists() { + std::fs::remove_file(&bin_in_target_dir).unwrap(); } - std::fs::create_dir_all(target_dir).unwrap(); - std::fs::copy(destination_paths.0, destination_paths.1).unwrap(); + std::fs::create_dir_all(bin_in_target_dir.parent().unwrap()).unwrap(); + std::fs::copy(&bin_in_manifest_dir, &bin_in_target_dir).unwrap(); } } diff --git a/extensions/nns/src/install_nns.rs b/extensions/nns/src/install_nns.rs index a4ea1bd..5a29222 100644 --- a/extensions/nns/src/install_nns.rs +++ b/extensions/nns/src/install_nns.rs @@ -430,7 +430,6 @@ pub async fn ic_nns_init(opts: &IcNnsInitOpts, dfx_cache_path: &Path) -> anyhow: args.push(subnets.into()); } call_extension_bundled_binary(dfx_cache_path, "ic-nns-init", &args) - .with_context(|| format!("Error executing `ic-nns-init` with args: {:?}", args)) } /// Sets the exchange rate between ICP and cycles. @@ -506,7 +505,7 @@ pub fn upload_nns_sns_wasms_canister_wasms(dfx_cache_path: &Path) -> anyhow::Res ]; call_extension_bundled_binary(dfx_cache_path,"sns-cli", &args) .map_err(|e| anyhow!( - "Failed to upload {upload_name} from {wasm_path:?} to the nns-sns-wasm canister by calling `sns-cli` with args {args:?}: {e}" + "Failed to upload {upload_name} from {wasm_path:?} to the nns-sns-wasm canister by calling `sns-cli`: {e}" ))?; } Ok(()) diff --git a/extensions/sns/build.rs b/extensions/sns/build.rs index 430f351..baa09d2 100644 --- a/extensions/sns/build.rs +++ b/extensions/sns/build.rs @@ -1,3 +1,4 @@ +use std::env; use std::path::PathBuf; const REPLICA_REV: &str = "90e2799c255733409d0e61682685afcc2431c928"; @@ -9,26 +10,25 @@ const BINARY_DEPENDENCIES: &[(&str, &str)] = &[ fn main() { // keep copy of the dependency in the root of the project, so that cargo-dist will be able to package it into a tarball - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - // and also in `target/debug` or `target/release` for development purposes (e.g. cargo run) - let target_dir = manifest_dir + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + // and also in `target/debug` or `target/release` for development purposes (e.g. cargo run), this is a bit hacky: https://github.com/rust-lang/cargo/issues/9661 + let target_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()) .parent() .unwrap() .parent() .unwrap() - .join("target") - .join(std::env::var("PROFILE").unwrap()); + .parent() + .unwrap() + .to_path_buf(); for (binary_name, renamed_binary_name) in BINARY_DEPENDENCIES { - let destination_paths = ( - manifest_dir.join(renamed_binary_name), - target_dir.join(renamed_binary_name), - ); - dbg!(&destination_paths); - dfx_extensions_utils::download_ic_binary(REPLICA_REV, binary_name, &destination_paths.0); - if destination_paths.1.exists() { - std::fs::remove_file(&destination_paths.1).unwrap(); + let bin_in_manifest_dir = manifest_dir.join(renamed_binary_name); + let bin_in_target_dir = target_dir.join(renamed_binary_name); + dbg!(&bin_in_manifest_dir, &bin_in_target_dir); + dfx_extensions_utils::download_ic_binary(REPLICA_REV, binary_name, &bin_in_manifest_dir); + if bin_in_target_dir.exists() { + std::fs::remove_file(&bin_in_target_dir).unwrap(); } - std::fs::create_dir_all(destination_paths.1.parent().unwrap()).unwrap(); - std::fs::copy(destination_paths.0, destination_paths.1).unwrap(); + std::fs::create_dir_all(bin_in_target_dir.parent().unwrap()).unwrap(); + std::fs::copy(&bin_in_manifest_dir, &bin_in_target_dir).unwrap(); } } From e68266e76b0432cb08efdc2d3afb214dcd85cc15 Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Wed, 9 Aug 2023 11:07:27 +0200 Subject: [PATCH 06/10] Update extensions/nns/build.rs Co-authored-by: Eric Swanson <64809312+ericswanson-dfinity@users.noreply.github.com> --- extensions/nns/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/nns/build.rs b/extensions/nns/build.rs index 1e9118c..4ef832c 100644 --- a/extensions/nns/build.rs +++ b/extensions/nns/build.rs @@ -30,7 +30,7 @@ fn main() { if bin_in_target_dir.exists() { std::fs::remove_file(&bin_in_target_dir).unwrap(); } - std::fs::create_dir_all(bin_in_target_dir.parent().unwrap()).unwrap(); + std::fs::create_dir_all(target_dir).unwrap(); std::fs::copy(&bin_in_manifest_dir, &bin_in_target_dir).unwrap(); } } From cc1a7368b1c9c89f2e42a9a091258d437325bb7e Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Wed, 9 Aug 2023 11:07:36 +0200 Subject: [PATCH 07/10] Update extensions/sns/build.rs Co-authored-by: Eric Swanson <64809312+ericswanson-dfinity@users.noreply.github.com> --- extensions/sns/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/sns/build.rs b/extensions/sns/build.rs index baa09d2..48a8e43 100644 --- a/extensions/sns/build.rs +++ b/extensions/sns/build.rs @@ -28,7 +28,7 @@ fn main() { if bin_in_target_dir.exists() { std::fs::remove_file(&bin_in_target_dir).unwrap(); } - std::fs::create_dir_all(bin_in_target_dir.parent().unwrap()).unwrap(); + std::fs::create_dir_all(target_dir).unwrap(); std::fs::copy(&bin_in_manifest_dir, &bin_in_target_dir).unwrap(); } } From 53a456f807acd5de45fad2c42c5ce190f252b0d9 Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Wed, 9 Aug 2023 11:24:43 +0200 Subject: [PATCH 08/10] Apply suggestions from code review --- extensions-utils/src/dependencies/call.rs | 2 +- extensions-utils/src/dependencies/dfx.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions-utils/src/dependencies/call.rs b/extensions-utils/src/dependencies/call.rs index e68abf6..7cfbe91 100644 --- a/extensions-utils/src/dependencies/call.rs +++ b/extensions-utils/src/dependencies/call.rs @@ -13,7 +13,7 @@ use std::{ /// - On success, returns stdout as a string. /// - On error, returns an error message including stdout and stderr. /// -/// Does not stream stdout/stderr, and instead returns it after the process has exited. +/// Does not print stdout/stderr to the console, and instead returns the output to the caller after the process has exited. #[context("Calling {} CLI failed, or, it returned an error.", binary_name)] pub fn call_extension_bundled_binary( dfx_cache_path: &Path, diff --git a/extensions-utils/src/dependencies/dfx.rs b/extensions-utils/src/dependencies/dfx.rs index 265a4c4..4e22599 100644 --- a/extensions-utils/src/dependencies/dfx.rs +++ b/extensions-utils/src/dependencies/dfx.rs @@ -14,7 +14,7 @@ use std::process::{self, Command}; /// - On success, returns stdout as a string. /// - On error, returns an error message including stdout and stderr. /// -/// Does not stream stdout/stderr, and instead returns it after the process has exited. +/// Does not print stdout/stderr to the console, and instead returns the output to the caller after the process has exited. #[context("Calling {} CLI, or, it returned an error.", command)] pub fn call_dfx_bundled_binary( dfx_cache_path: &Path, From 8b0699ad031bd9ba52a838710c90c0d37739d3de Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Wed, 9 Aug 2023 15:25:30 +0200 Subject: [PATCH 09/10] fix --- extensions/nns/build.rs | 2 +- extensions/sns/build.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions/nns/build.rs b/extensions/nns/build.rs index 4ef832c..ea2240d 100644 --- a/extensions/nns/build.rs +++ b/extensions/nns/build.rs @@ -30,7 +30,7 @@ fn main() { if bin_in_target_dir.exists() { std::fs::remove_file(&bin_in_target_dir).unwrap(); } - std::fs::create_dir_all(target_dir).unwrap(); + std::fs::create_dir_all(&target_dir).unwrap(); std::fs::copy(&bin_in_manifest_dir, &bin_in_target_dir).unwrap(); } } diff --git a/extensions/sns/build.rs b/extensions/sns/build.rs index 48a8e43..b871bd7 100644 --- a/extensions/sns/build.rs +++ b/extensions/sns/build.rs @@ -28,7 +28,7 @@ fn main() { if bin_in_target_dir.exists() { std::fs::remove_file(&bin_in_target_dir).unwrap(); } - std::fs::create_dir_all(target_dir).unwrap(); + std::fs::create_dir_all(&target_dir).unwrap(); std::fs::copy(&bin_in_manifest_dir, &bin_in_target_dir).unwrap(); } } From 7a56b034d9c71dcb0f365f34ab75502f0c3a2b84 Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Thu, 10 Aug 2023 15:55:06 +0200 Subject: [PATCH 10/10] dry --- extensions-utils/src/dependencies/call.rs | 34 ++++--------------- extensions-utils/src/dependencies/dfx.rs | 28 +++------------- extensions-utils/src/dependencies/mod.rs | 40 +++++++++++++++++++++++ extensions/nns/src/install_nns.rs | 8 ++--- extensions/sns/src/create_config.rs | 2 +- extensions/sns/src/deploy.rs | 2 +- extensions/sns/src/validate_config.rs | 2 +- 7 files changed, 59 insertions(+), 57 deletions(-) diff --git a/extensions-utils/src/dependencies/call.rs b/extensions-utils/src/dependencies/call.rs index 7cfbe91..a304b60 100644 --- a/extensions-utils/src/dependencies/call.rs +++ b/extensions-utils/src/dependencies/call.rs @@ -1,11 +1,7 @@ -use anyhow::Context; +use crate::dependencies::execute_command; +use anyhow::anyhow; use fn_error_context::context; -use std::{ - env, - ffi::OsStr, - path::Path, - process::{self, Command}, -}; +use std::{env, ffi::OsStr, path::Path}; /// Calls a binary that was delivered with an extension tarball. /// @@ -16,38 +12,22 @@ use std::{ /// Does not print stdout/stderr to the console, and instead returns the output to the caller after the process has exited. #[context("Calling {} CLI failed, or, it returned an error.", binary_name)] pub fn call_extension_bundled_binary( - dfx_cache_path: &Path, binary_name: &str, args: I, + dfx_cache_path: &Path, ) -> anyhow::Result where I: IntoIterator, S: AsRef, { let extension_binary_path = - std::env::current_exe().map_err(|e| anyhow::anyhow!("Failed to get current exe: {}", e))?; + env::current_exe().map_err(|e| anyhow!("Failed to get current exe: {}", e))?; let extension_dir_path = extension_binary_path.parent().ok_or_else(|| { - anyhow::anyhow!( + anyhow!( "Failed to locate parent of dir of executable: {}", extension_binary_path.display() ) })?; let binary_to_call = extension_dir_path.join(binary_name); - let mut command = Command::new(binary_to_call); - // If extension's dependency calls dfx; it should call dfx in this dir. - if let Some(path) = env::var_os("PATH") { - let mut paths = env::split_paths(&path).collect::>(); - paths.push(dfx_cache_path.to_path_buf()); - let new_path = env::join_paths(paths)?; - command.env("PATH", new_path); - } else { - command.env("PATH", dfx_cache_path); - } - command.args(args); - let output = command - .stdin(process::Stdio::null()) - .output() - .with_context(|| format!("Error executing {:#?}", command))? - .stdout; - Ok(String::from_utf8_lossy(&output).to_string()) + execute_command(&binary_to_call, args, dfx_cache_path) } diff --git a/extensions-utils/src/dependencies/dfx.rs b/extensions-utils/src/dependencies/dfx.rs index 4e22599..d8f9977 100644 --- a/extensions-utils/src/dependencies/dfx.rs +++ b/extensions-utils/src/dependencies/dfx.rs @@ -1,12 +1,10 @@ +use crate::dependencies::execute_command; use crate::error::dfx_executable::DfxError; -use anyhow::Context; use fn_error_context::context; use semver::Version; - -use std::env; use std::ffi::OsStr; use std::path::Path; -use std::process::{self, Command}; +use std::process::Command; /// Calls a binary from dfx cache. /// @@ -15,34 +13,18 @@ use std::process::{self, Command}; /// - On error, returns an error message including stdout and stderr. /// /// Does not print stdout/stderr to the console, and instead returns the output to the caller after the process has exited. -#[context("Calling {} CLI, or, it returned an error.", command)] +#[context("Calling {} CLI failed, or, it returned an error.", command)] pub fn call_dfx_bundled_binary( - dfx_cache_path: &Path, command: &str, args: I, + dfx_cache_path: &Path, ) -> anyhow::Result where I: IntoIterator, S: AsRef, { let binary = dfx_cache_path.join(command); - let mut command = Command::new(binary); - // If extension's dependency calls dfx; it should call dfx in this dir. - if let Some(path) = env::var_os("PATH") { - let mut paths = env::split_paths(&path).collect::>(); - paths.push(dfx_cache_path.to_path_buf()); - let new_path = env::join_paths(paths)?; - command.env("PATH", new_path); - } else { - command.env("PATH", dfx_cache_path); - } - command.args(args); - let output = command - .stdin(process::Stdio::null()) - .output() - .with_context(|| format!("Error executing {:#?}", command))? - .stdout; - Ok(String::from_utf8_lossy(&output).to_string()) + execute_command(&binary, args, dfx_cache_path) } pub fn replica_rev(dfx_cache_path: &Path) -> Result { diff --git a/extensions-utils/src/dependencies/mod.rs b/extensions-utils/src/dependencies/mod.rs index be50e04..0a53407 100644 --- a/extensions-utils/src/dependencies/mod.rs +++ b/extensions-utils/src/dependencies/mod.rs @@ -1,4 +1,44 @@ +use anyhow::anyhow; +use std::{ + env, + ffi::OsStr, + path::Path, + process::{self, Command}, +}; + pub mod call; pub mod dfx; pub mod download_ic_binaries; pub mod download_wasms; + +pub fn execute_command( + binary_path: &Path, + args: impl IntoIterator>, + dfx_cache_path: &Path, +) -> anyhow::Result { + let mut command = Command::new(binary_path); + command.args(args); + if let Some(old_path) = env::var_os("PATH") { + let mut paths = env::split_paths(&old_path).collect::>(); + paths.push(dfx_cache_path.to_path_buf()); + let new_path = env::join_paths(paths)?; + command.env("PATH", new_path); + } else { + command.env("PATH", dfx_cache_path); + } + command.stdin(process::Stdio::null()); + command.output().map_err(anyhow::Error::from).and_then( + |output| -> Result { + if output.status.success() { + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) + } else { + Err(anyhow!( + "Command failed:\n{:?}\nStdout:\n{}\n\nStderr:\n{}", + command, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )) + } + }, + ) +} diff --git a/extensions/nns/src/install_nns.rs b/extensions/nns/src/install_nns.rs index 5a29222..f67a18c 100644 --- a/extensions/nns/src/install_nns.rs +++ b/extensions/nns/src/install_nns.rs @@ -429,7 +429,7 @@ pub async fn ic_nns_init(opts: &IcNnsInitOpts, dfx_cache_path: &Path) -> anyhow: args.push("--sns-subnet".into()); args.push(subnets.into()); } - call_extension_bundled_binary(dfx_cache_path, "ic-nns-init", &args) + call_extension_bundled_binary("ic-nns-init", &args, dfx_cache_path) } /// Sets the exchange rate between ICP and cycles. @@ -452,7 +452,7 @@ pub fn set_xdr_rate(rate: u64, nns_url: &Url, dfx_cache_path: &Path) -> anyhow:: "--xdr-permyriad-per-icp", &xdr_permyriad_per_icp, ]; - call_extension_bundled_binary(dfx_cache_path, "ic-admin", args) + call_extension_bundled_binary("ic-admin", args, dfx_cache_path) .map_err(|e| anyhow!("Call to propose to set xdr rate failed: {e}")) } @@ -479,7 +479,7 @@ pub fn set_cmc_authorized_subnets( "--subnets", subnet, ]; - call_extension_bundled_binary(dfx_cache_path, "ic-admin", args) + call_extension_bundled_binary("ic-admin", args, dfx_cache_path) .map_err(|e| anyhow!("Call to propose to set authorized subnets failed: {e}")) } @@ -503,7 +503,7 @@ pub fn upload_nns_sns_wasms_canister_wasms(dfx_cache_path: &Path) -> anyhow::Res wasm_path.clone().into_os_string(), upload_name.into(), ]; - call_extension_bundled_binary(dfx_cache_path,"sns-cli", &args) + call_extension_bundled_binary("sns-cli", &args, dfx_cache_path) .map_err(|e| anyhow!( "Failed to upload {upload_name} from {wasm_path:?} to the nns-sns-wasm canister by calling `sns-cli`: {e}" ))?; diff --git a/extensions/sns/src/create_config.rs b/extensions/sns/src/create_config.rs index 67c4d2b..608bcf0 100644 --- a/extensions/sns/src/create_config.rs +++ b/extensions/sns/src/create_config.rs @@ -14,6 +14,6 @@ pub fn create_config(path: &Path, dfx_cache_path: &Path) -> anyhow::Result<()> { OsString::from(path), OsString::from("new"), ]; - call_extension_bundled_binary(dfx_cache_path, "sns-cli", &args)?; + call_extension_bundled_binary("sns-cli", &args, dfx_cache_path)?; Ok(()) } diff --git a/extensions/sns/src/deploy.rs b/extensions/sns/src/deploy.rs index 2e95952..cd87b7c 100644 --- a/extensions/sns/src/deploy.rs +++ b/extensions/sns/src/deploy.rs @@ -30,7 +30,7 @@ pub fn deploy_sns(path: &Path, dfx_cache_path: &Path) -> anyhow::Result OsString::from("--save-to"), OsString::from(canister_ids_file), ]; - call_extension_bundled_binary(dfx_cache_path, "sns-cli", &args).map(|stdout| { + call_extension_bundled_binary("sns-cli", &args, dfx_cache_path).map(|stdout| { format!( "Deployed SNS:\nSNS config: {}\nCanister ID file: {}\n\n{}", path.display(), diff --git a/extensions/sns/src/validate_config.rs b/extensions/sns/src/validate_config.rs index 02c793c..e0d603c 100644 --- a/extensions/sns/src/validate_config.rs +++ b/extensions/sns/src/validate_config.rs @@ -16,6 +16,6 @@ pub fn validate_config(path: &Path, dfx_cache_path: &Path) -> anyhow::Result