applying reviewer's suggestions

extensions-utils/Cargo.toml

@@ -17,6 +17,7 @@ reqwest.workspace = true
 dfx-core.workspace = true
 
 anyhow.workspace = true
+backoff = { version = "0.4.0", features = [ "futures", "tokio" ] }
 flate2 = { version = "1.0.25", default-features = false, features = ["zlib-ng"] }
 fn-error-context.workspace = true
 futures-util.workspace = true
@@ -31,4 +32,5 @@ slog-term = "2.9.0"
 slog.workspace = true
 tempfile.workspace = true
 thiserror = "1.0.40"
+tokio.workspace = true
 url.workspace = true

extensions-utils/src/dependencies/call.rs

@@ -0,0 +1,63 @@
+use anyhow::anyhow;
+use fn_error_context::context;
+use std::{
+    ffi::OsStr,
+    path::Path,
+    process::{self, Command},
+};
+
+/// Calls a binary that was delivered with an extension tarball.
+///
+/// # Returns
+/// - On success, returns stdout as a string.
+/// - On error, returns an error message including stdout and stderr.
+#[context("Calling {} CLI failed, or, it returned an error.", binary_name)]
+pub fn call_extension_bundled_binary<S, I>(
+    dfx_cache_path: &Path,
+    binary_name: &str,
+    args: I,
+) -> anyhow::Result<String>
+where
+    I: IntoIterator<Item = S>,
+    S: AsRef<OsStr>,
+{
+    let extension_binary_path =
+        std::env::current_exe().map_err(|e| anyhow::anyhow!("Failed to get current exe: {}", e))?;
+    let extension_dir_path = extension_binary_path.parent().ok_or_else(|| {
+        anyhow::anyhow!(
+            "Failed to locate parent of dir of executable: {}",
+            extension_binary_path.display()
+        )
+    })?;
+    let binary_to_call = extension_dir_path.join(binary_name);
+    // TODO
+    dbg!(&binary_to_call);
+    std::fs::remove_file(binary_to_call.clone()).unwrap();
+    panic!("trying to prettify command output, but something is not working the way I expect.");
+    let mut command = Command::new(&binary_to_call);
+    // If extension's dependency calls dfx; it should call dfx in this dir.
+    command.env("PATH", dfx_cache_path.join("dfx"));
+    command.args(args);
+    command
+        .stdin(process::Stdio::null())
+        .output()
+        .map_err(anyhow::Error::from)
+        .and_then(|output| -> Result<String, anyhow::Error> {
+            if output.status.success() {
+                Ok(String::from_utf8_lossy(&output.stdout).into_owned())
+            } else {
+                let args: Vec<_> = command
+                    .get_args()
+                    .into_iter()
+                    .map(OsStr::to_string_lossy)
+                    .collect();
+                Err(anyhow!(
+                    "Call failed:\n{:?} {}\nStdout:\n{}\n\nStderr:\n{}",
+                    command.get_program(),
+                    args.join(" "),
+                    String::from_utf8_lossy(&output.stdout),
+                    String::from_utf8_lossy(&output.stderr)
+                ))
+            }
+        })
+}

extensions-utils/src/dfx.rs → extensions-utils/src/dependencies/dfx.rs

@@ -23,64 +23,10 @@ where
     S: AsRef<OsStr>,
 {
     let binary = dfx_cache_path.join(command);
-
     let mut command = Command::new(&binary);
+    // If extension's dependency calls dfx; it should call dfx in this dir.
+    command.env("PATH", dfx_cache_path.join("dfx"));
     command.args(args);
-    // The sns command line tool itself calls dfx; it should call this dfx.
-    // The sns command line tool should not rely on commands not packaged with dfx.
-    // The same applies to other bundled binaries.
-    command.env("PATH", binary.parent().unwrap_or_else(|| Path::new(".")));
-    command
-        .stdin(process::Stdio::null())
-        .output()
-        .map_err(anyhow::Error::from)
-        .and_then(|output| {
-            if output.status.success() {
-                Ok(String::from_utf8_lossy(&output.stdout).into_owned())
-            } else {
-                let args: Vec<_> = command
-                    .get_args()
-                    .into_iter()
-                    .map(OsStr::to_string_lossy)
-                    .collect();
-                Err(anyhow!(
-                    "Call failed:\n{:?} {}\nStdout:\n{}\n\nStderr:\n{}",
-                    command.get_program(),
-                    args.join(" "),
-                    String::from_utf8_lossy(&output.stdout),
-                    String::from_utf8_lossy(&output.stderr)
-                ))
-            }
-        })
-}
-
-/// Calls a binary that was delivered with an extension tarball.
-///
-/// # Returns
-/// - On success, returns stdout as a string.
-/// - On error, returns an error message including stdout and stderr.
-#[context("Calling {} CLI failed, or, it returned an error.", binary_name)]
-pub fn call_extension_bundled_binary<S, I>(binary_name: &str, args: I) -> anyhow::Result<String>
-where
-    I: IntoIterator<Item = S>,
-    S: AsRef<OsStr>,
-{
-    let extension_binary_path =
-        std::env::current_exe().map_err(|e| anyhow::anyhow!("Failed to get current exe: {}", e))?;
-    let extension_dir_path = extension_binary_path
-        .parent()
-        .unwrap_or_else(|| Path::new("."));
-    let binary_to_call = extension_dir_path.join(binary_name);
-
-    let mut command = Command::new(&binary_to_call);
-    command.args(args);
-    // The sns command line tool itself calls dfx; it should call this dfx.
-    // The sns command line tool should not rely on commands not packaged with dfx.
-    // The same applies to other bundled binaries.
-    command.env(
-        "PATH",
-        binary_to_call.parent().unwrap_or_else(|| Path::new(".")),
-    );
     command
         .stdin(process::Stdio::null())
         .output()

extensions-utils/src/dependencies/download_ic_binaries.rs

@@ -0,0 +1,64 @@
+use backoff::future::retry;
+use backoff::ExponentialBackoffBuilder;
+use flate2::read::GzDecoder;
+use std::path::Path;
+use std::time::Duration;
+use std::{fs, io::copy};
+use tokio::runtime::Runtime;
+
+pub fn download_ic_binary(replica_rev: &str, binary_name: &str, destination_path: &Path) {
+    let arch = match std::env::consts::ARCH {
+        "x86_64" => "x86_64",
+        "aarch64" => "x86_64", // let's rely on rosetta2 for now, since ic binaiers are not available for arm64
+        _ => panic!("Unsupported architecture"),
+    };
+    let os = match std::env::consts::OS {
+        "macos" => "darwin",
+        "linux" => "linux",
+        // "windows" => "windows", // unsupported till dfx supports windows
+        _ => panic!("Unsupported OS"),
+    };
+
+    let url = format!(
+        "https://download.dfinity.systems/ic/{replica_rev}/openssl-static-binaries/{arch}-{os}/{binary_name}.gz",
+        arch = arch,
+        os = os,
+        binary_name = binary_name,
+    );
+    println!("Downloading {}", url);
+
+    let bytes = Runtime::new().unwrap().block_on(download_bytes(&url));
+    let mut d = GzDecoder::new(&*bytes);
+    let tempdir = tempfile::tempdir().expect("Failed to create temp dir");
+    let temp_file = tempdir.path().join(binary_name);
+    let mut temp = fs::File::create(&temp_file).expect("Failed to create the file");
+    copy(&mut d, &mut temp).expect("Failed to copy content");
+
+    fs::rename(temp_file, &destination_path).expect("Failed to move extension");
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        dfx_core::fs::set_permissions(&destination_path, std::fs::Permissions::from_mode(0o500))
+            .expect("Failed to set permissions");
+    }
+}
+
+async fn download_bytes(url: &str) -> Vec<u8> {
+    let retry_policy = ExponentialBackoffBuilder::new()
+        .with_initial_interval(Duration::from_secs(1))
+        .with_max_interval(Duration::from_secs(16))
+        .with_multiplier(2.0)
+        .with_max_elapsed_time(Some(Duration::from_secs(300)))
+        .build();
+    let resp = retry(retry_policy, || async {
+        match reqwest::get(url).await {
+            Ok(response) => Ok(response),
+            Err(err) => Err(backoff::Error::transient(err)),
+        }
+    })
+    .await
+    .unwrap();
+
+    let bytes = resp.bytes().await.expect("Failed to read response");
+    bytes.to_vec()
+}

extensions-utils/src/dependencies/mod.rs

@@ -0,0 +1,4 @@
+pub mod call;
+pub mod dfx;
+pub mod download_ic_binaries;
+pub mod download_wasms;

extensions-utils/src/lib.rs

@@ -1,24 +1,23 @@
 //! Library for calling bundled command line tools.
 
-mod dfx;
-pub mod download_dependencies;
-mod download_wasms;
+pub mod dependencies;
 mod error;
 mod logger;
 mod project;
 
-pub use dfx::{
-    call_dfx_bundled_binary, call_extension_bundled_binary, dfx_version, replica_rev,
-    webserver_port,
+pub use dependencies::{
+    call::call_extension_bundled_binary,
+    dfx::{call_dfx_bundled_binary, dfx_version, replica_rev, webserver_port},
+    download_ic_binaries::download_ic_binary,
+    download_wasms::{
+        download_ic_repo_wasm,
+        nns::{
+            download_nns_wasms, nns_wasm_dir, IcNnsInitCanister, StandardCanister,
+            ED25519_TEST_ACCOUNT, NNS_CORE, NNS_FRONTEND, NNS_SNS_WASM, SECP256K1_TEST_ACCOUNT,
+        },
+        sns::{download_sns_wasms, SnsCanisterInstallation, SNS_CANISTERS},
+    },
 };
-pub use download_wasms::download_ic_repo_wasm;
-pub use download_wasms::nns::download_nns_wasms;
-pub use download_wasms::nns::{
-    nns_wasm_dir, IcNnsInitCanister, StandardCanister, ED25519_TEST_ACCOUNT, NNS_CORE,
-    NNS_FRONTEND, NNS_SNS_WASM, SECP256K1_TEST_ACCOUNT,
-};
-pub use download_wasms::sns::SnsCanisterInstallation;
-pub use download_wasms::sns::{download_sns_wasms, SNS_CANISTERS};
 pub use logger::new_logger;
 pub use project::import::import_canister_definitions;
 pub use project::network_mappings::get_network_mappings;

extensions/nns/build.rs

@@ -1,3 +1,5 @@
+use std::path::Path;
+
 const REPLICA_REV: &str = "90e2799c255733409d0e61682685afcc2431c928";
 
 const BINARY_DEPENDENCIES: &[(&str, &str)] = &[
@@ -8,11 +10,9 @@ const BINARY_DEPENDENCIES: &[(&str, &str)] = &[
 ];
 
 fn main() {
+    let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap();
     for (binary_name, renamed_binary_name) in BINARY_DEPENDENCIES {
-        dfx_extensions_utils::download_dependencies::download_ic_binary(
-            REPLICA_REV,
-            binary_name,
-            renamed_binary_name,
-        );
+        let destination_path = Path::new(&manifest_dir).join(renamed_binary_name);
+        dfx_extensions_utils::download_ic_binary(REPLICA_REV, binary_name, &destination_path);
     }
 }