feat: dfx deps: wasm_hash_url and loose the hash check (#3510)

* feat: loose the hash check

* feat: specify wasm hash via wasm_hash_url

* feat: e2e test and error context

* fix: shellcheck

* docs: in concept

* docs: changelog

* docs: update schema

CHANGELOG.md

@@ -2,6 +2,12 @@
 
 # UNRELEASED
 
+### feat: dfx deps: wasm_hash_url and loose the hash check
+
+Providers can provide the hash through `wasm_hash_url` instead of hard coding the hash directly.
+
+If the hash of downloaded wasm doesn’t match the provided hash (`wasm_hash`, `wasm_hash_url` or read from mainnet state tree), dfx deps won’t abort. Instead, it will print a warning message.
+
 ### feat!: update `dfx cycles` commands with mainnet `cycles-ledger` canister ID
 
 The `dfx cycles` command no longer needs nor accepts the `--cycles-ledger-canister-id <canister id>` parameter.

docs/concepts/pull-dependencies.md

@@ -53,6 +53,14 @@ In most cases, the wasm module at `wasm_url` will be the same as the on-chain wa
 
 In other cases, the wasm module at `wasm_url` is not the same as the on-chain wasm module. For example, the Internet Identity canister provides Development flavor to be integrated locally. In these cases, `wasm_hash` provides the expected hash, and dfx verifies the downloaded wasm against this.
 
+### `wasm_hash_url` 
+
+A URL to get the SHA256 hash of the wasm module located at `wasm_url`.
+
+This field is optional.
+
+Aside from specifying SHA256 hash of the wasm module directly using `wasm_hash`, providers can also specify the hash with this URL. If both are defined, the `wasm_hash_url` field will be ignored.
+
 ### `dependencies`
 
 An array of Canister IDs (`Principal`) of direct dependencies.

docs/dfx-json-schema.json

@@ -982,7 +982,15 @@
         },
         "wasm_hash": {
           "title": "wasm_hash",
-          "description": "SHA256 hash of the wasm module located at wasm_url. Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url.",
+          "description": "SHA256 hash of the wasm module located at wasm_url. Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url. The hash can also be specified via a URL using the `wasm_hash_url` field. If both are defined, the `wasm_hash_url` field will be ignored.",
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "wasm_hash_url": {
+          "title": "wasm_hash_url",
+          "description": "Specify the SHA256 hash of the wasm module via this URL. Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url. The hash can also be specified directly using the `wasm_hash` field. If both are defined, the `wasm_hash_url` field will be ignored.",
           "type": [
             "string",
             "null"

e2e/tests-dfx/deps.bash

@@ -150,8 +150,6 @@ Failed to download from url: http://example.com/c.wasm."
 
   setup_onchain
 
-  # TODO: test gzipped wasm can be pulled when we have "gzip" option in dfx.json (SDK-1102)
-
   # pull canisters in app project
   cd app
   assert_file_not_exists "deps/pulled.json"
@@ -180,17 +178,16 @@ Failed to download from url: http://example.com/c.wasm."
   assert_command dfx deps pull --network local -vvv
   assert_contains "The canister wasm was found in the cache." # cache hit
 
-  # sad path 1: wasm hash doesn't match on chain
+  # warning: hash mismatch
   rm -r "${PULLED_DIR:?}/"
   cd ../onchain
   cp .dfx/local/canisters/c/c.wasm ../www/a.wasm
 
   cd ../app
-  assert_command_fail dfx deps pull --network local
-  assert_contains "Failed to pull canister $CANISTER_ID_A."
-  assert_contains "Hash mismatch."
+  assert_command dfx deps pull --network local
+  assert_contains "WARN: Canister $CANISTER_ID_A has different hash between on chain and download."
 
-  # sad path 2: url server doesn't have the file
+  # sad path: url server doesn't have the file
   rm -r "${PULLED_DIR:?}/"
   rm ../www/a.wasm
 
@@ -199,8 +196,7 @@ Failed to download from url: http://example.com/c.wasm."
   assert_contains "Failed to download from url:"
 }
 
-
-@test "dfx deps pull can check hash when dfx:wasm_hash specified" {
+@test "dfx deps pull works when wasm_hash or wasm_hash_url specified" {
   use_test_specific_cache_root # dfx deps pull will download files to cache
 
   # start a "mainnet" replica which host the onchain canisters
@@ -228,11 +224,20 @@ Failed to download from url: http://example.com/c.wasm."
   cp .dfx/local/canisters/b/b.wasm.gz ../www/b.wasm.gz
   cp .dfx/local/canisters/c/c.wasm ../www/c.wasm
 
-  CUSTOM_HASH="$(sha256sum .dfx/local/canisters/a/a.wasm | cut -d " " -f 1)"
-  jq '.canisters.a.pullable.wasm_hash="'"$CUSTOM_HASH"'"' dfx.json | sponge dfx.json
-  dfx build a # .dfx/local/canisters/a/a.wasm is replaced. The new wasm has wasm_hash defined and will be installed.
+  # A: set dfx:wasm_hash
+  CUSTOM_HASH_A="$(sha256sum .dfx/local/canisters/a/a.wasm | cut -d " " -f 1)"
+  jq '.canisters.a.pullable.wasm_hash="'"$CUSTOM_HASH_A"'"' dfx.json | sponge dfx.json
+  # B: set dfx:wasm_hash_url
+  echo -n "$(sha256sum .dfx/local/canisters/b/b.wasm.gz | cut -d " " -f 1)" > ../www/b.wasm.gz.sha256
+  jq '.canisters.b.pullable.wasm_hash_url="'"http://localhost:$E2E_WEB_SERVER_PORT/b.wasm.gz.sha256"'"' dfx.json | sponge dfx.json
+  # C: set both dfx:wasm_hash and dfx:wasm_hash_url. This should be avoided by providers.
+  CUSTOM_HASH_C="$(sha256sum .dfx/local/canisters/c/c.wasm | cut -d " " -f 1)"
+  jq '.canisters.c.pullable.wasm_hash="'"$CUSTOM_HASH_C"'"' dfx.json | sponge dfx.json
+  echo -n "$CUSTOM_HASH_C" > ../www/c.wasm.sha256
+  jq '.canisters.c.pullable.wasm_hash_url="'"http://localhost:$E2E_WEB_SERVER_PORT/c.wasm.sha256"'"' dfx.json | sponge dfx.json
+
+  dfx build
 
-  # cd ../../../
   dfx canister install a --argument 1
   dfx canister install b
   dfx canister install c --argument 3
@@ -243,18 +248,20 @@ Failed to download from url: http://example.com/c.wasm."
 
   assert_command dfx deps pull --network local -vvv
   assert_contains "Canister $CANISTER_ID_A specified a custom hash:"
+  assert_contains "Canister $CANISTER_ID_B specified a custom hash via url:"
+  assert_contains "WARN: Canister $CANISTER_ID_C specified both \`wasm_hash\` and \`wasm_hash_url\`. \`wasm_hash\` will be used."
+  assert_contains "Canister $CANISTER_ID_C specified a custom hash:"
 
-  # error case: hash mismatch
+  # warning: hash mismatch
   PULLED_DIR="$DFX_CACHE_ROOT/.cache/dfinity/pulled/"
   rm -r "${PULLED_DIR:?}/"
   cd ../onchain
   cp .dfx/local/canisters/a/a.wasm ../www/a.wasm # now the webserver has the onchain version of canister_a which won't match wasm_hash
 
   cd ../app
-  assert_command_fail dfx deps pull --network local -vvv
+  assert_command dfx deps pull --network local -vvv
   assert_contains "Canister $CANISTER_ID_A specified a custom hash:"
-  assert_contains "Failed to pull canister $CANISTER_ID_A."
-  assert_contains "Hash mismatch."
+  assert_contains "WARN: Canister $CANISTER_ID_A has different hash between on chain and download."
 }
 
 @test "dfx deps init works" {

src/dfx-core/src/config/model/dfinity.rs

@@ -170,7 +170,15 @@ pub struct Pullable {
     /// # wasm_hash
     /// SHA256 hash of the wasm module located at wasm_url.
     /// Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url.
+    /// The hash can also be specified via a URL using the `wasm_hash_url` field.
+    /// If both are defined, the `wasm_hash_url` field will be ignored.
     pub wasm_hash: Option<String>,
+    /// # wasm_hash_url
+    /// Specify the SHA256 hash of the wasm module via this URL.
+    /// Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url.
+    /// The hash can also be specified directly using the `wasm_hash` field.
+    /// If both are defined, the `wasm_hash_url` field will be ignored.
+    pub wasm_hash_url: Option<String>,
     /// # dependencies
     /// Canister IDs (Principal) of direct dependencies.
     #[schemars(with = "Vec::<String>")]

src/dfx/src/commands/deps/pull.rs

@@ -16,12 +16,13 @@ use crate::util::download_file;
 use anyhow::{anyhow, bail, Context};
 use candid::Principal;
 use clap::Parser;
+use dfx_core::config::model::dfinity::Pullable;
 use dfx_core::fs::composite::{ensure_dir_exists, ensure_parent_dir_exists};
 use fn_error_context::context;
 use ic_agent::{Agent, AgentError};
 use ic_wasm::metadata::get_metadata;
 use sha2::{Digest, Sha256};
-use slog::{error, info, trace, Logger};
+use slog::{error, info, trace, warn, Logger};
 use std::collections::{BTreeMap, BTreeSet, VecDeque};
 use std::io::Write;
 use std::path::Path;
@@ -150,24 +151,7 @@ async fn download_and_generate_pulled_canister(
     let dfx_metadata = fetch_dfx_metadata(agent, &canister_id).await?;
     let pullable = dfx_metadata.get_pullable()?;
 
-    // lookup `wasm_hash` in dfx metadata. If not available, get the hash of the on chain canister.
-    let hash_on_chain = match &pullable.wasm_hash {
-        Some(wasm_hash_str) => {
-            trace!(
-                logger,
-                "Canister {canister_id} specified a custom hash: {wasm_hash_str}"
-            );
-            hex::decode(wasm_hash_str)?
-        }
-        None => {
-            match read_state_tree_canister_module_hash(agent, canister_id).await? {
-                Some(hash_on_chain) => hash_on_chain,
-                None => {
-                    bail!("Canister {canister_id} doesn't have module hash. Perhaps it's not installed.");
-                }
-            }
-        }
-    };
+    let hash_on_chain = get_hash_on_chain(agent, logger, canister_id, pullable).await?;
 
     pulled_canister.wasm_hash = hex::encode(&hash_on_chain);
 
@@ -205,10 +189,12 @@ async fn download_and_generate_pulled_canister(
         // hash check
         let hash_download = Sha256::digest(&content);
         if hash_download.as_slice() != hash_on_chain {
-            bail!(
-                "Hash mismatch.
+            warn!(
+                logger,
+                "Canister {} has different hash between on chain and download.
 on chain: {}
 download: {}",
+                canister_id,
                 hex::encode(hash_on_chain),
                 hex::encode(hash_download.as_slice())
             );
@@ -289,6 +275,52 @@ async fn fetch_metadata(
     }
 }
 
+// Get expected hash of the canister wasm.
+// If `wasm_hash` is specified in dfx metadata, use it.
+// If `wasm_hash_url` is specified in dfx metadata, download the hash from the url.
+// Otherwise, get the hash of the on chain canister.
+async fn get_hash_on_chain(
+    agent: &Agent,
+    logger: &Logger,
+    canister_id: Principal,
+    pullable: &Pullable,
+) -> DfxResult<Vec<u8>> {
+    if pullable.wasm_hash.is_some() && pullable.wasm_hash_url.is_some() {
+        warn!(logger, "Canister {canister_id} specified both `wasm_hash` and `wasm_hash_url`. `wasm_hash` will be used.");
+    };
+    if let Some(wasm_hash_str) = &pullable.wasm_hash {
+        trace!(
+            logger,
+            "Canister {canister_id} specified a custom hash: {wasm_hash_str}"
+        );
+        Ok(hex::decode(wasm_hash_str)
+            .with_context(|| format!("Failed to decode {wasm_hash_str} as sha256 hash."))?)
+    } else if let Some(wasm_hash_url) = &pullable.wasm_hash_url {
+        trace!(
+            logger,
+            "Canister {canister_id} specified a custom hash via url: {wasm_hash_url}"
+        );
+        let wasm_hash_url = reqwest::Url::parse(wasm_hash_url)
+            .with_context(|| format!("{wasm_hash_url} is not a valid URL."))?;
+        let wasm_hash_content = download_file(&wasm_hash_url)
+            .await
+            .with_context(|| format!("Failed to download wasm_hash from {wasm_hash_url}."))?;
+        let wasm_hash_encoded = String::from_utf8(wasm_hash_content)
+            .with_context(|| format!("Content from {wasm_hash_url} is not valid text."))?;
+        Ok(hex::decode(&wasm_hash_encoded)
+            .with_context(|| format!("Failed to decode {wasm_hash_encoded} as sha256 hash."))?)
+    } else {
+        match read_state_tree_canister_module_hash(agent, canister_id).await? {
+            Some(hash_on_chain) => Ok(hash_on_chain),
+            None => {
+                bail!(
+                    "Canister {canister_id} doesn't have module hash. Perhaps it's not installed."
+                );
+            }
+        }
+    }
+}
+
 #[context("Failed to write to a tempfile then rename it to {}", path.display())]
 fn write_to_tempfile_then_rename(content: &[u8], path: &Path) -> DfxResult {
     assert!(path.is_absolute());