diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fbc9e1e30..e822d103d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ # UNRELEASED +### feat: dfx deps: wasm_hash_url and loose the hash check + +Providers can provide the hash through `wasm_hash_url` instead of hard coding the hash directly. + +If the hash of downloaded wasm doesn’t match the provided hash (`wasm_hash`, `wasm_hash_url` or read from mainnet state tree), dfx deps won’t abort. Instead, it will print a warning message. + ### feat!: update `dfx cycles` commands with mainnet `cycles-ledger` canister ID The `dfx cycles` command no longer needs nor accepts the `--cycles-ledger-canister-id ` parameter. diff --git a/docs/concepts/pull-dependencies.md b/docs/concepts/pull-dependencies.md index d48dc0699c..70a077c34f 100644 --- a/docs/concepts/pull-dependencies.md +++ b/docs/concepts/pull-dependencies.md @@ -53,6 +53,14 @@ In most cases, the wasm module at `wasm_url` will be the same as the on-chain wa In other cases, the wasm module at `wasm_url` is not the same as the on-chain wasm module. For example, the Internet Identity canister provides Development flavor to be integrated locally. In these cases, `wasm_hash` provides the expected hash, and dfx verifies the downloaded wasm against this. +### `wasm_hash_url` + +A URL to get the SHA256 hash of the wasm module located at `wasm_url`. + +This field is optional. + +Aside from specifying SHA256 hash of the wasm module directly using `wasm_hash`, providers can also specify the hash with this URL. If both are defined, the `wasm_hash_url` field will be ignored. + ### `dependencies` An array of Canister IDs (`Principal`) of direct dependencies. diff --git a/docs/dfx-json-schema.json b/docs/dfx-json-schema.json index 40f4e890ac..728f48d0dc 100644 --- a/docs/dfx-json-schema.json +++ b/docs/dfx-json-schema.json @@ -982,7 +982,15 @@ }, "wasm_hash": { "title": "wasm_hash", - "description": "SHA256 hash of the wasm module located at wasm_url. Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url.", + "description": "SHA256 hash of the wasm module located at wasm_url. Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url. The hash can also be specified via a URL using the `wasm_hash_url` field. If both are defined, the `wasm_hash_url` field will be ignored.", + "type": [ + "string", + "null" + ] + }, + "wasm_hash_url": { + "title": "wasm_hash_url", + "description": "Specify the SHA256 hash of the wasm module via this URL. Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url. The hash can also be specified directly using the `wasm_hash` field. If both are defined, the `wasm_hash_url` field will be ignored.", "type": [ "string", "null" diff --git a/e2e/tests-dfx/deps.bash b/e2e/tests-dfx/deps.bash index 5ef85acc16..c56aa217be 100644 --- a/e2e/tests-dfx/deps.bash +++ b/e2e/tests-dfx/deps.bash @@ -150,8 +150,6 @@ Failed to download from url: http://example.com/c.wasm." setup_onchain - # TODO: test gzipped wasm can be pulled when we have "gzip" option in dfx.json (SDK-1102) - # pull canisters in app project cd app assert_file_not_exists "deps/pulled.json" @@ -180,17 +178,16 @@ Failed to download from url: http://example.com/c.wasm." assert_command dfx deps pull --network local -vvv assert_contains "The canister wasm was found in the cache." # cache hit - # sad path 1: wasm hash doesn't match on chain + # warning: hash mismatch rm -r "${PULLED_DIR:?}/" cd ../onchain cp .dfx/local/canisters/c/c.wasm ../www/a.wasm cd ../app - assert_command_fail dfx deps pull --network local - assert_contains "Failed to pull canister $CANISTER_ID_A." - assert_contains "Hash mismatch." + assert_command dfx deps pull --network local + assert_contains "WARN: Canister $CANISTER_ID_A has different hash between on chain and download." - # sad path 2: url server doesn't have the file + # sad path: url server doesn't have the file rm -r "${PULLED_DIR:?}/" rm ../www/a.wasm @@ -199,8 +196,7 @@ Failed to download from url: http://example.com/c.wasm." assert_contains "Failed to download from url:" } - -@test "dfx deps pull can check hash when dfx:wasm_hash specified" { +@test "dfx deps pull works when wasm_hash or wasm_hash_url specified" { use_test_specific_cache_root # dfx deps pull will download files to cache # start a "mainnet" replica which host the onchain canisters @@ -228,11 +224,20 @@ Failed to download from url: http://example.com/c.wasm." cp .dfx/local/canisters/b/b.wasm.gz ../www/b.wasm.gz cp .dfx/local/canisters/c/c.wasm ../www/c.wasm - CUSTOM_HASH="$(sha256sum .dfx/local/canisters/a/a.wasm | cut -d " " -f 1)" - jq '.canisters.a.pullable.wasm_hash="'"$CUSTOM_HASH"'"' dfx.json | sponge dfx.json - dfx build a # .dfx/local/canisters/a/a.wasm is replaced. The new wasm has wasm_hash defined and will be installed. + # A: set dfx:wasm_hash + CUSTOM_HASH_A="$(sha256sum .dfx/local/canisters/a/a.wasm | cut -d " " -f 1)" + jq '.canisters.a.pullable.wasm_hash="'"$CUSTOM_HASH_A"'"' dfx.json | sponge dfx.json + # B: set dfx:wasm_hash_url + echo -n "$(sha256sum .dfx/local/canisters/b/b.wasm.gz | cut -d " " -f 1)" > ../www/b.wasm.gz.sha256 + jq '.canisters.b.pullable.wasm_hash_url="'"http://localhost:$E2E_WEB_SERVER_PORT/b.wasm.gz.sha256"'"' dfx.json | sponge dfx.json + # C: set both dfx:wasm_hash and dfx:wasm_hash_url. This should be avoided by providers. + CUSTOM_HASH_C="$(sha256sum .dfx/local/canisters/c/c.wasm | cut -d " " -f 1)" + jq '.canisters.c.pullable.wasm_hash="'"$CUSTOM_HASH_C"'"' dfx.json | sponge dfx.json + echo -n "$CUSTOM_HASH_C" > ../www/c.wasm.sha256 + jq '.canisters.c.pullable.wasm_hash_url="'"http://localhost:$E2E_WEB_SERVER_PORT/c.wasm.sha256"'"' dfx.json | sponge dfx.json + + dfx build - # cd ../../../ dfx canister install a --argument 1 dfx canister install b dfx canister install c --argument 3 @@ -243,18 +248,20 @@ Failed to download from url: http://example.com/c.wasm." assert_command dfx deps pull --network local -vvv assert_contains "Canister $CANISTER_ID_A specified a custom hash:" + assert_contains "Canister $CANISTER_ID_B specified a custom hash via url:" + assert_contains "WARN: Canister $CANISTER_ID_C specified both \`wasm_hash\` and \`wasm_hash_url\`. \`wasm_hash\` will be used." + assert_contains "Canister $CANISTER_ID_C specified a custom hash:" - # error case: hash mismatch + # warning: hash mismatch PULLED_DIR="$DFX_CACHE_ROOT/.cache/dfinity/pulled/" rm -r "${PULLED_DIR:?}/" cd ../onchain cp .dfx/local/canisters/a/a.wasm ../www/a.wasm # now the webserver has the onchain version of canister_a which won't match wasm_hash cd ../app - assert_command_fail dfx deps pull --network local -vvv + assert_command dfx deps pull --network local -vvv assert_contains "Canister $CANISTER_ID_A specified a custom hash:" - assert_contains "Failed to pull canister $CANISTER_ID_A." - assert_contains "Hash mismatch." + assert_contains "WARN: Canister $CANISTER_ID_A has different hash between on chain and download." } @test "dfx deps init works" { diff --git a/src/dfx-core/src/config/model/dfinity.rs b/src/dfx-core/src/config/model/dfinity.rs index f4ffd78cc6..e829c14f7f 100644 --- a/src/dfx-core/src/config/model/dfinity.rs +++ b/src/dfx-core/src/config/model/dfinity.rs @@ -170,7 +170,15 @@ pub struct Pullable { /// # wasm_hash /// SHA256 hash of the wasm module located at wasm_url. /// Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url. + /// The hash can also be specified via a URL using the `wasm_hash_url` field. + /// If both are defined, the `wasm_hash_url` field will be ignored. pub wasm_hash: Option, + /// # wasm_hash_url + /// Specify the SHA256 hash of the wasm module via this URL. + /// Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url. + /// The hash can also be specified directly using the `wasm_hash` field. + /// If both are defined, the `wasm_hash_url` field will be ignored. + pub wasm_hash_url: Option, /// # dependencies /// Canister IDs (Principal) of direct dependencies. #[schemars(with = "Vec::")] diff --git a/src/dfx/src/commands/deps/pull.rs b/src/dfx/src/commands/deps/pull.rs index 1f53471207..db92b3cb6f 100644 --- a/src/dfx/src/commands/deps/pull.rs +++ b/src/dfx/src/commands/deps/pull.rs @@ -16,12 +16,13 @@ use crate::util::download_file; use anyhow::{anyhow, bail, Context}; use candid::Principal; use clap::Parser; +use dfx_core::config::model::dfinity::Pullable; use dfx_core::fs::composite::{ensure_dir_exists, ensure_parent_dir_exists}; use fn_error_context::context; use ic_agent::{Agent, AgentError}; use ic_wasm::metadata::get_metadata; use sha2::{Digest, Sha256}; -use slog::{error, info, trace, Logger}; +use slog::{error, info, trace, warn, Logger}; use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::io::Write; use std::path::Path; @@ -150,24 +151,7 @@ async fn download_and_generate_pulled_canister( let dfx_metadata = fetch_dfx_metadata(agent, &canister_id).await?; let pullable = dfx_metadata.get_pullable()?; - // lookup `wasm_hash` in dfx metadata. If not available, get the hash of the on chain canister. - let hash_on_chain = match &pullable.wasm_hash { - Some(wasm_hash_str) => { - trace!( - logger, - "Canister {canister_id} specified a custom hash: {wasm_hash_str}" - ); - hex::decode(wasm_hash_str)? - } - None => { - match read_state_tree_canister_module_hash(agent, canister_id).await? { - Some(hash_on_chain) => hash_on_chain, - None => { - bail!("Canister {canister_id} doesn't have module hash. Perhaps it's not installed."); - } - } - } - }; + let hash_on_chain = get_hash_on_chain(agent, logger, canister_id, pullable).await?; pulled_canister.wasm_hash = hex::encode(&hash_on_chain); @@ -205,10 +189,12 @@ async fn download_and_generate_pulled_canister( // hash check let hash_download = Sha256::digest(&content); if hash_download.as_slice() != hash_on_chain { - bail!( - "Hash mismatch. + warn!( + logger, + "Canister {} has different hash between on chain and download. on chain: {} download: {}", + canister_id, hex::encode(hash_on_chain), hex::encode(hash_download.as_slice()) ); @@ -289,6 +275,52 @@ async fn fetch_metadata( } } +// Get expected hash of the canister wasm. +// If `wasm_hash` is specified in dfx metadata, use it. +// If `wasm_hash_url` is specified in dfx metadata, download the hash from the url. +// Otherwise, get the hash of the on chain canister. +async fn get_hash_on_chain( + agent: &Agent, + logger: &Logger, + canister_id: Principal, + pullable: &Pullable, +) -> DfxResult> { + if pullable.wasm_hash.is_some() && pullable.wasm_hash_url.is_some() { + warn!(logger, "Canister {canister_id} specified both `wasm_hash` and `wasm_hash_url`. `wasm_hash` will be used."); + }; + if let Some(wasm_hash_str) = &pullable.wasm_hash { + trace!( + logger, + "Canister {canister_id} specified a custom hash: {wasm_hash_str}" + ); + Ok(hex::decode(wasm_hash_str) + .with_context(|| format!("Failed to decode {wasm_hash_str} as sha256 hash."))?) + } else if let Some(wasm_hash_url) = &pullable.wasm_hash_url { + trace!( + logger, + "Canister {canister_id} specified a custom hash via url: {wasm_hash_url}" + ); + let wasm_hash_url = reqwest::Url::parse(wasm_hash_url) + .with_context(|| format!("{wasm_hash_url} is not a valid URL."))?; + let wasm_hash_content = download_file(&wasm_hash_url) + .await + .with_context(|| format!("Failed to download wasm_hash from {wasm_hash_url}."))?; + let wasm_hash_encoded = String::from_utf8(wasm_hash_content) + .with_context(|| format!("Content from {wasm_hash_url} is not valid text."))?; + Ok(hex::decode(&wasm_hash_encoded) + .with_context(|| format!("Failed to decode {wasm_hash_encoded} as sha256 hash."))?) + } else { + match read_state_tree_canister_module_hash(agent, canister_id).await? { + Some(hash_on_chain) => Ok(hash_on_chain), + None => { + bail!( + "Canister {canister_id} doesn't have module hash. Perhaps it's not installed." + ); + } + } + } +} + #[context("Failed to write to a tempfile then rename it to {}", path.display())] fn write_to_tempfile_then_rename(content: &[u8], path: &Path) -> DfxResult { assert!(path.is_absolute());