feat: facade pull ICP, ckBTC, ckETH ledger canisters

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 # UNRELEASED
 
+### feat: facade pull ICP, ckBTC, ckETH ledger canisters
+
+The ledger canisters can be pulled even though they are not really "pullable".
+The metadata like wasm_url and init_guide are hardcoded inside `dfx deps pull` logic.
+
+- ICP ledger: `ryjl3-tyaaa-aaaaa-aaaba-cai`
+- ckBTC ledger: `mxzaz-hqaaa-aaaar-qaada-cai`
+- ckETH ledger: `ss2fx-dyaaa-aaaar-qacoq-cai`
+
 ### chore: update agent version in frontend templates, and include `resolve.dedupe` in Vite config
 
 ### chore: improve error message when trying to use the local replica when it is not running

e2e/tests-dfx/deps.bash

@@ -673,3 +673,172 @@ Installing canister: $CANISTER_ID_C (dep_c)"
   # this command will fail if the pulled.json is not correct
   assert_command dfx deps init
 }
+
+@test "dfx deps can facade pull ICP ledger" {
+  use_test_specific_cache_root # dfx deps pull will download files to cache
+
+  dfx_new
+  jq '.canisters.e2e_project_backend.dependencies=["icp_ledger"]' dfx.json | sponge dfx.json
+  jq '.canisters.icp_ledger.type="pull"' dfx.json | sponge dfx.json
+  jq '.canisters.icp_ledger.id="ryjl3-tyaaa-aaaaa-aaaba-cai"' dfx.json | sponge dfx.json
+
+  dfx_start
+  assert_command dfx deps pull --network local
+  assert_contains "Using facade dependencies for canister ryjl3-tyaaa-aaaaa-aaaba-cai."
+
+  dfx identity new --storage-mode plaintext minter
+  assert_command_fail dfx deps init icp_ledger
+  assert_contains "1. Create a 'minter' identity: dfx identity new minter
+2. Run the following multi-line command:"
+
+  assert_command dfx deps init ryjl3-tyaaa-aaaaa-aaaba-cai --argument "(variant { 
+    Init = record {
+        minting_account = \"$(dfx --identity minter ledger account-id)\";
+        initial_values = vec {};
+        send_whitelist = vec {};
+        transfer_fee = opt record { e8s = 10_000 : nat64; };
+        token_symbol = opt \"LICP\";
+        token_name = opt \"Local ICP\"; 
+    }
+})"
+
+  assert_command dfx deps deploy
+
+  # Can mint tokens (transfer from minting_account)
+  assert_command dfx --identity minter canister call icp_ledger icrc1_transfer "(
+  record {
+    to = record {
+      owner = principal \"$(dfx --identity default identity get-principal)\";
+    };
+    amount = 1_000_000 : nat;
+  },
+)"
+
+  assert_command dfx canister call icp_ledger icrc1_balance_of "(
+  record {
+    owner = principal \"$(dfx --identity default identity get-principal)\";
+  },
+)"
+  assert_eq "(1_000_000 : nat)"
+}
+
+@test "dfx deps can facade pull ckBTC ledger" {
+  [[ "$USE_POCKETIC" ]] && skip "skipped for pocketic which doesn't have ckBTC subnet"
+
+  use_test_specific_cache_root # dfx deps pull will download files to cache
+
+  dfx_new
+  jq '.canisters.e2e_project_backend.dependencies=["ckbtc_ledger"]' dfx.json | sponge dfx.json
+  jq '.canisters.ckbtc_ledger.type="pull"' dfx.json | sponge dfx.json
+  jq '.canisters.ckbtc_ledger.id="mxzaz-hqaaa-aaaar-qaada-cai"' dfx.json | sponge dfx.json
+
+  dfx_start
+  assert_command dfx deps pull --network local
+  assert_contains "Using facade dependencies for canister mxzaz-hqaaa-aaaar-qaada-cai."
+
+  dfx identity new --storage-mode plaintext minter
+  assert_command_fail dfx deps init ckbtc_ledger
+  assert_contains "1. Create a 'minter' identity: dfx identity new minter
+2. Run the following multi-line command:"
+
+  assert_command dfx deps init mxzaz-hqaaa-aaaar-qaada-cai --argument "(variant {
+    Init = record {
+        minting_account = record { owner = principal \"$(dfx --identity minter identity get-principal)\"; };
+        transfer_fee = 10;
+        token_symbol = \"ckBTC\";
+        token_name = \"ckBTC\";
+        metadata = vec {};
+        initial_balances = vec {};
+        max_memo_length = opt 80;
+        archive_options = record {
+            num_blocks_to_archive = 1000;
+            trigger_threshold = 2000;
+            max_message_size_bytes = null;
+            cycles_for_archive_creation = opt 100_000_000_000_000;
+            node_max_memory_size_bytes = opt 3_221_225_472;
+            controller_id = principal \"2vxsx-fae\"
+        }
+    }
+})"
+
+  assert_command dfx deps deploy
+
+  # Can mint tokens (transfer from minting_account)
+  assert_command dfx --identity minter canister call ckbtc_ledger icrc1_transfer "(
+  record {
+    to = record {
+      owner = principal \"$(dfx --identity default identity get-principal)\";
+    };
+    amount = 1_000_000 : nat;
+  },
+)"
+
+  assert_command dfx canister call ckbtc_ledger icrc1_balance_of "(
+  record {
+    owner = principal \"$(dfx --identity default identity get-principal)\";
+  },
+)"
+  assert_eq "(1_000_000 : nat)"
+}
+
+
+@test "dfx deps can facade pull ckETH ledger" {
+  [[ "$USE_POCKETIC" ]] && skip "skipped for pocketic which doesn't have ckETH subnet"
+
+  use_test_specific_cache_root # dfx deps pull will download files to cache
+
+  dfx_new
+  jq '.canisters.e2e_project_backend.dependencies=["cketh_ledger"]' dfx.json | sponge dfx.json
+  jq '.canisters.cketh_ledger.type="pull"' dfx.json | sponge dfx.json
+  jq '.canisters.cketh_ledger.id="ss2fx-dyaaa-aaaar-qacoq-cai"' dfx.json | sponge dfx.json
+
+  dfx_start
+  assert_command dfx deps pull --network local
+  assert_contains "Using facade dependencies for canister ss2fx-dyaaa-aaaar-qacoq-cai."
+
+  dfx identity new --storage-mode plaintext minter
+  assert_command_fail dfx deps init cketh_ledger
+  assert_contains "1. Create a 'minter' identity: dfx identity new minter
+2. Run the following multi-line command:"
+
+  assert_command dfx deps init ss2fx-dyaaa-aaaar-qacoq-cai --argument "(variant {
+    Init = record {
+        minting_account = record { owner = principal \"$(dfx --identity minter identity get-principal)\"; };
+        decimals = opt 18;
+        max_memo_length = opt 80;
+        transfer_fee = 2_000_000_000_000;
+        token_symbol = \"ckETH\";
+        token_name = \"ckETH\";
+        feature_flags = opt record { icrc2 = true };
+        metadata = vec {};
+        initial_balances = vec {};
+        archive_options = record {
+            num_blocks_to_archive = 1000;
+            trigger_threshold = 2000;
+            max_message_size_bytes = null;
+            cycles_for_archive_creation = opt 100_000_000_000_000;
+            node_max_memory_size_bytes = opt 3_221_225_472;
+            controller_id = principal \"2vxsx-fae\"
+        }
+    }
+})"
+
+  assert_command dfx deps deploy
+
+  # Can mint tokens (transfer from minting_account)
+  assert_command dfx --identity minter canister call cketh_ledger icrc1_transfer "(
+  record {
+    to = record {
+      owner = principal \"$(dfx --identity default identity get-principal)\";
+    };
+    amount = 1_000_000 : nat;
+  },
+)"
+
+  assert_command dfx canister call cketh_ledger icrc1_balance_of "(
+  record {
+    owner = principal \"$(dfx --identity default identity get-principal)\";
+  },
+)"
+  assert_eq "(1_000_000 : nat)"
+}

src/dfx/src/lib/deps/mod.rs

@@ -22,7 +22,7 @@ pub struct PulledJson {
     pub canisters: BTreeMap<Principal, PulledCanister>,
 }
 
-#[derive(Serialize, Deserialize, Default)]
+#[derive(Serialize, Deserialize, Default, Clone)]
 pub struct PulledCanister {
     /// Name of `type: pull` in dfx.json. Omitted if indirect dependency.
     #[serde(skip_serializing_if = "Option::is_none")]

src/dfx/src/lib/deps/pull/facade.rs

@@ -0,0 +1,190 @@
+use candid::{pretty::candid::pp_args, Principal};
+use candid_parser::utils::{instantiate_candid, CandidSource};
+use dfx_core::config::cache::get_cache_root;
+use dfx_core::fs::{composite::ensure_parent_dir_exists, read, read_to_string, write};
+use sha2::{Digest, Sha256};
+use std::collections::HashMap;
+
+use super::{super::PulledCanister, write_to_tempfile_then_rename};
+use crate::lib::deps::{
+    get_pulled_canister_dir, get_pulled_service_candid_path, get_pulled_wasm_path,
+};
+use crate::lib::error::DfxResult;
+use crate::util::{download_file, download_file_to_path};
+
+static IC_REV: &str = "1eeb4d74deb00bd52739cbd6f37ce1dc72e0c76e";
+
+#[derive(Debug)]
+struct Facade {
+    wasm_url: String,
+    candid_url: String,
+    dependencies: Vec<Principal>,
+    init_guide: String,
+}
+
+lazy_static::lazy_static! {
+    static ref ICP_LEDGER: Principal=Principal::from_text("ryjl3-tyaaa-aaaaa-aaaba-cai").unwrap();
+    static ref CKBTC_LEDGER: Principal=Principal::from_text("mxzaz-hqaaa-aaaar-qaada-cai").unwrap();
+    static ref CKETH_LEDGER: Principal=Principal::from_text("ss2fx-dyaaa-aaaar-qacoq-cai").unwrap();
+    static ref FACADE: HashMap<Principal, Facade> = {
+        let mut m = HashMap::new();
+        // https://internetcomputer.org/docs/current/developer-docs/defi/tokens/ledger/setup/icp_ledger_setup
+        m.insert(
+            *ICP_LEDGER,
+            Facade {
+                wasm_url: format!("https://download.dfinity.systems/ic/{IC_REV}/canisters/ledger-canister.wasm.gz"),
+                candid_url: format!("https://raw.githubusercontent.com/dfinity/ic/{IC_REV}/rs/ledger_suite/icp/ledger.did"),
+                dependencies:vec![],
+                init_guide: r#"
+1. Create a 'minter' identity: dfx identity new minter
+2. Run the following multi-line command:
+
+dfx deps init ryjl3-tyaaa-aaaaa-aaaba-cai --argument "(variant { 
+    Init = record {
+        minting_account = \"$(dfx --identity minter ledger account-id)\";
+        initial_values = vec {};
+        send_whitelist = vec {};
+        transfer_fee = opt record { e8s = 10_000 : nat64; };
+        token_symbol = opt \"LICP\";
+        token_name = opt \"Local ICP\"; 
+    }
+})"
+"#.to_string(),
+            }
+        );
+        // https://github.com/dfinity/ic/blob/master/rs/bitcoin/ckbtc/mainnet/README.md#installing-the-ledger-mxzaz-hqaaa-aaaar-qaada-cai
+        m.insert(
+            *CKBTC_LEDGER,
+            Facade {
+                wasm_url: format!("https://download.dfinity.systems/ic/{IC_REV}/canisters/ic-icrc1-ledger.wasm.gz"),
+                candid_url: format!("https://raw.githubusercontent.com/dfinity/ic/{IC_REV}/rs/ledger_suite/icrc1/ledger/ledger.did"),
+                dependencies:vec![],
+                init_guide: r#"
+1. Create a 'minter' identity: dfx identity new minter
+2. Run the following multi-line command:
+
+dfx deps init mxzaz-hqaaa-aaaar-qaada-cai --argument "(variant {
+    Init = record {
+        minting_account = record { owner = principal \"$(dfx --identity minter identity get-principal)\"; };
+        transfer_fee = 10;
+        token_symbol = \"ckBTC\";
+        token_name = \"ckBTC\";
+        metadata = vec {};
+        initial_balances = vec {};
+        max_memo_length = opt 80;
+        archive_options = record {
+            num_blocks_to_archive = 1000;
+            trigger_threshold = 2000;
+            max_message_size_bytes = null;
+            cycles_for_archive_creation = opt 100_000_000_000_000;
+            node_max_memory_size_bytes = opt 3_221_225_472;
+            controller_id = principal \"2vxsx-fae\"
+        }
+    }
+})"
+"#.to_string(),
+            }
+        );
+        // https://github.com/dfinity/ic/blob/master/rs/ethereum/cketh/mainnet/README.md#installing-the-ledger
+        m.insert(
+            *CKETH_LEDGER,
+            Facade {
+                wasm_url: format!("https://download.dfinity.systems/ic/{IC_REV}/canisters/ic-icrc1-ledger-u256.wasm.gz"),
+                candid_url: format!("https://raw.githubusercontent.com/dfinity/ic/{IC_REV}/rs/ledger_suite/icrc1/ledger/ledger.did"),
+                dependencies:vec![],
+                init_guide: r#"
+1. Create a 'minter' identity: dfx identity new minter
+2. Run the following multi-line command:
+
+dfx deps init ss2fx-dyaaa-aaaar-qacoq-cai --argument "(variant {
+    Init = record {
+        minting_account = record { owner = principal \"$(dfx --identity minter identity get-principal)\"; };
+        decimals = opt 18;
+        max_memo_length = opt 80;
+        transfer_fee = 2_000_000_000_000;
+        token_symbol = \"ckETH\";
+        token_name = \"ckETH\";
+        feature_flags = opt record { icrc2 = true };
+        metadata = vec {};
+        initial_balances = vec {};
+        archive_options = record {
+            num_blocks_to_archive = 1000;
+            trigger_threshold = 2000;
+            max_message_size_bytes = null;
+            cycles_for_archive_creation = opt 100_000_000_000_000;
+            node_max_memory_size_bytes = opt 3_221_225_472;
+            controller_id = principal \"2vxsx-fae\"
+        }
+    }
+})"
+"#.to_string(),
+            }
+        );
+        m
+    };
+}
+
+pub(super) fn facade_dependencies(canister_id: &Principal) -> Option<Vec<Principal>> {
+    FACADE
+        .get(canister_id)
+        .map(|facade| facade.dependencies.clone())
+}
+
+pub(super) async fn facade_download(canister_id: &Principal) -> DfxResult<Option<PulledCanister>> {
+    if let Some(facade) = FACADE.get(canister_id) {
+        let mut pulled_canister = PulledCanister {
+            dependencies: facade.dependencies.clone(),
+            init_guide: facade.init_guide.clone(),
+            gzip: facade.wasm_url.ends_with(".gz"),
+            ..Default::default()
+        };
+        let ic_rev_path = get_cache_root()?
+            .join("pulled")
+            .join(".facade")
+            .join(canister_id.to_text());
+        let wasm_path = get_pulled_wasm_path(canister_id, pulled_canister.gzip)?;
+        let service_candid_path = get_pulled_service_candid_path(canister_id)?;
+        let mut cache_hit = false;
+        if ic_rev_path.exists() && wasm_path.exists() && service_candid_path.exists() {
+            let ic_rev = read_to_string(&ic_rev_path)?;
+            if ic_rev == IC_REV {
+                cache_hit = true;
+            }
+        }
+        if !cache_hit {
+            // delete files from previous pull
+            let pulled_canister_dir = get_pulled_canister_dir(canister_id)?;
+            if pulled_canister_dir.exists() {
+                dfx_core::fs::remove_dir_all(&pulled_canister_dir)?;
+            }
+            dfx_core::fs::create_dir_all(&pulled_canister_dir)?;
+            // download wasm and candid
+            let wasm_url = reqwest::Url::parse(&facade.wasm_url)?;
+            download_file_to_path(&wasm_url, &wasm_path).await?;
+            let candid_url = reqwest::Url::parse(&facade.candid_url)?;
+            let candid_bytes = download_file(&candid_url).await?;
+            let candid_service = String::from_utf8(candid_bytes)?;
+            write_to_tempfile_then_rename(candid_service.as_bytes(), &service_candid_path)?;
+            // write ic_rev for cache logic
+            ensure_parent_dir_exists(&ic_rev_path)?;
+            write(&ic_rev_path, IC_REV)?;
+        }
+
+        // wasm_hash
+        let wasm_content = read(&wasm_path)?;
+        let wasm_hash = Sha256::digest(wasm_content).to_vec();
+        pulled_canister.wasm_hash = hex::encode(&wasm_hash);
+        pulled_canister.wasm_hash_download = hex::encode(&wasm_hash);
+
+        // candid_args
+        let candid_service = read_to_string(&service_candid_path)?;
+        let candid_source = CandidSource::Text(&candid_service);
+        let (args, _service) = instantiate_candid(candid_source)?;
+        let candid_args = pp_args(&args).pretty(80).to_string();
+        pulled_canister.candid_args = candid_args;
+
+        Ok(Some(pulled_canister))
+    } else {
+        Ok(None)
+    }
+}