Merge branch 'master' into mnl/SDK-1238/ic-assetsjson-overrides

.github/workflows/bitcoin-canister-update.yml

@@ -0,0 +1,69 @@
+name: Check Bitcoin Canister Release Update
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * *' # Runs at UTC midnight every day
+
+jobs:
+  check-update:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout dfx repository
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Fetch Bitcoin Canister latest release tag
+      env:
+        GH_TOKEN: "${{ secrets.NIV_UPDATER_TOKEN }}"
+      run: |
+        LATEST_TAG=$(gh release view --repo dfinity/bitcoin-canister --json tagName -q .tagName)
+        echo "Latest tag is $LATEST_TAG"
+        echo "LATEST_TAG=$LATEST_TAG" >> $GITHUB_ENV
+
+    - name: Check if the latest release tag has been updated
+      run: |
+        URL_ENCODED_CURRENT_TAG=$(jq -r '.["ic-btc-canister"].version' nix/sources.json)
+        CURRENT_TAG=$(python -c "import sys, urllib.parse as ul; print(ul.unquote_plus(sys.argv[1]))" "$URL_ENCODED_CURRENT_TAG")
+        echo "Current tag is $CURRENT_TAG"
+        if [[ "$CURRENT_TAG" == "$LATEST_TAG" ]]; then
+          echo "No update is required."
+          exit 1
+        else
+          echo "An update is required."
+        fi
+
+
+    - name: install Nix
+      uses: cachix/install-nix-action@v21
+      with:
+        nix_path: nixpkgs=channel:nixos-unstable
+
+    - name: install niv (dependency manager for Nix projects)
+      run: nix-env -i niv -f '<nixpkgs>'
+
+    - name: install packages from nix/sources.json
+      run: niv update
+
+    - name: update sources
+      run: |
+        URL_ENCODED_LATEST_TAG=$(echo -n "$LATEST_TAG" | python -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.stdin.read().strip(), safe=""))')
+        niv update ic-btc-canister -a version=$URL_ENCODED_LATEST_TAG
+        ./scripts/write-dfx-asset-sources.sh
+
+    - name: Update dfx to use the latest Bitcoin Canister version
+      env:
+        GH_TOKEN: "${{ secrets.NIV_UPDATER_TOKEN }}"
+      run: |
+        git config user.name github-actions
+        git config user.email [email protected]
+        git checkout -b bot/update-bitcoin-canister/$LATEST_TAG
+        git add .
+        git commit -m "Update Bitcoin Canister to $LATEST_TAG"
+        git push --set-upstream origin bot/update-bitcoin-canister/$LATEST_TAG
+        PR_TITLE="chore: Update Bitcoin Canister Version to $LATEST_TAG"
+        PR_BODY="This PR updates the Bitcoin Canister version to the latest tag: $LATEST_TAG"
+        gh pr create --title "$PR_TITLE" --body "$PR_BODY" --base master --head $(git branch --show-current)
+

CHANGELOG.md

@@ -2,6 +2,25 @@
 
 # UNRELEASED
 
+=== fix: dfx extension install can no longer create a corrupt cache directory
+
+Running `dfx cache delete && dfx extension install nns` would previously
+create a cache directory containing only an `extensions` subdirectory.
+dfx only looks for the existence of a cache version subdirectory to
+determine whether it has been installed. The end result was that later
+commands would fail when the cache did not contain expected files.
+
+=== fix: output_env_file is now considered relative to project root
+
+The .env file location, whether specified as `output_env_file` in dfx.json
+or `--output-env-file <file>` on the commandline, is now considered relative
+to the project root, rather than relative to the current working directory.
+
+=== feat: Read dfx canister install argument from a file
+
+Enables passing large arguments that cannot be passed directly in the command line using the `--argument-file` flag. For example `dfx canister install --argument-file ./my/argument/file.txt my_canister_name`.
+
+
 ### feat: change `list_permitted` and `list_authorized` to an update call.
 
 This requires the `list_authorized` and `list_permitted` methods to be called as an update and disables the ability to

docs/cli-reference/dfx-canister.md

@@ -438,6 +438,7 @@ You can use the following optional flags with the `dfx canister install` command
 
 | Flag                  | Description                                                                                                                                                             |
 |-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `--argument-file`     | Specifies the file from which to read the argument to pass to the init method.  Stdin may be referred to as `-`.                                                        |
 | `--async-call`        | Enables you to continue without waiting for the result of the installation to be returned by polling the Internet Computer or the local canister execution environment. |
 | `--upgrade-unchanged` | Upgrade the canister even if the .wasm did not change.                                                                                                                  |
 

e2e/tests-dfx/dotenv.bash

@@ -14,6 +14,43 @@ teardown() {
   standard_teardown
 }
 
+
+@test "puts .env in project root" {
+  dfx_start
+  jq '.canisters["e2e_project_backend"].post_install="echo post install backend"' dfx.json | sponge dfx.json
+  jq '.canisters["e2e_project_frontend"].post_install="echo post install frontend"' dfx.json | sponge dfx.json
+
+  mkdir subdir
+  mkdir subdir/canister-install-all subdir/canister-install-single
+  mkdir subdir/build-all subdir/build-single
+  mkdir subdir/deploy-single subdir/deploy-all
+  dfx canister create --all
+  ( cd subdir/build-single && dfx build e2e_project_frontend )
+  ( cd subdir/build-all && dfx build --all )
+  ( cd subdir/canister-install-single && dfx canister install e2e_project_backend )
+  dfx canister uninstall-code e2e_project_backend
+  ( cd subdir/canister-install-all && dfx canister install --all )
+  rm -rf .dfx
+  ( cd subdir/deploy-single && dfx deploy e2e_project_backend)
+  ( cd subdir/deploy-all && dfx deploy )
+
+  assert_command find . -name .env
+  assert_eq "./.env"
+}
+
+@test "the output_env_file must be contained within project" {
+  dfx_start
+  mkdir ../outside
+
+  assert_command_fail dfx deploy --output-env-file nonexistent/.env
+  assert_contains "failed to canonicalize output_env_file"
+  assert_contains "working-dir/e2e_project/nonexistent: No such file or directory"
+  assert_command_fail dfx deploy --output-env-file /etc/passwd
+  assert_contains "The output_env_file must be a relative path, but is /etc/passwd"
+  assert_command_fail dfx deploy --output-env-file ../outside/.env
+  assert_match "The output_env_file must be within the project root, but is .*/working-dir/e2e_project/../outside/.env"
+}
+
 @test "writes environment variables to .env" {
   dfx_start
   dfx canister create --all

e2e/tests-dfx/extension.bash

@@ -13,6 +13,12 @@ teardown() {
   standard_teardown
 }
 
+@test "extension install with an empty cache does not create a corrupt cache" {
+  dfx cache delete
+  dfx extension install nns --version 0.2.1
+  dfx_start
+}
+
 @test "install extension from official registry" {
   assert_command_fail dfx snsx
 

e2e/tests-dfx/install.bash

@@ -182,7 +182,32 @@ teardown() {
   assert_contains db07e7e24f6f8ddf53c33a610713259a7c1eb71c270b819ebd311e2d223267f0
 }
 
+@test "installing one canister with an argument succeeds" {
+  dfx_start
+  assert_command dfx canister create e2e_project_backend
+  assert_command dfx build e2e_project_backend
+  assert_command dfx canister install e2e_project_backend --argument '()'
+}
+
+@test "installing with an argument in a file succeeds" {
+  dfx_start
+  assert_command dfx canister create e2e_project_backend
+  assert_command dfx build e2e_project_backend
+  TMPFILE="$(mktemp)"
+  echo '()' >"$TMPFILE"
+  assert_command dfx canister install e2e_project_backend --argument-file "$TMPFILE"
+}
+
+@test "installing with an argument on stdin succeeds" {
+  dfx_start
+  assert_command dfx canister create e2e_project_backend
+  assert_command dfx build e2e_project_backend
+  TMPFILE="$(mktemp)"
+  echo '()' >"$TMPFILE"
+  assert_command dfx canister install e2e_project_backend --argument-file - <"$TMPFILE"
+}
+
 @test "installing multiple canisters with arguments fails" {
-  assert_command_fail dfx canister install --all --argument hello
+  assert_command_fail dfx canister install --all --argument '()'
   assert_contains "error: the argument '--all' cannot be used with '--argument <ARGUMENT>'"
 }

src/dfx-core/src/config/model/dfinity.rs

@@ -3,6 +3,7 @@
 use crate::config::directories::get_user_dfx_config_dir;
 use crate::config::model::bitcoin_adapter::BitcoinAdapterLogLevel;
 use crate::config::model::canister_http_adapter::HttpAdapterLogLevel;
+use crate::error::config::GetOutputEnvFileError;
 use crate::error::dfx_config::AddDependenciesError::CanisterCircularDependency;
 use crate::error::dfx_config::GetCanisterNamesWithDependenciesError::AddDependenciesFailed;
 use crate::error::dfx_config::GetComputeAllocationError::GetComputeAllocationFailed;
@@ -1013,6 +1014,35 @@ impl Config {
         )
     }
 
+    // returns the path to the output env file if any, guaranteed to be
+    // a child relative to the project root
+    pub fn get_output_env_file(
+        &self,
+        from_cmdline: Option<PathBuf>,
+    ) -> Result<Option<PathBuf>, GetOutputEnvFileError> {
+        from_cmdline
+            .or(self.config.output_env_file.clone())
+            .map(|p| {
+                if p.is_relative() {
+                    let p = self.get_project_root().join(p);
+
+                    // cannot canonicalize a path that doesn't exist, but the parent should exist
+                    let env_parent =
+                        crate::fs::parent(&p).map_err(GetOutputEnvFileError::Parent)?;
+                    let env_parent = crate::fs::canonicalize(&env_parent)
+                        .map_err(GetOutputEnvFileError::Canonicalize)?;
+                    if !env_parent.starts_with(self.get_project_root()) {
+                        Err(GetOutputEnvFileError::OutputEnvFileMustBeInProjectRoot(p))
+                    } else {
+                        Ok(self.get_project_root().join(p))
+                    }
+                } else {
+                    Err(GetOutputEnvFileError::OutputEnvFileMustBeRelative(p))
+                }
+            })
+            .transpose()
+    }
+
     pub fn save(&self) -> Result<(), StructuredFileError> {
         save_json_file(&self.path, &self.json)
     }

src/dfx-core/src/error/config.rs

@@ -1,5 +1,6 @@
 use crate::error::fs::FsError;
 use crate::error::get_user_home::GetUserHomeError;
+use std::path::PathBuf;
 use thiserror::Error;
 
 #[derive(Error, Debug)]
@@ -13,3 +14,18 @@ pub enum ConfigError {
     #[error("Failed to determine shared network data directory: {0}")]
     DetermineSharedNetworkDirectoryFailed(GetUserHomeError),
 }
+
+#[derive(Error, Debug)]
+pub enum GetOutputEnvFileError {
+    #[error("failed to canonicalize output_env_file")]
+    Canonicalize(#[source] FsError),
+
+    #[error("The output_env_file must be within the project root, but is {}", .0.display())]
+    OutputEnvFileMustBeInProjectRoot(PathBuf),
+
+    #[error("The output_env_file must be a relative path, but is {}", .0.display())]
+    OutputEnvFileMustBeRelative(PathBuf),
+
+    #[error(transparent)]
+    Parent(FsError),
+}

src/dfx/src/commands/build.rs

@@ -40,9 +40,7 @@ pub fn exec(env: &dyn Environment, opts: CanisterBuildOpts) -> DfxResult {
 
     // Read the config.
     let config = env.get_config_or_anyhow()?;
-    let env_file = opts
-        .output_env_file
-        .or_else(|| config.get_config().output_env_file.clone());
+    let env_file = config.get_output_env_file(opts.output_env_file)?;
 
     // Check the cache. This will only install the cache if there isn't one installed
     // already.

src/dfx/src/commands/canister/install.rs

@@ -3,8 +3,12 @@ use crate::lib::environment::Environment;
 use crate::lib::error::DfxResult;
 use crate::lib::operations::canister::install_canister::install_canister;
 use crate::lib::root_key::fetch_root_key_if_needed;
+use crate::util::clap::parsers::file_or_stdin_parser;
 use crate::util::get_candid_init_type;
-use crate::{lib::canister_info::CanisterInfo, util::blob_from_arguments};
+use crate::{
+    lib::canister_info::CanisterInfo,
+    util::{arguments_from_file, blob_from_arguments},
+};
 use dfx_core::identity::CallSender;
 
 use anyhow::{anyhow, bail, Context};
@@ -40,9 +44,17 @@ pub struct CanisterInstallOpts {
     upgrade_unchanged: bool,
 
     /// Specifies the argument to pass to the method.
-    #[arg(long)]
+    #[arg(long, conflicts_with("argument_file"))]
     argument: Option<String>,
 
+    /// Specifies the file from which to read the argument to pass to the method.
+    #[arg(
+        long,
+        value_parser = file_or_stdin_parser,
+        conflicts_with("argument")
+    )]
+    argument_file: Option<PathBuf>,
+
     /// Specifies the data type for the argument when making the call using an argument.
     #[arg(long, requires("argument"), value_parser = ["idl", "raw"])]
     argument_type: Option<String>,
@@ -107,7 +119,14 @@ pub async fn exec(
 
         let canister_id =
             Principal::from_text(canister).or_else(|_| canister_id_store.get(canister))?;
+
+        let arguments_from_file = opts
+            .argument_file
+            .map(|v| arguments_from_file(&v))
+            .transpose()?;
         let arguments = opts.argument.as_deref();
+        let arguments = arguments_from_file.as_deref().or(arguments);
+
         let arg_type = opts.argument_type.as_deref();
         let canister_info = config.as_ref()
             .ok_or_else(|| anyhow!("Cannot find dfx configuration file in the current working directory. Did you forget to create one?"))
@@ -136,9 +155,7 @@ pub async fn exec(
         } else {
             let canister_info = canister_info?;
             let config = config.unwrap();
-            let env_file = opts
-                .output_env_file
-                .or_else(|| config.get_config().output_env_file.clone());
+            let env_file = config.get_output_env_file(opts.output_env_file)?;
             let idl_path = canister_info.get_constructor_idl_path();
             let init_type = get_candid_init_type(&idl_path);
             let install_args = || blob_from_arguments(arguments, None, arg_type, &init_type);
@@ -163,9 +180,7 @@ pub async fn exec(
     } else if opts.all {
         // Install all canisters.
         let config = env.get_config_or_anyhow()?;
-        let env_file = opts
-            .output_env_file
-            .or_else(|| config.get_config().output_env_file.clone());
+        let env_file = config.get_output_env_file(opts.output_env_file)?;
         if let Some(canisters) = &config.get_config().canisters {
             for canister in canisters.keys() {
                 if pull_canisters_in_config.contains_key(canister) {

src/dfx/src/commands/deploy.rs

@@ -115,9 +115,7 @@ pub fn exec(env: &dyn Environment, opts: DeployOpts) -> DfxResult {
         .map_err(|err| anyhow!(err))
         .context("Failed to parse InstallMode.")?;
     let config = env.get_config_or_anyhow()?;
-    let env_file = opts
-        .output_env_file
-        .or_else(|| config.get_config().output_env_file.clone());
+    let env_file = config.get_output_env_file(opts.output_env_file)?;
 
     let with_cycles = opts.with_cycles;
 

src/dfx/src/commands/extension/install.rs

@@ -1,4 +1,5 @@
 use crate::commands::DfxCommand;
+use crate::config::cache::DiskBasedCache;
 use crate::lib::environment::Environment;
 use crate::lib::error::DfxResult;
 use clap::Parser;
@@ -19,6 +20,9 @@ pub struct InstallOpts {
 }
 
 pub fn exec(env: &dyn Environment, opts: InstallOpts) -> DfxResult<()> {
+    // creating an `extensions` directory in an otherwise empty cache directory would
+    // cause the cache to be considered "installed" and later commands would fail
+    DiskBasedCache::install(&env.get_cache().version_str())?;
     let spinner = env.new_spinner(format!("Installing extension: {}", opts.name).into());
     let mgr = env.new_extension_manager()?;
     let effective_extension_name = opts.install_as.clone().unwrap_or_else(|| opts.name.clone());