From 1e467510b235c41e160684f5f9eb388a66254697 Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Wed, 25 Oct 2023 14:24:47 +0200 Subject: [PATCH 1/5] feat: custom canister types --- Cargo.lock | 2 + Cargo.toml | 8 + e2e/tests-dfx/build.bash | 2 +- e2e/tests-dfx/extension.bash | 59 +++ src/dfx-core/Cargo.toml | 4 +- src/dfx-core/src/config/model/dfinity.rs | 54 ++- src/dfx-core/src/error/extension.rs | 10 + .../manifest/custom_canister_type.rs | 364 ++++++++++++++++++ .../src/extension/manifest/extension.rs | 163 +++++++- src/dfx-core/src/extension/manifest/mod.rs | 2 + src/dfx-core/src/extension/mod.rs | 2 +- src/dfx-core/src/network/provider.rs | 15 +- src/dfx/Cargo.toml | 6 +- src/dfx/src/lib/environment.rs | 2 +- 14 files changed, 660 insertions(+), 33 deletions(-) create mode 100644 src/dfx-core/src/extension/manifest/custom_canister_type.rs diff --git a/Cargo.lock b/Cargo.lock index 7936f84b47..151eab74e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1429,6 +1429,7 @@ dependencies = [ "dialoguer", "directories-next", "flate2", + "handlebars", "hex", "humantime-serde", "ic-agent", @@ -1438,6 +1439,7 @@ dependencies = [ "keyring", "lazy_static", "proptest", + "regex", "reqwest", "ring", "schemars", diff --git a/Cargo.toml b/Cargo.toml index 18a85414e4..3d1528dcb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ dialoguer = "0.10.0" directories-next = "2.0.0" flate2 = { version = "1.0.11", default-features = false } futures = "0.3.21" +handlebars = "4.3.3" hex = "0.4.3" humantime = "2.1.0" itertools = "0.10.3" @@ -48,6 +49,13 @@ mime_guess = "2.0.4" num-traits = "0.2.14" pem = "1.0.2" proptest = "1.0.0" +regex = "1.5.5" +reqwest = { version = "0.11.9", default-features = false, features = [ + "blocking", + "json", + "rustls-tls", + "native-tls-vendored", +]} ring = "0.16.11" schemars = "0.8" sec1 = "0.3.0" diff --git a/e2e/tests-dfx/build.bash b/e2e/tests-dfx/build.bash index 2be504a37d..525d54224b 100644 --- a/e2e/tests-dfx/build.bash +++ b/e2e/tests-dfx/build.bash @@ -215,7 +215,7 @@ teardown() { jq '.canisters.e2e_project_backend.type="unknown_canister_type"' dfx.json | sponge dfx.json assert_command_fail dfx build # shellcheck disable=SC2016 - assert_match 'unknown variant `unknown_canister_type`' + assert_match 'ExtensionNotInstalled("unknown_canister_type")' # TODO: concrete error type # If canister type is invalid, `dfx stop` fails jq '.canisters.e2e_project_backend.type="motoko"' dfx.json | sponge dfx.json diff --git a/e2e/tests-dfx/extension.bash b/e2e/tests-dfx/extension.bash index f5688d51ef..0c67aac7b6 100644 --- a/e2e/tests-dfx/extension.bash +++ b/e2e/tests-dfx/extension.bash @@ -209,3 +209,62 @@ EOF assert_command dfx extension run test_extension abc --the-another-param 464646 --the-param 123 456 789 assert_eq "abc --the-another-param 464646 --the-param 123 456 789 --dfx-cache-path $CACHE_DIR" } + +@test "custom canister types" { + dfx cache install + + CACHE_DIR=$(dfx cache show) + mkdir -p "$CACHE_DIR"/extensions/playground + echo '#!/usr/bin/env bash + +echo testoutput' > "$CACHE_DIR"/extensions/playground/playground + chmod +x "$CACHE_DIR"/extensions/playground/playground + + echo '{ + "name": "playground", + "version": "0.1.0", + "homepage": "https://github.com/dfinity/playground", + "authors": "DFINITY", + "summary": "Motoko playground for the Internet Computer", + "categories": [], + "keywords": [], + "subcommands": {}, + "canister_types": { + "playground": { + "type": "custom", + "build": "echo the wasm-utils canister is prebuilt", + "candid": "{{canister_name}}.did", + "wasm": "{{canister_name}}.wasm", + "gzip": false + } + } +}' > "$CACHE_DIR"/extensions/playground/extension.json + + assert_command dfx extension list + assert_match "playground" + + dfx_new hello + create_networks_json + install_asset playground_backend + + echo '{ + "canisters": { + "wasm-utils": { + "type": "playground", + "gzip": true + } + }, + "defaults": { + "build": { + "args": "", + "packtool": "" + } + }, + "output_env_file": ".env", + "version": 1 +}' > dfx.json + + dfx_start + assert_command dfx deploy -v + assert_match 'Backend canister via Candid interface' +} diff --git a/src/dfx-core/Cargo.toml b/src/dfx-core/Cargo.toml index f1b9fef3f3..9981513324 100644 --- a/src/dfx-core/Cargo.toml +++ b/src/dfx-core/Cargo.toml @@ -18,6 +18,7 @@ clap = { workspace = true, features = ["string"] } dialoguer = "0.10.0" directories-next.workspace = true flate2 = { workspace = true, default-features = false, features = ["zlib-ng"] } +handlebars.workspace = true hex = { workspace = true, features = ["serde"] } humantime-serde = "1.1.1" ic-agent = { workspace = true, features = ["reqwest"] } @@ -26,7 +27,8 @@ ic-identity-hsm = { workspace = true } k256 = { version = "0.11.4", features = ["pem"] } keyring.workspace = true lazy_static.workspace = true -reqwest = { version = "0.11.9", features = ["blocking", "json"] } +reqwest = { workspace = true, features = ["blocking", "json"] } +regex.workspace = true ring.workspace = true schemars.workspace = true sec1 = { workspace = true, features = ["std"] } diff --git a/src/dfx-core/src/config/model/dfinity.rs b/src/dfx-core/src/config/model/dfinity.rs index e67d9a7c38..9b9eb73fe4 100644 --- a/src/dfx-core/src/config/model/dfinity.rs +++ b/src/dfx-core/src/config/model/dfinity.rs @@ -32,6 +32,7 @@ use crate::error::structured_file::StructuredFileError; use crate::error::structured_file::StructuredFileError::{ DeserializeJsonFileFailed, ReadJsonFileFailed, }; +use crate::extension::manifest::custom_canister_type; use crate::json::save_json_file; use crate::json::structure::{PossiblyStr, SerdeVec}; use byte_unit::Byte; @@ -950,42 +951,65 @@ impl Config { Ok(None) } - fn from_file(path: &Path) -> Result { + fn from_file( + path: &Path, + dfx_version: &semver::Version, + ) -> Result { let content = crate::fs::read(path).map_err(ReadJsonFileFailed)?; - Config::from_slice(path.to_path_buf(), &content) + Config::from_slice(path.to_path_buf(), &content, dfx_version) } - pub fn from_dir(working_dir: &Path) -> Result, LoadDfxConfigError> { + pub fn from_dir( + working_dir: &Path, + dfx_version: &semver::Version, + ) -> Result, LoadDfxConfigError> { let path = Config::resolve_config_path(working_dir)?; - path.map(|path| Config::from_file(&path)) + path.map(|path| Config::from_file(&path, dfx_version)) .transpose() .map_err(LoadFromFileFailed) } - pub fn from_current_dir() -> Result, LoadDfxConfigError> { - Config::from_dir(&std::env::current_dir().map_err(DetermineCurrentWorkingDirFailed)?) + pub fn from_current_dir( + dfx_version: &semver::Version, + ) -> Result, LoadDfxConfigError> { + Config::from_dir( + &std::env::current_dir().map_err(DetermineCurrentWorkingDirFailed)?, + dfx_version, + ) } - fn from_slice(path: PathBuf, content: &[u8]) -> Result { - let config = serde_json::from_slice(content) + fn from_slice( + path: PathBuf, + content: &[u8], + dfx_version: &semver::Version, + ) -> Result { + let mut json: serde_json::Value = serde_json::from_slice(content) .map_err(|e| DeserializeJsonFileFailed(Box::new(path.clone()), e))?; - let json = serde_json::from_slice(content) + let extension_manager = + crate::extension::manager::ExtensionManager::new(dfx_version).unwrap(); + custom_canister_type::transform_dfx_json_via_extension(&mut json, extension_manager) + .unwrap(); // TODO: error handling + let config = serde_json::from_value(json.clone()) .map_err(|e| DeserializeJsonFileFailed(Box::new(path.clone()), e))?; Ok(Config { path, json, config }) } /// Create a configuration from a string. #[cfg(test)] - pub(crate) fn from_str(content: &str) -> Result { - Config::from_slice(PathBuf::from("-"), content.as_bytes()) + pub(crate) fn from_str( + content: &str, + dfx_version: &semver::Version, + ) -> Result { + Config::from_slice(PathBuf::from("-"), content.as_bytes(), dfx_version) } #[cfg(test)] pub(crate) fn from_str_and_path( path: PathBuf, content: &str, + dfx_version: &semver::Version, ) -> Result { - Config::from_slice(path, content.as_bytes()) + Config::from_slice(path, content.as_bytes(), dfx_version) } pub fn get_path(&self) -> &PathBuf { @@ -1203,6 +1227,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1225,6 +1250,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1246,6 +1272,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1268,6 +1295,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1301,6 +1329,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1324,6 +1353,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); let config_interface = config_no_values.get_config(); diff --git a/src/dfx-core/src/error/extension.rs b/src/dfx-core/src/error/extension.rs index d6f6481427..f350488cdf 100644 --- a/src/dfx-core/src/error/extension.rs +++ b/src/dfx-core/src/error/extension.rs @@ -94,4 +94,14 @@ pub enum ExtensionError { #[error("Extension exited with non-zero status code '{0}'.")] ExtensionExitedWithNonZeroStatus(i32), + + // errors related to custom canister types + #[error("Extension '{0}' does not support any custom canister types.")] + ExtensionDoesNotSupportAnyCustomCanisterTypes(String), + + #[error("Extension '{0}' does not support the specific custom canister type '{1}'.")] + ExtensionDoesNotSupportSpecificCustomCanisterType(String, String), + + #[error("Failed to load custom canister type template for canister type '{0}' from extension '{1}': {2}")] + CustomCanisterTypeTemplateError(String, String, String), } diff --git a/src/dfx-core/src/extension/manifest/custom_canister_type.rs b/src/dfx-core/src/extension/manifest/custom_canister_type.rs new file mode 100644 index 0000000000..8501dd9aee --- /dev/null +++ b/src/dfx-core/src/extension/manifest/custom_canister_type.rs @@ -0,0 +1,364 @@ +use crate::error::extension::ExtensionError; +use crate::extension::manager::ExtensionManager; +use crate::extension::manifest::ExtensionManifest; +use handlebars::Handlebars; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use serde_json::{Map as JsonMap, Value as JsonValue}; +use std::collections::BTreeMap; + +pub(crate) fn transform_dfx_json_via_extension( + json: &mut JsonValue, + extension_manager: ExtensionManager, +) -> Result<(), ExtensionError> { + let canisters = match json.get_mut("canisters").and_then(|v| v.as_object_mut()) { + Some(canisters) => canisters, + None => return Ok(()), + }; + for (canister_name, canister_declaration) in canisters.iter_mut() { + if let Some(canister_type) = get_valid_canister_type(canister_declaration) { + let canister_declaration = canister_declaration.as_object_mut().unwrap(); + let (extension_name, canister_type) = + get_extension_name_and_custom_canister_type(&canister_type); + let extension_manifest = ExtensionManifest::get(extension_name, &extension_manager)?; + *canister_declaration = process_canister_declaration( + canister_declaration, + extension_name, + &extension_manifest, + canister_name, + canister_type, + )?; + } + } + Ok(()) +} + +fn get_valid_canister_type(canister_declaration: &mut JsonValue) -> Option { + canister_declaration + .get("type") + .and_then(|v| v.as_str()) + .and_then(|s| { + if !["rust", "motoko", "custom", "assets", "pull"].contains(&s) { + Some(s.to_owned()) + } else { + None + } + }) +} + +/// Split the canister type on ':', returning `extension_name` and `canister_type` +/// If there's no ':', `canister_type` is the same as `extension_name` +pub(super) fn get_extension_name_and_custom_canister_type(canister_type: &str) -> (&str, &str) { + if let Some(i) = canister_type.find(':') { + (&canister_type[..i], &canister_type[i + 1..]) + } else { + (canister_type, canister_type) + } +} + +pub(super) fn process_canister_declaration( + canister_declaration: &mut JsonMap, + extension_name: &str, + extension_manifest: &ExtensionManifest, + canister_name: &str, + canister_type: &str, +) -> Result, ExtensionError> { + let extension_manifest_canister_type = extension_manifest.canister_types.as_ref().ok_or( + ExtensionError::ExtensionDoesNotSupportAnyCustomCanisterTypes(extension_name.into()), + )?; + let extension_manifest_canister_type = extension_manifest_canister_type.get(canister_type); + + let custom_canister_declaration = match extension_manifest_canister_type { + Some(val) => val, + None => { + return Err( + ExtensionError::ExtensionDoesNotSupportSpecificCustomCanisterType( + canister_type.into(), + extension_name.into(), + ), + ); + } + }; + let mut values: BTreeMap = canister_declaration + .into_iter() + .filter_map(|(k, v)| { + if v.is_array() || v.is_object() { + None + } else { + Some((k.clone(), v.clone())) + } + }) + .collect(); + values.insert("canister_name".into(), canister_name.into()); + + custom_canister_declaration + .apply_template(values) + .map_err(|e| { + ExtensionError::CustomCanisterTypeTemplateError( + extension_name.to_string(), + canister_type.to_string(), + e.to_string(), + ) + }) +} + +type FieldName = String; + +#[derive(Clone, Deserialize, Serialize, Debug)] +pub struct CustomCanisterTypeDeclaration(BTreeMap); + +#[cfg_attr(test, derive(Eq, PartialEq))] +#[derive(Clone, Serialize, Deserialize, Debug)] +#[serde(untagged)] +enum Op { + Replace { replace: Replace }, + Remove { remove: bool }, + Template(String), + BoolValue(bool), + NumberValue(serde_json::Number), +} + +#[cfg_attr(test, derive(Eq, PartialEq))] +#[derive(Clone, Serialize, Deserialize, Debug)] +struct Replace { + input: String, + search: String, + output: String, +} + +#[derive(Debug)] +enum OpError { + InvalidTemplate(handlebars::RenderError), + InvalidReplace(regex::Error), +} + +impl std::fmt::Display for OpError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + OpError::InvalidTemplate(e) => write!(f, "Invalid template: {}", e), + OpError::InvalidReplace(e) => write!(f, "Invalid regex: {}", e), + } + } +} + +impl CustomCanisterTypeDeclaration { + fn apply_template( + &self, + values: BTreeMap, + ) -> Result, OpError> { + let mut remove_fields = vec![]; + let mut final_fields = JsonMap::new(); + for (field_name, op) in self + .0 + .clone() + .into_iter() + .collect::>() + .clone() + .into_iter() + { + match op { + Op::NumberValue(x) => { + final_fields.insert(field_name, x.into()); + } + Op::BoolValue(x) => { + final_fields.insert(field_name, x.into()); + } + + Op::Template(template) => { + let x = Handlebars::new() + .render_template(&template, &values) + .map_err(OpError::InvalidTemplate)?; + final_fields.insert(field_name, x.into()); + } + Op::Replace { replace } => { + let input = Handlebars::new() + .render_template(&replace.input, &values) + .map_err(OpError::InvalidTemplate)?; + let re = Regex::new(&replace.search).map_err(OpError::InvalidReplace)?; + let x = re.replace_all(&input, &replace.output).to_string(); + final_fields.insert(field_name, x.into()); + } + Op::Remove { remove } if remove => { + remove_fields.push(field_name); + } + _ => {} + } + } + // Removing fields should be done last because of the order of the fields in the map. + // It's easier to do in second for loop than to sort Ops beforehand, bacause Op would need to implement PartialOrd, + // which is not possible, because serde_json::Number does not implement it. + for field_name in remove_fields { + final_fields.remove(&field_name); + } + // Override custom canister declaration values by the real canister_declaration + for (key, value) in values.iter() { + if key != "type" && key != "canister_name" { + final_fields.insert(key.clone(), value.clone()); + } + } + + Ok(final_fields) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + macro_rules! test_op { + ( + custom_canister_template = $custom_canister_template:expr, + dfx_json_canister_values = $dfx_json_canister:expr, + expected = $expected:expr + ) => { + let custom_canister_template = serde_json::from_str::($custom_canister_template).unwrap(); + // dfx_json_canister_values is a transformed version of canister declaration from dfx.json. + // Below is the example of the transformation; FROM: + // "frontend_canister": { + // "type": "custom" + // } + // transformed INTO: + // let values: BTreeMap = [ + // ("canister_name".into(), Value::String("frontend_canister".into())) + // ("type".into(), Value::String("custom".into())) + // ].into(); + let dfx_json_canister_values = serde_json::from_str($dfx_json_canister).unwrap(); + let expected = serde_json::from_str($expected).unwrap(); + assert_eq!( + custom_canister_template + .apply_template(dfx_json_canister_values) + .unwrap(), + expected + ); + };} + + #[test] + fn test_op_replace() {} + + #[test] + fn test_op_remove() { + test_op!( + custom_canister_template = r#" + { + "type": "{{canister_name}}", + "main": "src/main.ts", + "main": { "remove": true } + } + "#, + dfx_json_canister_values = r#" + { + "canister_name": "something", + "main": "oowee.exe" + } + "#, + expected = r#" + { + "main": "oowee.exe", + "type": "something" + } + "# + ); + } + + #[test] + fn test_op_template() { + test_op!( + custom_canister_template = r#" + { + "type": "{{canister_name}}", + "main": "src/main.ts" + } + "#, + dfx_json_canister_values = r#" + { + "type": "custom", + "canister_name": "something" + } + "#, + expected = r#" + { + "main": "src/main.ts", + "type": "something" + } + "# + ); + } + + #[test] + fn test_op_bool() { + test_op!( + custom_canister_template = r#" + { + "main": true + } + "#, + dfx_json_canister_values = r#" + { + "type": "my_bool_extension", + "canister_name": "something" + } + "#, + expected = r#" + { + "main": true + } + "# + ); + } + + #[test] + fn test_op_number() { + test_op!( + custom_canister_template = r#" + { + "main": 3 + } + "#, + dfx_json_canister_values = r#" + { + "type": "my_number_extension", + "canister_name": "something" + } + "#, + expected = r#" + { + "main": 3 + } + "# + ); + } + + #[test] + fn test_ops() { + test_op!( + custom_canister_template = r#" + { + "type": "custom", + "main": "src/main.ts", + "ts": { "replace": { "input": "{{main}}", "search": "(.*).ts", "output": "$1.ts" }}, + "wasm": ".azyl/{{canister_name}}/{{canister_name}}.wasm.gz", + "build": "npx azyl {{canister_name}}", + "candid": { "replace": { "input": "{{main}}", "search": "(.*).ts", "output": "$1.did" }}, + "main": { "remove": true }, + "gzip": true + }"#, + dfx_json_canister_values = r#" + { + "canister_name": "azyl_frontend", + "main": "src/main.ts", + "gzip": false + }"#, + expected = r#" + { + "build": "npx azyl azyl_frontend", + "candid": "src/main.did", + "gzip": false, + "main": "src/main.ts", + "ts": "src/main.ts", + "type": "custom", + "wasm": ".azyl/azyl_frontend/azyl_frontend.wasm.gz" + } + "# + ); + } +} diff --git a/src/dfx-core/src/extension/manifest/extension.rs b/src/dfx-core/src/extension/manifest/extension.rs index a9fcfa3927..e74c26f615 100644 --- a/src/dfx-core/src/extension/manifest/extension.rs +++ b/src/dfx-core/src/extension/manifest/extension.rs @@ -1,9 +1,7 @@ -use crate::error::extension::ExtensionError; +use super::custom_canister_type::CustomCanisterTypeDeclaration; +use crate::{error::extension::ExtensionError, extension::manager::ExtensionManager}; use serde::{Deserialize, Deserializer}; -use std::{ - collections::{BTreeMap, HashMap}, - path::Path, -}; +use std::collections::{BTreeMap, HashMap}; pub static MANIFEST_FILE_NAME: &str = "extension.json"; @@ -23,14 +21,23 @@ pub struct ExtensionManifest { pub description: Option, pub subcommands: Option, pub dependencies: Option>, + pub canister_types: Option>, } impl ExtensionManifest { - pub fn new(name: &str, extensions_root_dir: &Path) -> Result { - let manifest_path = extensions_root_dir.join(name).join(MANIFEST_FILE_NAME); + pub fn get( + extension_name: &str, + extension_manager: &ExtensionManager, + ) -> Result { + if !extension_manager.is_extension_installed(extension_name) { + return Err(ExtensionError::ExtensionNotInstalled(extension_name.into())); + } + let manifest_path = extension_manager + .get_extension_directory(extension_name) + .join(MANIFEST_FILE_NAME); let mut m: ExtensionManifest = crate::json::load_json_file(&manifest_path) .map_err(ExtensionError::LoadExtensionManifestFailed)?; - m.name = name.to_string(); + m.name = extension_name.to_string(); Ok(m) } @@ -182,7 +189,7 @@ impl ExtensionSubcommandOpts { } #[test] -fn parse_test_file() { +fn parse_test_extension_manifest_file() { let f = r#" { "name": "sns", @@ -295,6 +302,27 @@ fn parse_test_file() { } } } + }, + "canister_types": { + "azyl": { + "type": "custom", + "main": "fff", + "wasm": ".azyl/{{canister_name}}/{{canister_name}}.wasm.gz", + "candid": { "replace": { "input": "{{main}}", "search": "(.*).ts", "output": "$1.did" }}, + "build": "npx azyl {{canister_name}}", + "root": { "replace": { "input": "{{main}}", "search": "(.*)/[^/]*", "output": "$1"}}, + "ts": "{{main}}", + "main": { "remove": true } + }, + "other": { + "type": "custom", + "main": "src/main.ts", + "wasm": { "replace": { "input": "{{candid}}", "search": "(.*)/candid/(.*).did", "output": "$1/wasm/$2.wasm" }}, + "build": "", + "candid": { "replace": { "input": "{{main}}", "search": "(.*).ts", "output": "$1.did" }}, + "main": { "remove": true }, + "gzip": true + } } } "#; @@ -317,11 +345,13 @@ fn parse_test_file() { }}; } - let m: Result = dbg!(serde_json::from_str(f)); - assert!(m.is_ok()); - - let mut subcmds = dbg!(m.unwrap().into_clap_commands().unwrap()); + // test manifest parsing + let extension_manifest: Result = + dbg!(serde_json::from_str(f)); + assert!(extension_manifest.is_ok()); + // test parsing mock CLI commands + let mut subcmds = dbg!(extension_manifest.unwrap().into_clap_commands().unwrap()); use clap::error::ErrorKind::*; for c in &mut subcmds { c.print_long_help().unwrap(); @@ -365,11 +395,118 @@ fn parse_test_file() { _ => {} } } + + // display how the `help` will look like clap::Command::new("sns") .subcommands(&subcmds) .print_help() .unwrap(); + // see docs for clap's debug_assert clap::Command::new("sns") .subcommands(&subcmds) .debug_assert(); + + // test custom canister type + // notice, we're testing if we can overwrite `gzip` field + let extension_manifest: ExtensionManifest = serde_json::from_str(f).unwrap(); + const DFX_JSON_WITH_CUSTOM_CANISTER_TYPE: &str = r#" + { + "canisters": { + "azyl_frontend": { + "type": "azyl", + "main": "src/main.ts", + "gzip": false + }, + "another_canister": { + "type": "azyl:other", + "main": "path/to/file/main.ts", + "candid":"custom/candid/file/main.did", + "build": "./node_modules/.bin/webpack --mode production" + } + } + } + "#; + let canister_declarations = DFX_JSON_WITH_CUSTOM_CANISTER_TYPE + .parse::() + .unwrap(); + let canister_declarations = canister_declarations + .get("canisters") + .unwrap() + .as_object() + .unwrap() + .into_iter() + .collect::>(); + + // first canister declaration + let (canister_name, canister_declaration) = canister_declarations[0]; + let canister_declaration = canister_declaration.as_object().unwrap(); + let canister_type = canister_declaration + .get("type") + .unwrap() + .as_str() + .unwrap() + .to_owned(); + let (extension_name, custom_canister_type) = + super::custom_canister_type::get_extension_name_and_custom_canister_type(&canister_type); + let processed_canister_declaration = super::custom_canister_type::process_canister_declaration( + &mut canister_declaration.clone(), + extension_name, + &extension_manifest, + canister_name, + custom_canister_type, + ) + .unwrap(); + let expected_output = serde_json::from_str::>( + r#" + { + "build": "./node_modules/.bin/webpack --mode production", + "candid":"custom/candid/file/main.did", + "gzip": true, + "main": "path/to/file/main.ts", + "type": "custom", + "wasm": "custom/wasm/file/main.wasm" + } + "#, + ) + .unwrap() + .clone(); + + assert_eq!(processed_canister_declaration, expected_output); + + // second canister declaration + let (canister_name, canister_declaration) = canister_declarations[1]; + let canister_declaration = canister_declaration.as_object().unwrap(); + let canister_type = canister_declaration + .get("type") + .unwrap() + .as_str() + .unwrap() + .to_owned(); + let (extension_name, custom_canister_type) = + super::custom_canister_type::get_extension_name_and_custom_canister_type(&canister_type); + let processed_canister_declaration = super::custom_canister_type::process_canister_declaration( + &mut canister_declaration.clone(), + extension_name, + &extension_manifest, + canister_name, + custom_canister_type, + ) + .unwrap(); + let expected_output = serde_json::from_str::>( + r#" + { + "build": "npx azyl azyl_frontend", + "candid": "src/main.did", + "gzip": false, + "main": "src/main.ts", + "root": "src", + "ts": "src/main.ts", + "type": "custom", + "wasm": ".azyl/azyl_frontend/azyl_frontend.wasm.gz" + } + "#, + ) + .unwrap() + .clone(); + assert_eq!(processed_canister_declaration, expected_output); } diff --git a/src/dfx-core/src/extension/manifest/mod.rs b/src/dfx-core/src/extension/manifest/mod.rs index 40023b3550..1d63c65f9b 100644 --- a/src/dfx-core/src/extension/manifest/mod.rs +++ b/src/dfx-core/src/extension/manifest/mod.rs @@ -12,3 +12,5 @@ pub mod extension; pub use extension::ExtensionManifest; /// File name for the file describing the extension. pub use extension::MANIFEST_FILE_NAME; + +pub mod custom_canister_type; diff --git a/src/dfx-core/src/extension/mod.rs b/src/dfx-core/src/extension/mod.rs index b0041c885f..e4b5082ad0 100644 --- a/src/dfx-core/src/extension/mod.rs +++ b/src/dfx-core/src/extension/mod.rs @@ -33,7 +33,7 @@ impl Extension { self, manager: &ExtensionManager, ) -> Result { - let manifest = ExtensionManifest::new(&self.name, &manager.dir)?; + let manifest = ExtensionManifest::get(&self.name, manager)?; let cmd = Command::new(&self.name) .bin_name(&self.name) // don't accept unknown options diff --git a/src/dfx-core/src/network/provider.rs b/src/dfx-core/src/network/provider.rs index 4d39284ab0..1461417958 100644 --- a/src/dfx-core/src/network/provider.rs +++ b/src/dfx-core/src/network/provider.rs @@ -588,7 +588,9 @@ mod tests { .unwrap(); } - let config = Config::from_dir(&project_dir).unwrap().unwrap(); + let config = Config::from_dir(&project_dir, &semver::Version::new(0, 0, 0)) + .unwrap() + .unwrap(); let network_descriptor = create_network_descriptor( Some(Arc::new(config)), Arc::new(NetworksConfig::new().unwrap()), @@ -617,6 +619,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -648,6 +651,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -668,6 +672,7 @@ mod tests { "networks": { } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); let network_descriptor = create_network_descriptor( @@ -693,6 +698,7 @@ mod tests { let config = Config::from_str( r#"{ }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); let network_descriptor = create_network_descriptor( @@ -730,6 +736,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -773,6 +780,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -816,6 +824,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -857,6 +866,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -899,6 +909,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -946,6 +957,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -986,6 +998,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); diff --git a/src/dfx/Cargo.toml b/src/dfx/Cargo.toml index 11c755ce20..b6486f9710 100644 --- a/src/dfx/Cargo.toml +++ b/src/dfx/Cargo.toml @@ -57,7 +57,7 @@ flate2 = { workspace = true, default-features = false, features = ["zlib-ng"] } fn-error-context = "0.2.0" futures-util = "0.3.21" futures.workspace = true -handlebars = "4.3.3" +handlebars.workspace = true hex = { workspace = true, features = ["serde"] } humantime.workspace = true hyper-rustls = { version = "0.24.1", features = ["webpki-roots", "http2"] } @@ -82,8 +82,8 @@ patch = "0.7.0" pem.workspace = true petgraph = "0.6.0" rand = "0.8.5" -regex = "1.5.5" -reqwest = { version = "0.11.9", default-features = false, features = [ +regex.workspace = true +reqwest = { workspace = true, default-features = false, features = [ "blocking", "json", "rustls-tls", diff --git a/src/dfx/src/lib/environment.rs b/src/dfx/src/lib/environment.rs index f1056a72ee..b7b86a2c24 100644 --- a/src/dfx/src/lib/environment.rs +++ b/src/dfx/src/lib/environment.rs @@ -99,7 +99,7 @@ pub struct EnvironmentImpl { impl EnvironmentImpl { pub fn new() -> DfxResult { let shared_networks_config = NetworksConfig::new()?; - let config = Config::from_current_dir()?; + let config = Config::from_current_dir(dfx_version())?; if let Some(ref config) = config { let temp_dir = config.get_temp_path(); create_dir_all(&temp_dir).with_context(|| { From 518c32d9dec80e84fef7f03b483bb0d364c9fbda Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Mon, 30 Oct 2023 17:12:11 +0100 Subject: [PATCH 2/5] extend unit tests --- .../manifest/custom_canister_type.rs | 146 ++++++++++++++++-- 1 file changed, 135 insertions(+), 11 deletions(-) diff --git a/src/dfx-core/src/extension/manifest/custom_canister_type.rs b/src/dfx-core/src/extension/manifest/custom_canister_type.rs index 8501dd9aee..4a6414de70 100644 --- a/src/dfx-core/src/extension/manifest/custom_canister_type.rs +++ b/src/dfx-core/src/extension/manifest/custom_canister_type.rs @@ -184,19 +184,21 @@ impl CustomCanisterTypeDeclaration { _ => {} } } + // Removing fields should be done last because of the order of the fields in the map. // It's easier to do in second for loop than to sort Ops beforehand, bacause Op would need to implement PartialOrd, // which is not possible, because serde_json::Number does not implement it. for field_name in remove_fields { final_fields.remove(&field_name); } + // Override custom canister declaration values by the real canister_declaration + // see: https://github.com/dfinity/sdk/pull/3222#issuecomment-1624073606 for (key, value) in values.iter() { if key != "type" && key != "canister_name" { final_fields.insert(key.clone(), value.clone()); } } - Ok(final_fields) } } @@ -233,28 +235,132 @@ mod tests { };} #[test] - fn test_op_replace() {} + fn test_op_replace_basic() { + test_op!( + custom_canister_template = r#" + { + "main": { "replace": { "input": "{{canister_name}}", "search": "frontend_(.*)", "output": "thecanister/$1/main.ts" } } + } + "#, + dfx_json_canister_values = r#" + { + "canister_name": "frontend_xyz" + } + "#, + expected = r#" + { + "main": "thecanister/xyz/main.ts" + } + "# + ); + } + + #[test] + fn test_op_replace_nested() { + test_op!( + custom_canister_template = r#" + { + "main": { "replace": { "input": "{{main}}", "search": "(.*)", "output": "thecanister/$1" } } + } + "#, + dfx_json_canister_values = r#" + { + "main": "src/main.ts" + } + "#, + expected = r#" + { + "main": "thecanister/src/main.ts" + } + "# + ); + } #[test] fn test_op_remove() { test_op!( custom_canister_template = r#" { - "type": "{{canister_name}}", "main": "src/main.ts", "main": { "remove": true } } "#, dfx_json_canister_values = r#" { - "canister_name": "something", - "main": "oowee.exe" + "main": "thecanister.exe" } "#, expected = r#" { - "main": "oowee.exe", - "type": "something" + "main": "thecanister.exe" + } + "# + ); + } + + #[test] + fn test_op_replacement_1() { + test_op!( + custom_canister_template = r#" + { + "main": "something.py" + } + "#, + dfx_json_canister_values = r#" + { + "gzip": true + } + "#, + expected = r#" + { + "gzip": true, + "main": "something.py" + } + "# + ); + } + + #[test] + fn test_op_replacement_2() { + test_op!( + custom_canister_template = r#" + { + "main": "something.py", + "gzip": false + } + "#, + dfx_json_canister_values = r#" + { + "gzip": true + } + "#, + expected = r#" + { + "gzip": true, + "main": "something.py" + } + "# + ); + } + + #[test] + fn test_op_replacement_3() { + test_op!( + custom_canister_template = r#" + { + "main": "{{gzip}}.py", + "gzip": false + } + "#, + dfx_json_canister_values = r#" + { + "gzip": true + } + "#, + expected = r#" + { + "gzip": true, + "main": "true.py" } "# ); @@ -265,19 +371,16 @@ mod tests { test_op!( custom_canister_template = r#" { - "type": "{{canister_name}}", - "main": "src/main.ts" + "type": "{{canister_name}}" } "#, dfx_json_canister_values = r#" { - "type": "custom", "canister_name": "something" } "#, expected = r#" { - "main": "src/main.ts", "type": "something" } "# @@ -328,6 +431,27 @@ mod tests { ); } + #[test] + fn test_overwrite() { + test_op!( + custom_canister_template = r#" + { + "gzip": true + } + "#, + dfx_json_canister_values = r#" + { + "gzip": false + } + "#, + expected = r#" + { + "gzip": false + } + "# + ); + } + #[test] fn test_ops() { test_op!( From 0c5c0b80e78147aa22b191f2ad91bb0b0461d28d Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Tue, 21 Nov 2023 17:19:31 +0100 Subject: [PATCH 3/5] lints, and more tests --- e2e/tests-dfx/extension.bash | 22 +- .../manifest/custom_canister_type.rs | 226 +++++++++++++++--- 2 files changed, 203 insertions(+), 45 deletions(-) diff --git a/e2e/tests-dfx/extension.bash b/e2e/tests-dfx/extension.bash index 0c67aac7b6..783da12d0c 100644 --- a/e2e/tests-dfx/extension.bash +++ b/e2e/tests-dfx/extension.bash @@ -215,12 +215,8 @@ EOF CACHE_DIR=$(dfx cache show) mkdir -p "$CACHE_DIR"/extensions/playground - echo '#!/usr/bin/env bash - -echo testoutput' > "$CACHE_DIR"/extensions/playground/playground - chmod +x "$CACHE_DIR"/extensions/playground/playground - - echo '{ + cat > "$CACHE_DIR"/extensions/playground/extension.json < "$CACHE_DIR"/extensions/playground/playground "gzip": false } } -}' > "$CACHE_DIR"/extensions/playground/extension.json +} +EOF + cat > "$CACHE_DIR"/extensions/playground/playground < "$CACHE_DIR"/extensions/playground/playground create_networks_json install_asset playground_backend - echo '{ + cat > dfx.json < "$CACHE_DIR"/extensions/playground/playground }, "output_env_file": ".env", "version": 1 -}' > dfx.json +} +EOF dfx_start assert_command dfx deploy -v diff --git a/src/dfx-core/src/extension/manifest/custom_canister_type.rs b/src/dfx-core/src/extension/manifest/custom_canister_type.rs index 4a6414de70..8bf0a77658 100644 --- a/src/dfx-core/src/extension/manifest/custom_canister_type.rs +++ b/src/dfx-core/src/extension/manifest/custom_canister_type.rs @@ -111,11 +111,11 @@ pub struct CustomCanisterTypeDeclaration(BTreeMap); #[derive(Clone, Serialize, Deserialize, Debug)] #[serde(untagged)] enum Op { - Replace { replace: Replace }, - Remove { remove: bool }, - Template(String), BoolValue(bool), NumberValue(serde_json::Number), + Remove { remove: bool }, + Replace { replace: Replace }, + Template(String), } #[cfg_attr(test, derive(Eq, PartialEq))] @@ -157,17 +157,10 @@ impl CustomCanisterTypeDeclaration { .into_iter() { match op { - Op::NumberValue(x) => { - final_fields.insert(field_name, x.into()); - } Op::BoolValue(x) => { final_fields.insert(field_name, x.into()); } - - Op::Template(template) => { - let x = Handlebars::new() - .render_template(&template, &values) - .map_err(OpError::InvalidTemplate)?; + Op::NumberValue(x) => { final_fields.insert(field_name, x.into()); } Op::Replace { replace } => { @@ -181,6 +174,12 @@ impl CustomCanisterTypeDeclaration { Op::Remove { remove } if remove => { remove_fields.push(field_name); } + Op::Template(template) => { + let x = Handlebars::new() + .render_template(&template, &values) + .map_err(OpError::InvalidTemplate)?; + final_fields.insert(field_name, x.into()); + } _ => {} } } @@ -194,10 +193,9 @@ impl CustomCanisterTypeDeclaration { // Override custom canister declaration values by the real canister_declaration // see: https://github.com/dfinity/sdk/pull/3222#issuecomment-1624073606 - for (key, value) in values.iter() { - if key != "type" && key != "canister_name" { - final_fields.insert(key.clone(), value.clone()); - } + let skip_keys = ["type", "canister_name"].map(String::from); + for (key, value) in values.iter().filter(|(k, _)| !skip_keys.contains(&k)) { + final_fields.insert(key.clone(), value.clone()); } Ok(final_fields) } @@ -235,76 +233,144 @@ mod tests { };} #[test] - fn test_op_replace_basic() { + fn test_op_replace_1() { test_op!( custom_canister_template = r#" { - "main": { "replace": { "input": "{{canister_name}}", "search": "frontend_(.*)", "output": "thecanister/$1/main.ts" } } + "main": "something.py" } "#, dfx_json_canister_values = r#" { - "canister_name": "frontend_xyz" + "gzip": true } "#, expected = r#" { - "main": "thecanister/xyz/main.ts" + "gzip": true, + "main": "something.py" } "# ); } #[test] - fn test_op_replace_nested() { + fn test_op_replace_2() { test_op!( custom_canister_template = r#" { - "main": { "replace": { "input": "{{main}}", "search": "(.*)", "output": "thecanister/$1" } } + "main": "something.py", + "gzip": false } "#, dfx_json_canister_values = r#" { - "main": "src/main.ts" + "gzip": true } "#, expected = r#" { - "main": "thecanister/src/main.ts" + "gzip": true, + "main": "something.py" } "# ); } #[test] - fn test_op_remove() { + fn test_op_replace_3() { test_op!( custom_canister_template = r#" { - "main": "src/main.ts", - "main": { "remove": true } + "main": "{{gzip}}.py", + "gzip": false } "#, dfx_json_canister_values = r#" { - "main": "thecanister.exe" + "gzip": true } "#, expected = r#" { - "main": "thecanister.exe" + "gzip": true, + "main": "true.py" } "# ); } #[test] - fn test_op_replacement_1() { + fn test_op_replace_4() { test_op!( custom_canister_template = r#" + { + "main": "path/to/{{main}}" + } + "#, + dfx_json_canister_values = r#" { "main": "something.py" } + "#, + expected = r#" + { + "main": "path/to/something.py" + } + "# + ); + } + + #[test] + fn test_op_replace_5() { + test_op!( + custom_canister_template = r#" + { + "main": { "replace": { "input": "{{main}}", "search": ".*/(.*).ts", "output": "thecanister/$1.exe" } } + } + "#, + dfx_json_canister_values = r#" + { + "main": "src/main.ts" + } + "#, + expected = r#" + { + "main": "thecanister/main.exe" + } + "# + ); + } + + #[test] + fn test_op_replace_6() { + test_op!( + custom_canister_template = r#" + { + "main": { "replace": { "input": "{{canister_name}}", "search": "frontend_(.*)", "output": "thecanister/$1/main.ts" } } + } + "#, + dfx_json_canister_values = r#" + { + "canister_name": "frontend_xyz" + } + "#, + expected = r#" + { + "main": "thecanister/xyz/main.ts" + } + "# + ); + } + + #[test] + fn test_op_replace_and_delete_1() { + test_op!( + custom_canister_template = r#" + { + "main": "something.py", + "gzip": { "remove": true } + } "#, dfx_json_canister_values = r#" { @@ -313,7 +379,6 @@ mod tests { "#, expected = r#" { - "gzip": true, "main": "something.py" } "# @@ -321,12 +386,13 @@ mod tests { } #[test] - fn test_op_replacement_2() { + fn test_op_replace_and_delete_2() { test_op!( custom_canister_template = r#" { "main": "something.py", - "gzip": false + "gzip": false, + "gzip": { "remove": true } } "#, dfx_json_canister_values = r#" @@ -336,7 +402,7 @@ mod tests { "#, expected = r#" { - "gzip": true, + "gzip": false, "main": "something.py" } "# @@ -344,12 +410,13 @@ mod tests { } #[test] - fn test_op_replacement_3() { + fn test_op_replace_and_delete_3() { test_op!( custom_canister_template = r#" { "main": "{{gzip}}.py", - "gzip": false + "gzip": false, + "gzip": { "remove": true } } "#, dfx_json_canister_values = r#" @@ -359,13 +426,100 @@ mod tests { "#, expected = r#" { - "gzip": true, + "gzip": false, "main": "true.py" } "# ); } + #[test] + fn test_op_replace_and_delete_4() { + test_op!( + custom_canister_template = r#" + { + "main": "path/to/{{main}}", + "main": { "remove": true } + } + "#, + dfx_json_canister_values = r#" + { + "main": "something.py" + } + "#, + expected = r#" + { + "main": "path/to/something.py" + } + "# + ); + } + + #[test] + fn test_op_replace_and_delete_5() { + test_op!( + custom_canister_template = r#" + { + "main": { "replace": { "input": "{{main}}", "search": ".*/(.*).ts", "output": "thecanister/$1.exe" } }, + "main": { "remove": true } + } + "#, + dfx_json_canister_values = r#" + { + "main": "src/main.ts" + } + "#, + expected = r#" + { + "main": "thecanister/main.exe" + } + "# + ); + } + + #[test] + fn test_op_replace_and_delete_6() { + test_op!( + custom_canister_template = r#" + { + "main": { "replace": { "input": "{{canister_name}}", "search": "frontend_(.*)", "output": "thecanister/$1/main.ts" } } + } + "#, + dfx_json_canister_values = r#" + { + "canister_name": "frontend_xyz" + } + "#, + expected = r#" + { + "main": "thecanister/xyz/main.ts" + } + "# + ); + } + + #[test] + fn test_op_remove() { + test_op!( + custom_canister_template = r#" + { + "main": "src/main.ts", + "main": { "remove": true } + } + "#, + dfx_json_canister_values = r#" + { + "main": "thecanister.exe" + } + "#, + expected = r#" + { + "main": "thecanister.exe" + } + "# + ); + } + #[test] fn test_op_template() { test_op!( From 4ccdc9e76abe14651431a15357d5a2eaa54e0877 Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Tue, 21 Nov 2023 17:47:51 +0100 Subject: [PATCH 4/5] lint --- e2e/tests-dfx/extension.bash | 5 ++--- src/dfx-core/src/extension/manifest/custom_canister_type.rs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/e2e/tests-dfx/extension.bash b/e2e/tests-dfx/extension.bash index 5fa0511de4..1f13fafe7c 100644 --- a/e2e/tests-dfx/extension.bash +++ b/e2e/tests-dfx/extension.bash @@ -237,7 +237,7 @@ EOF "build": "echo the wasm-utils canister is prebuilt", "candid": "{{canister_name}}.did", "wasm": "{{canister_name}}.wasm", - "gzip": false + "gzip": true } } } @@ -259,8 +259,7 @@ EOF { "canisters": { "wasm-utils": { - "type": "playground", - "gzip": true + "type": "playground" } }, "defaults": { diff --git a/src/dfx-core/src/extension/manifest/custom_canister_type.rs b/src/dfx-core/src/extension/manifest/custom_canister_type.rs index 8bf0a77658..6714b54060 100644 --- a/src/dfx-core/src/extension/manifest/custom_canister_type.rs +++ b/src/dfx-core/src/extension/manifest/custom_canister_type.rs @@ -194,7 +194,7 @@ impl CustomCanisterTypeDeclaration { // Override custom canister declaration values by the real canister_declaration // see: https://github.com/dfinity/sdk/pull/3222#issuecomment-1624073606 let skip_keys = ["type", "canister_name"].map(String::from); - for (key, value) in values.iter().filter(|(k, _)| !skip_keys.contains(&k)) { + for (key, value) in values.iter().filter(|(k, _)| !skip_keys.contains(k)) { final_fields.insert(key.clone(), value.clone()); } Ok(final_fields) From 984524cf0dfb8261d76b7a345a2ae1d3fd45dc0c Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Wed, 29 Nov 2023 18:27:45 +0100 Subject: [PATCH 5/5] refactor/error-types --- src/dfx-core/src/config/model/dfinity.rs | 67 ++++++++++--------- src/dfx-core/src/error/load_dfx_config.rs | 8 +-- src/dfx-core/src/error/structured_file.rs | 18 +++++ .../manifest/custom_canister_type.rs | 60 ++++++++++------- .../src/extension/manifest/extension.rs | 2 +- src/dfx-core/src/extension/mod.rs | 2 +- src/dfx-core/src/network/provider.rs | 15 +---- src/dfx/src/lib/environment.rs | 4 +- src/dfx/src/main.rs | 8 +-- 9 files changed, 104 insertions(+), 80 deletions(-) diff --git a/src/dfx-core/src/config/model/dfinity.rs b/src/dfx-core/src/config/model/dfinity.rs index ad5f40cba7..db396ead79 100644 --- a/src/dfx-core/src/config/model/dfinity.rs +++ b/src/dfx-core/src/config/model/dfinity.rs @@ -29,11 +29,13 @@ use crate::error::socket_addr_conversion::SocketAddrConversionError; use crate::error::socket_addr_conversion::SocketAddrConversionError::{ EmptyIterator, ParseSocketAddrFailed, }; -use crate::error::structured_file::StructuredFileError; use crate::error::structured_file::StructuredFileError::{ DeserializeJsonFileFailed, ReadJsonFileFailed, }; -use crate::extension::manifest::custom_canister_type; +use crate::error::structured_file::{ + ReadConfigurationError, StructuredFileError, TransformConfigurationError, +}; +use crate::extension::manifest::custom_canister_type::TransformConfiguration; use crate::json::save_json_file; use crate::json::structure::{PossiblyStr, SerdeVec}; use byte_unit::Byte; @@ -952,44 +954,45 @@ impl Config { Ok(None) } - fn from_file( + fn from_file( path: &Path, - dfx_version: &semver::Version, - ) -> Result { + transformer: &mut T, + ) -> Result { let content = crate::fs::read(path).map_err(ReadJsonFileFailed)?; - Config::from_slice(path.to_path_buf(), &content, dfx_version) + Config::from_slice(path.to_path_buf(), &content, transformer) } - pub fn from_dir( + pub fn from_dir( working_dir: &Path, - dfx_version: &semver::Version, + transformer: &mut T, ) -> Result, LoadDfxConfigError> { let path = Config::resolve_config_path(working_dir)?; - path.map(|path| Config::from_file(&path, dfx_version)) + path.map(|path| Config::from_file(&path, transformer)) .transpose() .map_err(LoadFromFileFailed) } - pub fn from_current_dir( - dfx_version: &semver::Version, + pub fn from_current_dir( + transformer: &mut T, ) -> Result, LoadDfxConfigError> { Config::from_dir( &std::env::current_dir().map_err(DetermineCurrentWorkingDirFailed)?, - dfx_version, + transformer, ) } - fn from_slice( + fn from_slice( path: PathBuf, content: &[u8], - dfx_version: &semver::Version, - ) -> Result { + transformer: &mut T, + ) -> Result { let mut json: serde_json::Value = serde_json::from_slice(content) .map_err(|e| DeserializeJsonFileFailed(Box::new(path.clone()), e))?; - let extension_manager = - crate::extension::manager::ExtensionManager::new(dfx_version).unwrap(); - custom_canister_type::transform_dfx_json_via_extension(&mut json, extension_manager) - .unwrap(); // TODO: error handling + + transformer + .transform(&mut json) + .map_err(TransformConfigurationError::from)?; + let config = serde_json::from_value(json.clone()) .map_err(|e| DeserializeJsonFileFailed(Box::new(path.clone()), e))?; Ok(Config { path, json, config }) @@ -997,20 +1000,24 @@ impl Config { /// Create a configuration from a string. #[cfg(test)] - pub(crate) fn from_str( - content: &str, - dfx_version: &semver::Version, - ) -> Result { - Config::from_slice(PathBuf::from("-"), content.as_bytes(), dfx_version) + pub(crate) fn from_str(content: &str) -> Result { + let mut no_op_transformer = + crate::extension::manifest::custom_canister_type::NoopTransformConfiguration; + Config::from_slice( + PathBuf::from("-"), + content.as_bytes(), + &mut no_op_transformer, + ) } #[cfg(test)] pub(crate) fn from_str_and_path( path: PathBuf, content: &str, - dfx_version: &semver::Version, - ) -> Result { - Config::from_slice(path, content.as_bytes(), dfx_version) + ) -> Result { + let mut no_op_transformer = + crate::extension::manifest::custom_canister_type::NoopTransformConfiguration; + Config::from_slice(path, content.as_bytes(), &mut no_op_transformer) } pub fn get_path(&self) -> &PathBuf { @@ -1257,7 +1264,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1280,7 +1286,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1302,7 +1307,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1325,7 +1329,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1359,7 +1362,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1383,7 +1385,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); let config_interface = config_no_values.get_config(); diff --git a/src/dfx-core/src/error/load_dfx_config.rs b/src/dfx-core/src/error/load_dfx_config.rs index 674d93954e..92c39e484b 100644 --- a/src/dfx-core/src/error/load_dfx_config.rs +++ b/src/dfx-core/src/error/load_dfx_config.rs @@ -1,5 +1,5 @@ use crate::error::fs::FsError; -use crate::error::structured_file::StructuredFileError; +use crate::error::structured_file::ReadConfigurationError; use thiserror::Error; #[derive(Error, Debug)] @@ -7,9 +7,9 @@ pub enum LoadDfxConfigError { #[error("Failed to resolve config path: {0}")] ResolveConfigPathFailed(FsError), - #[error("Failed to load dfx configuration: {0}")] - LoadFromFileFailed(StructuredFileError), - #[error("Failed to determine current working dir: {0}")] DetermineCurrentWorkingDirFailed(std::io::Error), + + #[error("Failed to load dfx configuration: {0}")] + LoadFromFileFailed(ReadConfigurationError), } diff --git a/src/dfx-core/src/error/structured_file.rs b/src/dfx-core/src/error/structured_file.rs index 375c19be63..b5c9e71954 100644 --- a/src/dfx-core/src/error/structured_file.rs +++ b/src/dfx-core/src/error/structured_file.rs @@ -2,6 +2,8 @@ use crate::error::fs::FsError; use std::path::PathBuf; use thiserror::Error; +use super::extension::ExtensionError; + #[derive(Error, Debug)] pub enum StructuredFileError { #[error("Failed to parse contents of {0} as json: {1}")] @@ -16,3 +18,19 @@ pub enum StructuredFileError { #[error("Failed to write JSON file: {0}")] WriteJsonFileFailed(FsError), } + +#[derive(Error, Debug)] +pub enum ReadConfigurationError { + #[error(transparent)] + StructuredFile(#[from] StructuredFileError), + #[error(transparent)] + TransformConfiguration(#[from] TransformConfigurationError), +} + +#[derive(Error, Debug)] +pub enum TransformConfigurationError { + #[error("Configuration transformation failed: {0}")] + ConfigurationTransformationFailed(String), // Or another error type if necessary + #[error("Extension error: {0}")] + ExtensionError(#[from] ExtensionError), // Note that `from` here allows automatic conversion +} diff --git a/src/dfx-core/src/extension/manifest/custom_canister_type.rs b/src/dfx-core/src/extension/manifest/custom_canister_type.rs index 6714b54060..1d16e8992b 100644 --- a/src/dfx-core/src/extension/manifest/custom_canister_type.rs +++ b/src/dfx-core/src/extension/manifest/custom_canister_type.rs @@ -7,30 +7,34 @@ use serde::{Deserialize, Serialize}; use serde_json::{Map as JsonMap, Value as JsonValue}; use std::collections::BTreeMap; -pub(crate) fn transform_dfx_json_via_extension( - json: &mut JsonValue, - extension_manager: ExtensionManager, -) -> Result<(), ExtensionError> { - let canisters = match json.get_mut("canisters").and_then(|v| v.as_object_mut()) { - Some(canisters) => canisters, - None => return Ok(()), - }; - for (canister_name, canister_declaration) in canisters.iter_mut() { - if let Some(canister_type) = get_valid_canister_type(canister_declaration) { - let canister_declaration = canister_declaration.as_object_mut().unwrap(); - let (extension_name, canister_type) = - get_extension_name_and_custom_canister_type(&canister_type); - let extension_manifest = ExtensionManifest::get(extension_name, &extension_manager)?; - *canister_declaration = process_canister_declaration( - canister_declaration, - extension_name, - &extension_manifest, - canister_name, - canister_type, - )?; +pub trait TransformConfiguration { + fn transform(&mut self, json: &mut serde_json::Value) -> Result<(), ExtensionError>; +} + +impl TransformConfiguration for ExtensionManager { + fn transform(&mut self, json: &mut JsonValue) -> Result<(), ExtensionError> { + let canisters = match json.get_mut("canisters").and_then(|v| v.as_object_mut()) { + Some(canisters) => canisters, + None => return Ok(()), + }; + for (canister_name, canister_declaration) in canisters.iter_mut() { + if let Some(canister_type) = get_valid_canister_type(canister_declaration) { + let canister_declaration = canister_declaration.as_object_mut().unwrap(); + let (extension_name, canister_type) = + get_extension_name_and_custom_canister_type(&canister_type); + let extension_manifest = + ExtensionManifest::get_by_extension_name(extension_name, self)?; + *canister_declaration = process_canister_declaration( + canister_declaration, + extension_name, + &extension_manifest, + canister_name, + canister_type, + )?; + } } + Ok(()) } - Ok(()) } fn get_valid_canister_type(canister_declaration: &mut JsonValue) -> Option { @@ -202,7 +206,7 @@ impl CustomCanisterTypeDeclaration { } #[cfg(test)] -mod tests { +mod custom_canister_type_declaration_tests { use super::*; macro_rules! test_op { @@ -640,3 +644,13 @@ mod tests { ); } } + +#[cfg(test)] +pub struct NoopTransformConfiguration; +#[cfg(test)] +impl TransformConfiguration for NoopTransformConfiguration { + fn transform(&mut self, _: &mut serde_json::Value) -> Result<(), ExtensionError> { + // Do nothing + Ok(()) + } +} diff --git a/src/dfx-core/src/extension/manifest/extension.rs b/src/dfx-core/src/extension/manifest/extension.rs index e74c26f615..caffffcc74 100644 --- a/src/dfx-core/src/extension/manifest/extension.rs +++ b/src/dfx-core/src/extension/manifest/extension.rs @@ -25,7 +25,7 @@ pub struct ExtensionManifest { } impl ExtensionManifest { - pub fn get( + pub fn get_by_extension_name( extension_name: &str, extension_manager: &ExtensionManager, ) -> Result { diff --git a/src/dfx-core/src/extension/mod.rs b/src/dfx-core/src/extension/mod.rs index e4b5082ad0..988db47afb 100644 --- a/src/dfx-core/src/extension/mod.rs +++ b/src/dfx-core/src/extension/mod.rs @@ -33,7 +33,7 @@ impl Extension { self, manager: &ExtensionManager, ) -> Result { - let manifest = ExtensionManifest::get(&self.name, manager)?; + let manifest = ExtensionManifest::get_by_extension_name(&self.name, manager)?; let cmd = Command::new(&self.name) .bin_name(&self.name) // don't accept unknown options diff --git a/src/dfx-core/src/network/provider.rs b/src/dfx-core/src/network/provider.rs index 1461417958..d382869892 100644 --- a/src/dfx-core/src/network/provider.rs +++ b/src/dfx-core/src/network/provider.rs @@ -588,7 +588,9 @@ mod tests { .unwrap(); } - let config = Config::from_dir(&project_dir, &semver::Version::new(0, 0, 0)) + let mut no_op_transformer = + crate::extension::manifest::custom_canister_type::NoopTransformConfiguration; + let config = Config::from_dir(&project_dir, &mut no_op_transformer) .unwrap() .unwrap(); let network_descriptor = create_network_descriptor( @@ -619,7 +621,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -651,7 +652,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -672,7 +672,6 @@ mod tests { "networks": { } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); let network_descriptor = create_network_descriptor( @@ -698,7 +697,6 @@ mod tests { let config = Config::from_str( r#"{ }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); let network_descriptor = create_network_descriptor( @@ -736,7 +734,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -780,7 +777,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -824,7 +820,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -866,7 +861,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -909,7 +903,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -957,7 +950,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -998,7 +990,6 @@ mod tests { } } }"#, - &semver::Version::new(0, 0, 0), ) .unwrap(); diff --git a/src/dfx/src/lib/environment.rs b/src/dfx/src/lib/environment.rs index 6ee64479c1..267b534a6b 100644 --- a/src/dfx/src/lib/environment.rs +++ b/src/dfx/src/lib/environment.rs @@ -97,9 +97,9 @@ pub struct EnvironmentImpl { } impl EnvironmentImpl { - pub fn new() -> DfxResult { + pub fn new(extension_manager: &mut ExtensionManager) -> DfxResult { let shared_networks_config = NetworksConfig::new()?; - let config = Config::from_current_dir(dfx_version())?; + let config = Config::from_current_dir(extension_manager)?; if let Some(ref config) = config { let temp_dir = config.get_temp_path(); create_dir_all(&temp_dir).with_context(|| { diff --git a/src/dfx/src/main.rs b/src/dfx/src/main.rs index 712d2df3f3..eac923f437 100644 --- a/src/dfx/src/main.rs +++ b/src/dfx/src/main.rs @@ -164,7 +164,7 @@ fn main() { let mut args = std::env::args_os().collect::>(); let mut error_diagnosis: Diagnosis = NULL_DIAGNOSIS; - ExtensionManager::new(dfx_version()) + let mut extension_manager = ExtensionManager::new(dfx_version()) .and_then(|em| { let installed_extensions = em.installed_extensions_as_clap_commands()?; if !installed_extensions.is_empty() { @@ -179,7 +179,7 @@ fn main() { args.splice(idx..idx, ["extension", "run"].iter().map(OsString::from)); } } - Ok(()) + Ok(em) }) .unwrap_or_else(|err| { print_error_and_diagnosis(err.into(), error_diagnosis.clone()); @@ -191,10 +191,10 @@ fn main() { let identity = cli_opts.identity; let effective_canister_id = cli_opts.provisional_create_canister_effective_canister_id; let command = cli_opts.command; - let result = match EnvironmentImpl::new() { + let result = match EnvironmentImpl::new(&mut extension_manager) { Ok(env) => { maybe_redirect_dfx(env.get_version()).map_or((), |_| unreachable!()); - match EnvironmentImpl::new().map(|env| { + match EnvironmentImpl::new(&mut extension_manager).map(|env| { env.with_logger(log) .with_identity_override(identity) .with_verbose_level(verbose_level)