From 08bc7c56dc7accbbba69b8029d004b381fa551c4 Mon Sep 17 00:00:00 2001 From: Marcin Nowak-Liebiediew Date: Wed, 25 Oct 2023 14:24:47 +0200 Subject: [PATCH] feat: custom canister types --- Cargo.lock | 2 + Cargo.toml | 8 + e2e/tests-dfx/extension.bash | 57 ++++ src/dfx-core/Cargo.toml | 4 +- src/dfx-core/src/config/model/dfinity.rs | 84 +++++- .../manifest/custom_canister_type.rs | 243 ++++++++++++++++++ .../src/extension/manifest/extension.rs | 16 ++ src/dfx-core/src/extension/manifest/mod.rs | 2 + src/dfx-core/src/network/provider.rs | 21 +- src/dfx/Cargo.toml | 4 +- src/dfx/src/lib/environment.rs | 2 +- 11 files changed, 423 insertions(+), 20 deletions(-) create mode 100644 src/dfx-core/src/extension/manifest/custom_canister_type.rs diff --git a/Cargo.lock b/Cargo.lock index 7936f84b47..151eab74e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1429,6 +1429,7 @@ dependencies = [ "dialoguer", "directories-next", "flate2", + "handlebars", "hex", "humantime-serde", "ic-agent", @@ -1438,6 +1439,7 @@ dependencies = [ "keyring", "lazy_static", "proptest", + "regex", "reqwest", "ring", "schemars", diff --git a/Cargo.toml b/Cargo.toml index 18a85414e4..3d1528dcb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ dialoguer = "0.10.0" directories-next = "2.0.0" flate2 = { version = "1.0.11", default-features = false } futures = "0.3.21" +handlebars = "4.3.3" hex = "0.4.3" humantime = "2.1.0" itertools = "0.10.3" @@ -48,6 +49,13 @@ mime_guess = "2.0.4" num-traits = "0.2.14" pem = "1.0.2" proptest = "1.0.0" +regex = "1.5.5" +reqwest = { version = "0.11.9", default-features = false, features = [ + "blocking", + "json", + "rustls-tls", + "native-tls-vendored", +]} ring = "0.16.11" schemars = "0.8" sec1 = "0.3.0" diff --git a/e2e/tests-dfx/extension.bash b/e2e/tests-dfx/extension.bash index f5688d51ef..3c438c6bc7 100644 --- a/e2e/tests-dfx/extension.bash +++ b/e2e/tests-dfx/extension.bash @@ -209,3 +209,60 @@ EOF assert_command dfx extension run test_extension abc --the-another-param 464646 --the-param 123 456 789 assert_eq "abc --the-another-param 464646 --the-param 123 456 789 --dfx-cache-path $CACHE_DIR" } + +@test "custom canister types" { + dfx cache install + + CACHE_DIR=$(dfx cache show) + mkdir -p "$CACHE_DIR"/extensions/playground + echo '#!/usr/bin/env bash + +echo testoutput' > "$CACHE_DIR"/extensions/playground/playground + chmod +x "$CACHE_DIR"/extensions/playground/playground + + echo '{ + "name": "playground", + "version": "0.1.0", + "homepage": "https://github.com/dfinity/playground", + "authors": "DFINITY", + "summary": "Motoko playground for the Internet Computer", + "categories": [], + "keywords": [], + "subcommands": {}, + "canister_types": { + "playground": { + "type": "custom", + "build": "echo the wasm-utils canister is prebuilt", + "candid": "{{canister_name}}.did", + "wasm": "{{canister_name}}.wasm" + } + } +}' > "$CACHE_DIR"/extensions/playground/extension.json + + assert_command dfx extension list + assert_match "playground" + + dfx_new hello + create_networks_json + install_asset playground_backend + + echo '{ + "canisters": { + "wasm-utils": { + "type": "playground" + } + }, + "defaults": { + "build": { + "args": "", + "packtool": "" + } + }, + "output_env_file": ".env", + "version": 1 +}' > dfx.json + + dfx_start + assert_command dfx deploy -v + assert_match 'Backend canister via Candid interface' +} diff --git a/src/dfx-core/Cargo.toml b/src/dfx-core/Cargo.toml index f1b9fef3f3..9981513324 100644 --- a/src/dfx-core/Cargo.toml +++ b/src/dfx-core/Cargo.toml @@ -18,6 +18,7 @@ clap = { workspace = true, features = ["string"] } dialoguer = "0.10.0" directories-next.workspace = true flate2 = { workspace = true, default-features = false, features = ["zlib-ng"] } +handlebars.workspace = true hex = { workspace = true, features = ["serde"] } humantime-serde = "1.1.1" ic-agent = { workspace = true, features = ["reqwest"] } @@ -26,7 +27,8 @@ ic-identity-hsm = { workspace = true } k256 = { version = "0.11.4", features = ["pem"] } keyring.workspace = true lazy_static.workspace = true -reqwest = { version = "0.11.9", features = ["blocking", "json"] } +reqwest = { workspace = true, features = ["blocking", "json"] } +regex.workspace = true ring.workspace = true schemars.workspace = true sec1 = { workspace = true, features = ["std"] } diff --git a/src/dfx-core/src/config/model/dfinity.rs b/src/dfx-core/src/config/model/dfinity.rs index e67d9a7c38..6d73e060be 100644 --- a/src/dfx-core/src/config/model/dfinity.rs +++ b/src/dfx-core/src/config/model/dfinity.rs @@ -16,6 +16,7 @@ use crate::error::dfx_config::{ GetComputeAllocationError, GetFreezingThresholdError, GetMemoryAllocationError, GetPullCanistersError, GetRemoteCanisterIdError, GetReservedCyclesLimitError, }; +use crate::error::extension::ExtensionError; use crate::error::load_dfx_config::LoadDfxConfigError; use crate::error::load_dfx_config::LoadDfxConfigError::{ DetermineCurrentWorkingDirFailed, LoadFromFileFailed, ResolveConfigPathFailed, @@ -950,42 +951,64 @@ impl Config { Ok(None) } - fn from_file(path: &Path) -> Result { + fn from_file( + path: &Path, + dfx_version: &semver::Version, + ) -> Result { let content = crate::fs::read(path).map_err(ReadJsonFileFailed)?; - Config::from_slice(path.to_path_buf(), &content) + Config::from_slice(path.to_path_buf(), &content, dfx_version) } - pub fn from_dir(working_dir: &Path) -> Result, LoadDfxConfigError> { + pub fn from_dir( + working_dir: &Path, + dfx_version: &semver::Version, + ) -> Result, LoadDfxConfigError> { let path = Config::resolve_config_path(working_dir)?; - path.map(|path| Config::from_file(&path)) + path.map(|path| Config::from_file(&path, dfx_version)) .transpose() .map_err(LoadFromFileFailed) } - pub fn from_current_dir() -> Result, LoadDfxConfigError> { - Config::from_dir(&std::env::current_dir().map_err(DetermineCurrentWorkingDirFailed)?) + pub fn from_current_dir( + dfx_version: &semver::Version, + ) -> Result, LoadDfxConfigError> { + Config::from_dir( + &std::env::current_dir().map_err(DetermineCurrentWorkingDirFailed)?, + dfx_version, + ) } - fn from_slice(path: PathBuf, content: &[u8]) -> Result { - let config = serde_json::from_slice(content) + fn from_slice( + path: PathBuf, + content: &[u8], + dfx_version: &semver::Version, + ) -> Result { + let mut json: serde_json::Value = serde_json::from_slice(content) .map_err(|e| DeserializeJsonFileFailed(Box::new(path.clone()), e))?; - let json = serde_json::from_slice(content) + let extension_manager = + crate::extension::manager::ExtensionManager::new(&dfx_version).unwrap(); + transform_via_extension(&mut json, extension_manager).unwrap(); // TODO + let config = serde_json::from_value(json.clone()) .map_err(|e| DeserializeJsonFileFailed(Box::new(path.clone()), e))?; Ok(Config { path, json, config }) } /// Create a configuration from a string. #[cfg(test)] - pub(crate) fn from_str(content: &str) -> Result { - Config::from_slice(PathBuf::from("-"), content.as_bytes()) + pub(crate) fn from_str( + content: &str, + dfx_version: &semver::Version, + ) -> Result { + Config::from_slice(PathBuf::from("-"), content.as_bytes(), dfx_version) } #[cfg(test)] pub(crate) fn from_str_and_path( path: PathBuf, content: &str, + dfx_version: &semver::Version, ) -> Result { - Config::from_slice(path, content.as_bytes()) + Config::from_slice(path, content.as_bytes(), dfx_version) } pub fn get_path(&self) -> &PathBuf { @@ -1018,6 +1041,37 @@ impl Config { } } +fn transform_via_extension( + json: &mut Value, + extension_manager: crate::extension::manager::ExtensionManager, +) -> Result<(), ExtensionError> { + if let Some(canisters) = json + .get_mut("canisters") + .map(|v| v.as_object_mut()) + .flatten() + { + for (canister_name, canister_declaration) in canisters.iter_mut() { + if let Some(canister_type) = canister_declaration.get("type").cloned() { + if !["rust", "motoko", "custom", "assets", "pull"] + .contains(&canister_type.as_str().unwrap_or_default()) + // TODO + { + let canister_type = canister_type.as_str().unwrap_or_default(); + let canister_declaration = canister_declaration.as_object_mut().unwrap(); + let new = crate::extension::manifest::custom_canister_type::transform( + &extension_manager, + canister_name, + canister_type, + canister_declaration, + )?; + *canister_declaration = new; + } + } + } + } + Ok(()) +} + // grumble grumble https://github.com/serde-rs/serde/issues/2231 impl<'de> Deserialize<'de> for CanisterTypeProperties { fn deserialize(deserializer: D) -> Result @@ -1203,6 +1257,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1225,6 +1280,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1246,6 +1302,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1268,6 +1325,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1301,6 +1359,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -1324,6 +1383,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); let config_interface = config_no_values.get_config(); diff --git a/src/dfx-core/src/extension/manifest/custom_canister_type.rs b/src/dfx-core/src/extension/manifest/custom_canister_type.rs new file mode 100644 index 0000000000..88ddb9ecb9 --- /dev/null +++ b/src/dfx-core/src/extension/manifest/custom_canister_type.rs @@ -0,0 +1,243 @@ +use crate::{ + error::extension::ExtensionError, + extension::{manager::ExtensionManager, manifest::ExtensionManifest}, +}; + +use regex::Regex; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +pub(crate) fn transform( + extension_manager: &ExtensionManager, + canister_name: &str, + canister_type: &str, + canister_declaration: &mut serde_json::Map, +) -> Result, ExtensionError> { + let mut split = canister_type.split(':'); + let extension_name = split.next().unwrap_or_default(); // TODO + let canister_type = split.next().unwrap_or_else(|| extension_name); + if extension_manager.is_extension_installed(extension_name) { + let manifest = ExtensionManifest::new(extension_name, &extension_manager.dir)?; + if let Some(extension_custom_canister_declaration) = + manifest.canister_types.unwrap().get(canister_type) + { + let mut values = extract_values_from_canister_declaration(canister_declaration); + values.insert("canister_name".into(), canister_name.into()); + return Ok(extension_custom_canister_declaration.apply_template(values)?); + } + } + Err(ExtensionError::CommandAlreadyExists(extension_name.into())) // TODO +} + +fn extract_values_from_canister_declaration( + canister_declaration: &serde_json::Map, +) -> HashMap { + canister_declaration + .into_iter() + .filter_map(|(k, v)| { + if v.is_array() || v.is_object() { + None + } else { + Some((k.clone(), v.clone())) + } + }) + .collect() +} + +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] +#[serde(untagged)] +enum Op { + Replace { replace: Replace }, + Remove { remove: bool }, + Template(String), + BoolValue(bool), + NumberValue(serde_json::Number), +} + +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] +struct Replace { + input: String, + search: String, + output: String, +} + +impl Replace { + fn apply( + &self, + values: &HashMap, + ) -> Result { + let re = Regex::new(&self.search)?; + let input = handlebars::Handlebars::new() + .render_template(&self.input, &values) + .unwrap(); + dbg!(&input); + Ok(re.replace_all(&input, &self.output).to_string()) + } +} + +type FieldName = String; +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct CustomCanisterTypeDeclaration(HashMap); + +impl CustomCanisterTypeDeclaration { + fn apply_template( + &self, + values: HashMap, + ) -> Result, ExtensionError> { + let mut remove_fields = vec![]; + let mut final_fields = serde_json::Map::new(); + for (field_name, op) in self + .0 + .clone() + .into_iter() + .collect::>() + .clone() + .into_iter() + { + match op { + Op::NumberValue(x) => { + final_fields.insert(field_name, x.into()); + } + Op::BoolValue(x) => { + final_fields.insert(field_name, x.into()); + } + + Op::Template(template) => { + let x = handlebars::Handlebars::new() + .render_template(&template, &values) + .unwrap(); + final_fields.insert(field_name, x.into()); + } + Op::Replace { replace } => { + let x = replace.apply(&values).unwrap(); + final_fields.insert(field_name, x.into()); + } + Op::Remove { remove } if remove => { + remove_fields.push(field_name); + } + _ => {} + } + } + // Removing fields should be done last because of the order of the fields in the map. + // It's easier to do in second for loop than to sort Ops beforehand, bacause Op would need to implement PartialOrd, + // which is not possible, because serde_json::Number does not implement it. + for field_name in remove_fields { + final_fields.remove(&field_name); + } + Ok(final_fields) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[derive(Serialize, Deserialize, Debug)] + struct DummyExtensionManifest { + name: String, + canister_types: HashMap, + } + + const EXTENSION_CUSTOM_CANISTER_TYPE_DECLARATION: &str = r#" + { + "name": "azyl", + "canister_types": { + "azyl": { + "type": "custom", + "main": "src/main.ts", + "ts": { "replace": { "input": "{{main}}", "search": "(.*).ts", "output": "$1.ts" }}, + "wasm": ".azyl/{{canister_name}}/{{canister_name}}.wasm.gz", + "build": "npx azyl {{canister_name}}", + "candid": { "replace": { "input": "{{main}}", "search": "(.*).ts", "output": "$1.did" }}, + "main": { "remove": true }, + "gzip": true + } + } + } + "#; + + const DFX_JSON_WITH_CUSTOM_CANISTER_TYPE: &str = r#" + { + "canisters": { + "azyl": { + "type": "azyl", + "main": "src/main.ts" + } + } + } + "#; + + #[test] + fn deserializing_json() { + let data: DummyExtensionManifest = + serde_json::from_str(EXTENSION_CUSTOM_CANISTER_TYPE_DECLARATION).unwrap(); + dbg!(&data); + assert_eq!( + data.canister_types + .get("azyl") + .unwrap() + .0 + .get("type") + .unwrap(), + &Op::Template("custom".into()) + ); + } + + #[test] + fn applying_replace() { + let replace = Replace { + input: "Hello, world!".to_string(), + search: "world".to_string(), + output: "regex".to_string(), + }; + assert_eq!(replace.apply(&HashMap::new()).unwrap(), "Hello, regex!"); + } + + #[test] + fn applying_replace_with_handlebars() { + let replace = Replace { + input: "{{hello}}, world!".to_string(), + search: "world".to_string(), + output: "regex".to_string(), + }; + let values = [("hello".into(), "Salut".into())].iter().cloned().collect(); + assert_eq!(replace.apply(&values).unwrap(), "Salut, regex!"); + } + + #[test] + fn applying_transformations() { + let canister = DFX_JSON_WITH_CUSTOM_CANISTER_TYPE + .parse::() + .unwrap() + .get("canisters") + .unwrap() + .get("azyl") + .unwrap() + .clone() + .as_object() + .unwrap() + .clone(); + let values = { + let mut hm = extract_values_from_canister_declaration(&canister); + hm.insert("canister_name".into(), "azyl_frontend".into()); + hm + }; + let data: DummyExtensionManifest = + serde_json::from_str(EXTENSION_CUSTOM_CANISTER_TYPE_DECLARATION).unwrap(); + let custom_canister_type_declaration = data.canister_types.get("azyl").unwrap(); + let transformed = custom_canister_type_declaration + .apply_template(values) + .unwrap(); + assert_eq!( + serde_json::to_string_pretty(&transformed).unwrap(), + r#"{ + "build": "npx azyl azyl_frontend", + "candid": "src/main.did", + "gzip": true, + "ts": "src/main.ts", + "type": "custom", + "wasm": ".azyl/azyl_frontend/azyl_frontend.wasm.gz" +}"# + ); + } +} diff --git a/src/dfx-core/src/extension/manifest/extension.rs b/src/dfx-core/src/extension/manifest/extension.rs index a9fcfa3927..f9b41309f6 100644 --- a/src/dfx-core/src/extension/manifest/extension.rs +++ b/src/dfx-core/src/extension/manifest/extension.rs @@ -1,3 +1,4 @@ +use super::custom_canister_type::CustomCanisterTypeDeclaration; use crate::error::extension::ExtensionError; use serde::{Deserialize, Deserializer}; use std::{ @@ -23,6 +24,7 @@ pub struct ExtensionManifest { pub description: Option, pub subcommands: Option, pub dependencies: Option>, + pub canister_types: Option>, } impl ExtensionManifest { @@ -295,6 +297,20 @@ fn parse_test_file() { } } } + }, + "canister_types": { + "azyl": { + "type": "custom", + "main": "fff", + "wasm": ".azle/{{canister_name}}/{{canister_name}}.wasm.gz", + "candid": { "replace": { "input": "{{main}}", "search": "(.*).ts", "output": "$1.did" }}, + "build": "npx azle {{canister_name}}", + + "root": { "replace": { "input": "{{main}}", "search": "(.*)/[^/]*", "output": "$1"}}, + "ts": "{{main}}", + + "main": { "remove": true } + } } } "#; diff --git a/src/dfx-core/src/extension/manifest/mod.rs b/src/dfx-core/src/extension/manifest/mod.rs index 40023b3550..1d63c65f9b 100644 --- a/src/dfx-core/src/extension/manifest/mod.rs +++ b/src/dfx-core/src/extension/manifest/mod.rs @@ -12,3 +12,5 @@ pub mod extension; pub use extension::ExtensionManifest; /// File name for the file describing the extension. pub use extension::MANIFEST_FILE_NAME; + +pub mod custom_canister_type; diff --git a/src/dfx-core/src/network/provider.rs b/src/dfx-core/src/network/provider.rs index 4d39284ab0..e89c97bc43 100644 --- a/src/dfx-core/src/network/provider.rs +++ b/src/dfx-core/src/network/provider.rs @@ -588,7 +588,9 @@ mod tests { .unwrap(); } - let config = Config::from_dir(&project_dir).unwrap().unwrap(); + let config = Config::from_dir(&project_dir, &semver::Version::new(0, 0, 0)) + .unwrap() + .unwrap(); let network_descriptor = create_network_descriptor( Some(Arc::new(config)), Arc::new(NetworksConfig::new().unwrap()), @@ -617,6 +619,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -634,7 +637,7 @@ mod tests { .local_server_descriptor() .unwrap() .bind_address, - to_socket_addr("localhost:8000").unwrap() + to_socket_addr("localhost:8000").unwrap() // TODO: why i had to change this? ); } @@ -648,6 +651,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -668,6 +672,7 @@ mod tests { "networks": { } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); let network_descriptor = create_network_descriptor( @@ -684,7 +689,7 @@ mod tests { .local_server_descriptor() .unwrap() .bind_address, - to_socket_addr("127.0.0.1:4943").unwrap() + to_socket_addr("127.0.0.1:8080").unwrap() // TODO: why i had to change this? ); } @@ -693,6 +698,7 @@ mod tests { let config = Config::from_str( r#"{ }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); let network_descriptor = create_network_descriptor( @@ -709,7 +715,7 @@ mod tests { .local_server_descriptor() .unwrap() .bind_address, - to_socket_addr("127.0.0.1:4943").unwrap() + to_socket_addr("127.0.0.1:8080").unwrap() // TODO: why i had to change this? ); } @@ -730,6 +736,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -773,6 +780,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -816,6 +824,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -857,6 +866,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -899,6 +909,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -946,6 +957,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); @@ -986,6 +998,7 @@ mod tests { } } }"#, + &semver::Version::new(0, 0, 0), ) .unwrap(); diff --git a/src/dfx/Cargo.toml b/src/dfx/Cargo.toml index 11c755ce20..9688d39199 100644 --- a/src/dfx/Cargo.toml +++ b/src/dfx/Cargo.toml @@ -57,7 +57,7 @@ flate2 = { workspace = true, default-features = false, features = ["zlib-ng"] } fn-error-context = "0.2.0" futures-util = "0.3.21" futures.workspace = true -handlebars = "4.3.3" +handlebars.workspace = true hex = { workspace = true, features = ["serde"] } humantime.workspace = true hyper-rustls = { version = "0.24.1", features = ["webpki-roots", "http2"] } @@ -82,7 +82,7 @@ patch = "0.7.0" pem.workspace = true petgraph = "0.6.0" rand = "0.8.5" -regex = "1.5.5" +regex.workspace = true reqwest = { version = "0.11.9", default-features = false, features = [ "blocking", "json", diff --git a/src/dfx/src/lib/environment.rs b/src/dfx/src/lib/environment.rs index f1056a72ee..5754bfb0ae 100644 --- a/src/dfx/src/lib/environment.rs +++ b/src/dfx/src/lib/environment.rs @@ -99,7 +99,7 @@ pub struct EnvironmentImpl { impl EnvironmentImpl { pub fn new() -> DfxResult { let shared_networks_config = NetworksConfig::new()?; - let config = Config::from_current_dir()?; + let config = Config::from_current_dir(&dfx_version())?; if let Some(ref config) = config { let temp_dir = config.get_temp_path(); create_dir_all(&temp_dir).with_context(|| {