feat(extension): add support for custom canister types

Cargo.toml

@@ -38,6 +38,7 @@ dialoguer = "0.10.0"
 directories-next = "2.0.0"
 flate2 = { version = "1.0.11", default-features = false }
 futures = "0.3.21"
+handlebars = "4.3.3"
 hex = "0.4.3"
 humantime = "2.1.0"
 itertools = "0.10.3"
@@ -48,6 +49,13 @@ mime_guess = "2.0.4"
 num-traits = "0.2.14"
 pem = "1.0.2"
 proptest = "1.0.0"
+regex = "1.5.5"
+reqwest = { version = "0.11.9", default-features = false, features = [
+    "blocking",
+    "json",
+    "rustls-tls",
+    "native-tls-vendored",
+]}
 ring = "0.16.11"
 schemars = "0.8"
 sec1 = "0.3.0"

e2e/tests-dfx/build.bash

@@ -215,7 +215,7 @@ teardown() {
   jq '.canisters.e2e_project_backend.type="unknown_canister_type"' dfx.json | sponge dfx.json
   assert_command_fail dfx build
   # shellcheck disable=SC2016
-  assert_match 'unknown variant `unknown_canister_type`'
+  assert_match 'ExtensionNotInstalled("unknown_canister_type")' # TODO: concrete error type
 
   # If canister type is invalid, `dfx stop` fails
   jq '.canisters.e2e_project_backend.type="motoko"' dfx.json | sponge dfx.json

e2e/tests-dfx/extension.bash

@@ -215,3 +215,65 @@ EOF
   assert_command dfx extension run test_extension abc --the-another-param 464646 --the-param 123 456 789
   assert_eq "abc --the-another-param 464646 --the-param 123 456 789 --dfx-cache-path $CACHE_DIR"
 }
+
+@test "custom canister types" {
+    dfx cache install
+
+    CACHE_DIR=$(dfx cache show)
+    mkdir -p "$CACHE_DIR"/extensions/playground
+    cat > "$CACHE_DIR"/extensions/playground/extension.json <<EOF
+{
+  "name": "playground",
+  "version": "0.1.0",
+  "homepage": "https://github.com/dfinity/playground",
+  "authors": "DFINITY",
+  "summary": "Motoko playground for the Internet Computer",
+  "categories": [],
+  "keywords": [],
+  "subcommands": {},
+  "canister_types": {
+    "playground": {
+      "type": "custom",
+      "build": "echo the wasm-utils canister is prebuilt",
+      "candid": "{{canister_name}}.did",
+      "wasm": "{{canister_name}}.wasm",
+      "gzip": true
+    }
+  }
+}
+EOF
+    cat > "$CACHE_DIR"/extensions/playground/playground <<EOF
+#!/usr/bin/env bash
+echo testoutput
+EOF
+    chmod +x "$CACHE_DIR"/extensions/playground/playground
+
+    assert_command dfx extension list
+    assert_match "playground"
+
+    dfx_new hello
+    create_networks_json
+    install_asset playground_backend
+
+    cat > dfx.json <<EOF
+{
+  "canisters": {
+      "wasm-utils": {
+          "type": "playground"
+      }
+  },
+  "defaults": {
+    "build": {
+      "args": "",
+      "packtool": ""
+    }
+  },
+  "output_env_file": ".env",
+  "version": 1
+}
+EOF
+
+    dfx_start
+    assert_command dfx deploy -v
+    assert_match 'Backend canister via Candid interface'
+}

src/dfx-core/Cargo.toml

@@ -18,6 +18,7 @@ clap = { workspace = true, features = ["string"] }
 dialoguer = "0.10.0"
 directories-next.workspace = true
 flate2 = { workspace = true, default-features = false, features = ["zlib-ng"] }
+handlebars.workspace = true
 hex = { workspace = true, features = ["serde"] }
 humantime-serde = "1.1.1"
 ic-agent = { workspace = true, features = ["reqwest"] }
@@ -26,7 +27,8 @@ ic-identity-hsm = { workspace = true }
 k256 = { version = "0.11.4", features = ["pem"] }
 keyring.workspace = true
 lazy_static.workspace = true
-reqwest = { version = "0.11.9", features = ["blocking", "json"] }
+reqwest = { workspace = true, features = ["blocking", "json"] }
+regex.workspace = true
 ring.workspace = true
 schemars.workspace = true
 sec1 = { workspace = true, features = ["std"] }

src/dfx-core/src/config/model/dfinity.rs

@@ -29,10 +29,13 @@ use crate::error::socket_addr_conversion::SocketAddrConversionError;
 use crate::error::socket_addr_conversion::SocketAddrConversionError::{
     EmptyIterator, ParseSocketAddrFailed,
 };
-use crate::error::structured_file::StructuredFileError;
 use crate::error::structured_file::StructuredFileError::{
     DeserializeJsonFileFailed, ReadJsonFileFailed,
 };
+use crate::error::structured_file::{
+    ReadConfigurationError, StructuredFileError, TransformConfigurationError,
+};
+use crate::extension::manifest::custom_canister_type::TransformConfiguration;
 use crate::json::save_json_file;
 use crate::json::structure::{PossiblyStr, SerdeVec};
 use byte_unit::Byte;
@@ -951,42 +954,70 @@ impl Config {
         Ok(None)
     }
 
-    fn from_file(path: &Path) -> Result<Config, StructuredFileError> {
+    fn from_file<T: TransformConfiguration>(
+        path: &Path,
+        transformer: &mut T,
+    ) -> Result<Config, ReadConfigurationError> {
         let content = crate::fs::read(path).map_err(ReadJsonFileFailed)?;
-        Config::from_slice(path.to_path_buf(), &content)
+        Config::from_slice(path.to_path_buf(), &content, transformer)
     }
 
-    pub fn from_dir(working_dir: &Path) -> Result<Option<Config>, LoadDfxConfigError> {
+    pub fn from_dir<T: TransformConfiguration>(
+        working_dir: &Path,
+        transformer: &mut T,
+    ) -> Result<Option<Config>, LoadDfxConfigError> {
         let path = Config::resolve_config_path(working_dir)?;
-        path.map(|path| Config::from_file(&path))
+        path.map(|path| Config::from_file(&path, transformer))
             .transpose()
             .map_err(LoadFromFileFailed)
     }
 
-    pub fn from_current_dir() -> Result<Option<Config>, LoadDfxConfigError> {
-        Config::from_dir(&std::env::current_dir().map_err(DetermineCurrentWorkingDirFailed)?)
+    pub fn from_current_dir<T: TransformConfiguration>(
+        transformer: &mut T,
+    ) -> Result<Option<Config>, LoadDfxConfigError> {
+        Config::from_dir(
+            &std::env::current_dir().map_err(DetermineCurrentWorkingDirFailed)?,
+            transformer,
+        )
     }
 
-    fn from_slice(path: PathBuf, content: &[u8]) -> Result<Config, StructuredFileError> {
-        let config = serde_json::from_slice(content)
+    fn from_slice<T: TransformConfiguration>(
+        path: PathBuf,
+        content: &[u8],
+        transformer: &mut T,
+    ) -> Result<Config, ReadConfigurationError> {
+        let mut json: serde_json::Value = serde_json::from_slice(content)
             .map_err(|e| DeserializeJsonFileFailed(Box::new(path.clone()), e))?;
-        let json = serde_json::from_slice(content)
+
+        transformer
+            .transform(&mut json)
+            .map_err(TransformConfigurationError::from)?;
+
+        let config = serde_json::from_value(json.clone())
             .map_err(|e| DeserializeJsonFileFailed(Box::new(path.clone()), e))?;
         Ok(Config { path, json, config })
     }
 
     /// Create a configuration from a string.
     #[cfg(test)]
-    pub(crate) fn from_str(content: &str) -> Result<Config, StructuredFileError> {
-        Config::from_slice(PathBuf::from("-"), content.as_bytes())
+    pub(crate) fn from_str(content: &str) -> Result<Config, ReadConfigurationError> {
+        let mut no_op_transformer =
+            crate::extension::manifest::custom_canister_type::NoopTransformConfiguration;
+        Config::from_slice(
+            PathBuf::from("-"),
+            content.as_bytes(),
+            &mut no_op_transformer,
+        )
     }
 
     #[cfg(test)]
     pub(crate) fn from_str_and_path(
         path: PathBuf,
         content: &str,
-    ) -> Result<Config, StructuredFileError> {
-        Config::from_slice(path, content.as_bytes())
+    ) -> Result<Config, ReadConfigurationError> {
+        let mut no_op_transformer =
+            crate::extension::manifest::custom_canister_type::NoopTransformConfiguration;
+        Config::from_slice(path, content.as_bytes(), &mut no_op_transformer)
     }
 
     pub fn get_path(&self) -> &PathBuf {

src/dfx-core/src/error/extension.rs

@@ -94,4 +94,14 @@ pub enum ExtensionError {
 
     #[error("Extension exited with non-zero status code '{0}'.")]
     ExtensionExitedWithNonZeroStatus(i32),
+
+    // errors related to custom canister types
+    #[error("Extension '{0}' does not support any custom canister types.")]
+    ExtensionDoesNotSupportAnyCustomCanisterTypes(String),
+
+    #[error("Extension '{0}' does not support the specific custom canister type '{1}'.")]
+    ExtensionDoesNotSupportSpecificCustomCanisterType(String, String),
+
+    #[error("Failed to load custom canister type template for canister type '{0}' from extension '{1}': {2}")]
+    CustomCanisterTypeTemplateError(String, String, String),
 }

src/dfx-core/src/error/load_dfx_config.rs

@@ -1,15 +1,15 @@
 use crate::error::fs::FsError;
-use crate::error::structured_file::StructuredFileError;
+use crate::error::structured_file::ReadConfigurationError;
 use thiserror::Error;
 
 #[derive(Error, Debug)]
 pub enum LoadDfxConfigError {
     #[error("Failed to resolve config path: {0}")]
     ResolveConfigPathFailed(FsError),
 
-    #[error("Failed to load dfx configuration: {0}")]
-    LoadFromFileFailed(StructuredFileError),
-
     #[error("Failed to determine current working dir: {0}")]
     DetermineCurrentWorkingDirFailed(std::io::Error),
+
+    #[error("Failed to load dfx configuration: {0}")]
+    LoadFromFileFailed(ReadConfigurationError),
 }

src/dfx-core/src/error/structured_file.rs

@@ -2,6 +2,8 @@ use crate::error::fs::FsError;
 use std::path::PathBuf;
 use thiserror::Error;
 
+use super::extension::ExtensionError;
+
 #[derive(Error, Debug)]
 pub enum StructuredFileError {
     #[error("Failed to parse contents of {0} as json: {1}")]
@@ -16,3 +18,19 @@ pub enum StructuredFileError {
     #[error("Failed to write JSON file: {0}")]
     WriteJsonFileFailed(FsError),
 }
+
+#[derive(Error, Debug)]
+pub enum ReadConfigurationError {
+    #[error(transparent)]
+    StructuredFile(#[from] StructuredFileError),
+    #[error(transparent)]
+    TransformConfiguration(#[from] TransformConfigurationError),
+}
+
+#[derive(Error, Debug)]
+pub enum TransformConfigurationError {
+    #[error("Configuration transformation failed: {0}")]
+    ConfigurationTransformationFailed(String), // Or another error type if necessary
+    #[error("Extension error: {0}")]
+    ExtensionError(#[from] ExtensionError), // Note that `from` here allows automatic conversion
+}