Merge branch 'master' into ens/sdk-1784-exclude-remote-canisters

CHANGELOG.md

@@ -20,6 +20,11 @@ Added a flag `--replica` to `dfx start`. This flag currently has no effect.
 Once PocketIC becomes the default for `dfx start` this flag will start the replica instead.
 You can use the `--replica` flag already to write scripts that anticipate that change.
 
+### feat: extensions can define project templates
+
+An extension can define one or more project templates for `dfx new` to use.
+These can be new templates or replace the built-in project templates.
+
 ### fix: all commands with --all parameter skip remote canisters
 
 This affects the following commands:

docs/concepts/extension-defined-project-templates.md

@@ -0,0 +1,66 @@
+# Extension-Defined Project Templates
+
+## Overview
+
+An extension can define one or more project templates for `dfx new` to use.
+
+A project template is a set of files that `dfx new` copies or patches into a new project.
+
+For examples of project template files, see the [project_templates] directory in the SDK repository.
+
+# Specification
+
+The `project_templates` field in an extension's `extension.json` defines the project templates
+included in the extension.  It is an object field mapping `project template name -> project template properties`.
+These are the properties of a project template:
+
+| Field                        | Type                      | Description                                                                                          |
+|------------------------------|---------------------------|------------------------------------------------------------------------------------------------------|
+| `display`                    | String                    | Display name of the project template                                                                 |
+| `category`                   | String                    | Category for inclusion in `--backend` and `--frontend` CLI options, as well as interactive selection |
+| `requirements`               | Array of String           | Required project templates                                                                           |
+| `post_create`                | String or Array of String | Command(s) to run after adding the canister to the project                                           |
+| `post_create_spinner_message` | String                    | Message to display while running the post_create command                                            |
+| `post_create_failure_warning` | String                    | Warning to display if the post_create command fails                                                 |
+
+Within the files distributed with the extension, the project template files are
+located in the `project_templates/{project template name}` directory.
+
+## The `display` field
+
+The `display` field is a string that describes the project template.
+`dfx new` will use this value for interactive selection of project templates.
+
+## The `category` field
+
+The `category` field is an array of strings that categorize the project template.
+`dfx new` uses this field to determine whether to include this project template
+as an option for the `--backend` and `-frontend` flags, as well as in interactive setup.
+
+Valid values for the field:
+- `frontend`
+- `backend`
+- `extra`
+- `frontend-test`
+- `support`
+
+## The `requirements` field
+
+The `requirements` field lists any project templates that `dfx new` must apply before this project template.
+For example, many of the frontend templates depend on the `dfx_js_base` template, which adds
+package.json and tsconfig.json to the project.
+
+## The `post_create` field
+
+The `post_create` field specifies a command or commands to run after adding the project template files to the project.
+For example, the rust project template runs `cargo update` after adding the files.
+
+## The `post_create_spinner_message` field
+
+If this field is set, `dfx new` will display a spinner with this message while running the `post_create` command.
+
+## The `post_create_failure_warning` field
+
+If this field is present and the `post_create` command fails, `dfx new` will display this warning but won't stop creating the project.
+
+[project_templates]: https://github.com/dfinity/sdk/tree/master/src/dfx/assets/project_templates

docs/concepts/index.md

@@ -2,3 +2,5 @@
 
 - [Asset Canister Interface](../design/asset-canister-interface.md)
 - [Canister metadata](./canister-metadata.md)
+- [Extension-Defined Canister Types](./extension-defined-canister-types.md)
+- [Extension-Defined Project Templates](./extension-defined-project-templates.md)

docs/extension-manifest-schema.json

@@ -70,6 +70,15 @@
     "name": {
       "type": "string"
     },
+    "project_templates": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "additionalProperties": {
+        "$ref": "#/definitions/ExtensionProjectTemplate"
+      }
+    },
     "subcommands": {
       "anyOf": [
         {
@@ -155,6 +164,58 @@
         }
       ]
     },
+    "ExtensionProjectTemplate": {
+      "type": "object",
+      "required": [
+        "category",
+        "display",
+        "post_create",
+        "requirements"
+      ],
+      "properties": {
+        "category": {
+          "description": "Used to determine which CLI group (`--type`, `--backend`, `--frontend`) as well as for interactive selection",
+          "allOf": [
+            {
+              "$ref": "#/definitions/ProjectTemplateCategory"
+            }
+          ]
+        },
+        "display": {
+          "description": "The name used for display and sorting",
+          "type": "string"
+        },
+        "post_create": {
+          "description": "Run a command after adding the canister to dfx.json",
+          "allOf": [
+            {
+              "$ref": "#/definitions/SerdeVec_for_String"
+            }
+          ]
+        },
+        "post_create_failure_warning": {
+          "description": "If the post-create command fails, display this warning but don't fail",
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "post_create_spinner_message": {
+          "description": "If set, display a spinner while this command runs",
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "requirements": {
+          "description": "Other project templates to patch in alongside this one",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
     "ExtensionSubcommandArgOpts": {
       "type": "object",
       "properties": {
@@ -231,6 +292,16 @@
         "$ref": "#/definitions/ExtensionSubcommandOpts"
       }
     },
+    "ProjectTemplateCategory": {
+      "type": "string",
+      "enum": [
+        "backend",
+        "frontend",
+        "frontend-test",
+        "extra",
+        "support"
+      ]
+    },
     "Range_of_uint": {
       "type": "object",
       "required": [
@@ -249,6 +320,19 @@
           "minimum": 0.0
         }
       }
+    },
+    "SerdeVec_for_String": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      ]
     }
   }
 }

e2e/tests-dfx/extension.bash

@@ -14,6 +14,133 @@ teardown() {
   standard_teardown
 }
 
+@test "extension-defined project template" {
+  start_webserver --directory www
+  EXTENSION_URL="http://localhost:$E2E_WEB_SERVER_PORT/arbitrary/extension.json"
+  mkdir -p  www/arbitrary/downloads www/arbitrary/project_templates/a-template
+
+  cat > www/arbitrary/extension.json <<EOF
+{
+  "name": "an-extension",
+  "version": "0.1.0",
+  "homepage": "https://github.com/dfinity/dfx-extensions",
+  "authors": "DFINITY",
+  "summary": "Test extension for e2e purposes.",
+  "categories": [],
+  "keywords": [],
+  "project_templates": {
+    "rust-by-extension": {
+      "category": "backend",
+      "display": "rust by extension",
+      "requirements": [],
+      "post_create": "cargo update",
+      "port_create_failure_warning": "You will need to run it yourself (or a similar command like 'cargo vendor'), because 'dfx build' will use the --locked flag with Cargo."
+    }
+  },
+  "download_url_template": "http://localhost:$E2E_WEB_SERVER_PORT/arbitrary/downloads/{{tag}}.{{archive-format}}"
+}
+EOF
+
+  cat > www/arbitrary/dependencies.json <<EOF
+{
+  "0.1.0": {
+    "dfx": {
+      "version": ">=0.8.0"
+    }
+  }
+}
+EOF
+
+  cp -R "${BATS_TEST_DIRNAME}/../../src/dfx/assets/project_templates/rust" www/arbitrary/project_templates/rust-by-extension
+
+  ARCHIVE_BASENAME="an-extension-v0.1.0"
+
+  mkdir "$ARCHIVE_BASENAME"
+  cp www/arbitrary/extension.json "$ARCHIVE_BASENAME"
+  cp -R www/arbitrary/project_templates "$ARCHIVE_BASENAME"
+  tar -czf "$ARCHIVE_BASENAME".tar.gz "$ARCHIVE_BASENAME"
+  rm -rf "$ARCHIVE_BASENAME"
+
+  mv "$ARCHIVE_BASENAME".tar.gz www/arbitrary/downloads/
+
+  assert_command dfx extension install "$EXTENSION_URL"
+
+  setup_rust
+
+  dfx new rbe --type rust-by-extension --no-frontend
+  cd rbe || exit
+
+  dfx_start
+  assert_command dfx deploy
+  assert_command dfx canister call rbe_backend greet '("Rust By Extension")'
+  assert_contains "Hello, Rust By Extension!"
+}
+
+@test "extension-defined project template replaces built-in type" {
+  start_webserver --directory www
+  EXTENSION_URL="http://localhost:$E2E_WEB_SERVER_PORT/arbitrary/extension.json"
+  mkdir -p  www/arbitrary/downloads www/arbitrary/project_templates/a-template
+
+  cat > www/arbitrary/extension.json <<EOF
+{
+  "name": "an-extension",
+  "version": "0.1.0",
+  "homepage": "https://github.com/dfinity/dfx-extensions",
+  "authors": "DFINITY",
+  "summary": "Test extension for e2e purposes.",
+  "categories": [],
+  "keywords": [],
+  "project_templates": {
+    "rust": {
+      "category": "backend",
+      "display": "rust by extension",
+      "requirements": [],
+      "post_create": "cargo update"
+    }
+  },
+  "download_url_template": "http://localhost:$E2E_WEB_SERVER_PORT/arbitrary/downloads/{{tag}}.{{archive-format}}"
+}
+EOF
+
+  cat > www/arbitrary/dependencies.json <<EOF
+{
+  "0.1.0": {
+    "dfx": {
+      "version": ">=0.8.0"
+    }
+  }
+}
+EOF
+
+  cp -R "${BATS_TEST_DIRNAME}/../../src/dfx/assets/project_templates/rust" www/arbitrary/project_templates/rust
+  echo "just-proves-it-used-the-project-template" > www/arbitrary/project_templates/rust/proof.txt
+
+  ARCHIVE_BASENAME="an-extension-v0.1.0"
+
+  mkdir "$ARCHIVE_BASENAME"
+  cp www/arbitrary/extension.json "$ARCHIVE_BASENAME"
+  cp -R www/arbitrary/project_templates "$ARCHIVE_BASENAME"
+  tar -czf "$ARCHIVE_BASENAME".tar.gz "$ARCHIVE_BASENAME"
+  rm -rf "$ARCHIVE_BASENAME"
+
+  mv "$ARCHIVE_BASENAME".tar.gz www/arbitrary/downloads/
+
+  assert_command dfx extension install "$EXTENSION_URL"
+
+  setup_rust
+
+  dfx new rbe --type rust --no-frontend
+  assert_command cat rbe/proof.txt
+  assert_eq "just-proves-it-used-the-project-template"
+
+  cd rbe || exit
+
+  dfx_start
+  assert_command dfx deploy
+  assert_command dfx canister call rbe_backend greet '("Rust By Extension")'
+  assert_contains "Hello, Rust By Extension!"
+}
+
 @test "run an extension command with a canister type defined by another extension" {
   install_shared_asset subnet_type/shared_network_settings/system
   dfx_start_for_nns_install

e2e/utils/_.bash

@@ -82,10 +82,14 @@ dfx_new() {
     echo PWD: "$(pwd)" >&2
 }
 
-dfx_new_rust() {
-    local project_name=${1:-e2e_project}
+setup_rust() {
     rustup default stable
     rustup target add wasm32-unknown-unknown
+}
+
+dfx_new_rust() {
+    local project_name=${1:-e2e_project}
+    setup_rust
     dfx new "${project_name}" --type=rust --no-frontend
     test -d "${project_name}"
     test -f "${project_name}/dfx.json"

src/dfx-core/src/config/model/project_template.rs

@@ -1,7 +1,12 @@
-#[derive(Debug, Clone, Eq, PartialEq)]
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, JsonSchema)]
+#[serde(rename_all = "lowercase")]
 pub enum ProjectTemplateCategory {
     Backend,
     Frontend,
+    #[serde(rename = "frontend-test")]
     FrontendTest,
     Extra,
     Support,

src/dfx-core/src/config/project_templates.rs

@@ -3,6 +3,7 @@ use itertools::Itertools;
 use std::collections::BTreeMap;
 use std::fmt::Display;
 use std::io;
+use std::path::PathBuf;
 use std::sync::OnceLock;
 
 type GetArchiveFn = fn() -> Result<tar::Archive<flate2::read::GzDecoder<&'static [u8]>>, io::Error>;
@@ -11,6 +12,9 @@ type GetArchiveFn = fn() -> Result<tar::Archive<flate2::read::GzDecoder<&'static
 pub enum ResourceLocation {
     /// The template's assets are compiled into the dfx binary
     Bundled { get_archive_fn: GetArchiveFn },
+
+    /// The templates assets are in a directory on the filesystem
+    Directory { path: PathBuf },
 }
 
 #[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
@@ -74,10 +78,11 @@ type ProjectTemplates = BTreeMap<ProjectTemplateName, ProjectTemplate>;
 
 static PROJECT_TEMPLATES: OnceLock<ProjectTemplates> = OnceLock::new();
 
-pub fn populate(builtin_templates: Vec<ProjectTemplate>) {
-    let templates = builtin_templates
-        .iter()
-        .map(|t| (t.name.clone(), t.clone()))
+pub fn populate(builtin_templates: Vec<ProjectTemplate>, loaded_templates: Vec<ProjectTemplate>) {
+    let templates: ProjectTemplates = builtin_templates
+        .into_iter()
+        .map(|t| (t.name.clone(), t))
+        .chain(loaded_templates.into_iter().map(|t| (t.name.clone(), t)))
         .collect();
 
     PROJECT_TEMPLATES.set(templates).unwrap();