Merge branch 'master' into vincent/SDK-1867

CHANGELOG.md

@@ -20,6 +20,23 @@ Added a flag `--replica` to `dfx start`. This flag currently has no effect.
 Once PocketIC becomes the default for `dfx start` this flag will start the replica instead.
 You can use the `--replica` flag already to write scripts that anticipate that change.
 
+### feat: extensions can define project templates
+
+An extension can define one or more project templates for `dfx new` to use.
+These can be new templates or replace the built-in project templates.
+
+### fix: all commands with --all parameter skip remote canisters
+
+This affects the following commands:
+- `dfx canister delete`
+- `dfx canister deposit-cycles`
+- `dfx canister start`
+- `dfx canister status`
+- `dfx canister stop`
+- `dfx canister uninstall-code`
+- `dfx canister update-settings`
+- `dfx ledger fabricate-cycles`
+
 ### chore: improve `dfx deploy` messages.
 
 If users run `dfx deploy` without enough cycles, show additional messages to indicate what to do next.

docs/concepts/extension-defined-project-templates.md

@@ -0,0 +1,66 @@
+# Extension-Defined Project Templates
+
+## Overview
+
+An extension can define one or more project templates for `dfx new` to use.
+
+A project template is a set of files that `dfx new` copies or patches into a new project.
+
+For examples of project template files, see the [project_templates] directory in the SDK repository.
+
+# Specification
+
+The `project_templates` field in an extension's `extension.json` defines the project templates
+included in the extension.  It is an object field mapping `project template name -> project template properties`.
+These are the properties of a project template:
+
+| Field                        | Type                      | Description                                                                                          |
+|------------------------------|---------------------------|------------------------------------------------------------------------------------------------------|
+| `display`                    | String                    | Display name of the project template                                                                 |
+| `category`                   | String                    | Category for inclusion in `--backend` and `--frontend` CLI options, as well as interactive selection |
+| `requirements`               | Array of String           | Required project templates                                                                           |
+| `post_create`                | String or Array of String | Command(s) to run after adding the canister to the project                                           |
+| `post_create_spinner_message` | String                    | Message to display while running the post_create command                                            |
+| `post_create_failure_warning` | String                    | Warning to display if the post_create command fails                                                 |
+
+Within the files distributed with the extension, the project template files are
+located in the `project_templates/{project template name}` directory.
+
+## The `display` field
+
+The `display` field is a string that describes the project template.
+`dfx new` will use this value for interactive selection of project templates.
+
+## The `category` field
+
+The `category` field is an array of strings that categorize the project template.
+`dfx new` uses this field to determine whether to include this project template
+as an option for the `--backend` and `-frontend` flags, as well as in interactive setup.
+
+Valid values for the field:
+- `frontend`
+- `backend`
+- `extra`
+- `frontend-test`
+- `support`
+
+## The `requirements` field
+
+The `requirements` field lists any project templates that `dfx new` must apply before this project template.
+For example, many of the frontend templates depend on the `dfx_js_base` template, which adds
+package.json and tsconfig.json to the project.
+
+## The `post_create` field
+
+The `post_create` field specifies a command or commands to run after adding the project template files to the project.
+For example, the rust project template runs `cargo update` after adding the files.
+
+## The `post_create_spinner_message` field
+
+If this field is set, `dfx new` will display a spinner with this message while running the `post_create` command.
+
+## The `post_create_failure_warning` field
+
+If this field is present and the `post_create` command fails, `dfx new` will display this warning but won't stop creating the project.
+
+[project_templates]: https://github.com/dfinity/sdk/tree/master/src/dfx/assets/project_templates

docs/concepts/index.md

@@ -2,3 +2,5 @@
 
 - [Asset Canister Interface](../design/asset-canister-interface.md)
 - [Canister metadata](./canister-metadata.md)
+- [Extension-Defined Canister Types](./extension-defined-canister-types.md)
+- [Extension-Defined Project Templates](./extension-defined-project-templates.md)

docs/extension-manifest-schema.json

@@ -70,6 +70,15 @@
     "name": {
       "type": "string"
     },
+    "project_templates": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "additionalProperties": {
+        "$ref": "#/definitions/ExtensionProjectTemplate"
+      }
+    },
     "subcommands": {
       "anyOf": [
         {
@@ -155,6 +164,58 @@
         }
       ]
     },
+    "ExtensionProjectTemplate": {
+      "type": "object",
+      "required": [
+        "category",
+        "display",
+        "post_create",
+        "requirements"
+      ],
+      "properties": {
+        "category": {
+          "description": "Used to determine which CLI group (`--type`, `--backend`, `--frontend`) as well as for interactive selection",
+          "allOf": [
+            {
+              "$ref": "#/definitions/ProjectTemplateCategory"
+            }
+          ]
+        },
+        "display": {
+          "description": "The name used for display and sorting",
+          "type": "string"
+        },
+        "post_create": {
+          "description": "Run a command after adding the canister to dfx.json",
+          "allOf": [
+            {
+              "$ref": "#/definitions/SerdeVec_for_String"
+            }
+          ]
+        },
+        "post_create_failure_warning": {
+          "description": "If the post-create command fails, display this warning but don't fail",
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "post_create_spinner_message": {
+          "description": "If set, display a spinner while this command runs",
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "requirements": {
+          "description": "Other project templates to patch in alongside this one",
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
     "ExtensionSubcommandArgOpts": {
       "type": "object",
       "properties": {
@@ -231,6 +292,16 @@
         "$ref": "#/definitions/ExtensionSubcommandOpts"
       }
     },
+    "ProjectTemplateCategory": {
+      "type": "string",
+      "enum": [
+        "backend",
+        "frontend",
+        "frontend-test",
+        "extra",
+        "support"
+      ]
+    },
     "Range_of_uint": {
       "type": "object",
       "required": [
@@ -249,6 +320,19 @@
           "minimum": 0.0
         }
       }
+    },
+    "SerdeVec_for_String": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      ]
     }
   }
 }

e2e/tests-dfx/cycles-ledger.bash

@@ -458,6 +458,13 @@ current_time_nanoseconds() {
   assert_eq "2399699700000 cycles."
   assert_command dfx canister status e2e_project_backend
   assert_contains "Balance: 3_100_002_100_000 Cycles"
+
+  # deposit-cycles --all skips remote canisters
+  jq '.canisters.remote.remote.id.local="rdmx6-jaaaa-aaaaa-aaadq-cai"' dfx.json | sponge dfx.json
+  assert_command dfx canister deposit-cycles 10000 --all --identity bob
+  assert_contains "Skipping canister 'remote' because it is remote for network 'local'"
+  assert_contains "Depositing 10000 cycles onto e2e_project_backend"
+  assert_not_contains "Depositing 10000 cycles onto remote"
 }
 
 @test "top-up deduplication" {

e2e/tests-dfx/extension.bash

@@ -14,6 +14,133 @@ teardown() {
   standard_teardown
 }
 
+@test "extension-defined project template" {
+  start_webserver --directory www
+  EXTENSION_URL="http://localhost:$E2E_WEB_SERVER_PORT/arbitrary/extension.json"
+  mkdir -p  www/arbitrary/downloads www/arbitrary/project_templates/a-template
+
+  cat > www/arbitrary/extension.json <<EOF
+{
+  "name": "an-extension",
+  "version": "0.1.0",
+  "homepage": "https://github.com/dfinity/dfx-extensions",
+  "authors": "DFINITY",
+  "summary": "Test extension for e2e purposes.",
+  "categories": [],
+  "keywords": [],
+  "project_templates": {
+    "rust-by-extension": {
+      "category": "backend",
+      "display": "rust by extension",
+      "requirements": [],
+      "post_create": "cargo update",
+      "port_create_failure_warning": "You will need to run it yourself (or a similar command like 'cargo vendor'), because 'dfx build' will use the --locked flag with Cargo."
+    }
+  },
+  "download_url_template": "http://localhost:$E2E_WEB_SERVER_PORT/arbitrary/downloads/{{tag}}.{{archive-format}}"
+}
+EOF
+
+  cat > www/arbitrary/dependencies.json <<EOF
+{
+  "0.1.0": {
+    "dfx": {
+      "version": ">=0.8.0"
+    }
+  }
+}
+EOF
+
+  cp -R "${BATS_TEST_DIRNAME}/../../src/dfx/assets/project_templates/rust" www/arbitrary/project_templates/rust-by-extension
+
+  ARCHIVE_BASENAME="an-extension-v0.1.0"
+
+  mkdir "$ARCHIVE_BASENAME"
+  cp www/arbitrary/extension.json "$ARCHIVE_BASENAME"
+  cp -R www/arbitrary/project_templates "$ARCHIVE_BASENAME"
+  tar -czf "$ARCHIVE_BASENAME".tar.gz "$ARCHIVE_BASENAME"
+  rm -rf "$ARCHIVE_BASENAME"
+
+  mv "$ARCHIVE_BASENAME".tar.gz www/arbitrary/downloads/
+
+  assert_command dfx extension install "$EXTENSION_URL"
+
+  setup_rust
+
+  dfx new rbe --type rust-by-extension --no-frontend
+  cd rbe || exit
+
+  dfx_start
+  assert_command dfx deploy
+  assert_command dfx canister call rbe_backend greet '("Rust By Extension")'
+  assert_contains "Hello, Rust By Extension!"
+}
+
+@test "extension-defined project template replaces built-in type" {
+  start_webserver --directory www
+  EXTENSION_URL="http://localhost:$E2E_WEB_SERVER_PORT/arbitrary/extension.json"
+  mkdir -p  www/arbitrary/downloads www/arbitrary/project_templates/a-template
+
+  cat > www/arbitrary/extension.json <<EOF
+{
+  "name": "an-extension",
+  "version": "0.1.0",
+  "homepage": "https://github.com/dfinity/dfx-extensions",
+  "authors": "DFINITY",
+  "summary": "Test extension for e2e purposes.",
+  "categories": [],
+  "keywords": [],
+  "project_templates": {
+    "rust": {
+      "category": "backend",
+      "display": "rust by extension",
+      "requirements": [],
+      "post_create": "cargo update"
+    }
+  },
+  "download_url_template": "http://localhost:$E2E_WEB_SERVER_PORT/arbitrary/downloads/{{tag}}.{{archive-format}}"
+}
+EOF
+
+  cat > www/arbitrary/dependencies.json <<EOF
+{
+  "0.1.0": {
+    "dfx": {
+      "version": ">=0.8.0"
+    }
+  }
+}
+EOF
+
+  cp -R "${BATS_TEST_DIRNAME}/../../src/dfx/assets/project_templates/rust" www/arbitrary/project_templates/rust
+  echo "just-proves-it-used-the-project-template" > www/arbitrary/project_templates/rust/proof.txt
+
+  ARCHIVE_BASENAME="an-extension-v0.1.0"
+
+  mkdir "$ARCHIVE_BASENAME"
+  cp www/arbitrary/extension.json "$ARCHIVE_BASENAME"
+  cp -R www/arbitrary/project_templates "$ARCHIVE_BASENAME"
+  tar -czf "$ARCHIVE_BASENAME".tar.gz "$ARCHIVE_BASENAME"
+  rm -rf "$ARCHIVE_BASENAME"
+
+  mv "$ARCHIVE_BASENAME".tar.gz www/arbitrary/downloads/
+
+  assert_command dfx extension install "$EXTENSION_URL"
+
+  setup_rust
+
+  dfx new rbe --type rust --no-frontend
+  assert_command cat rbe/proof.txt
+  assert_eq "just-proves-it-used-the-project-template"
+
+  cd rbe || exit
+
+  dfx_start
+  assert_command dfx deploy
+  assert_command dfx canister call rbe_backend greet '("Rust By Extension")'
+  assert_contains "Hello, Rust By Extension!"
+}
+
 @test "run an extension command with a canister type defined by another extension" {
   install_shared_asset subnet_type/shared_network_settings/system
   dfx_start_for_nns_install

e2e/tests-dfx/remote.bash

@@ -137,7 +137,7 @@ teardown() {
   assert_match "Canister 'remote' is a remote canister on network 'actuallylocal', and cannot be installed from here."
 }
 
-@test "canister create --all, canister install --all skip remote canisters" {
+@test "all commands with --all skip remote canisters" {
   install_asset remote/actual
   dfx_start
   setup_actuallylocal_shared_network
@@ -201,6 +201,32 @@ teardown() {
   assert_command jq .remote canister_ids.json
   assert_eq "null"
 
+  assert_command dfx ledger fabricate-cycles --all --t 100 --network actuallylocal
+  assert_contains "Skipping canister 'remote' because it is remote for network 'actuallylocal'"
+
+  assert_command dfx canister status --all --network actuallylocal
+  assert_contains "Skipping canister 'remote' because it is remote for network 'actuallylocal'"
+
+  assert_command dfx canister update-settings --log-visibility public --all --network actuallylocal
+  assert_contains "Skipping canister 'remote' because it is remote for network 'actuallylocal'"
+
+  assert_command dfx canister stop --all --network actuallylocal
+  assert_contains "Skipping canister 'remote' because it is remote for network 'actuallylocal'"
+
+  assert_command dfx canister start --all --network actuallylocal
+  assert_contains "Skipping canister 'remote' because it is remote for network 'actuallylocal'"
+
+  # have to stop to uninstall
+  assert_command dfx canister stop --all --network actuallylocal
+
+  assert_command dfx canister uninstall-code --all --network actuallylocal
+  assert_contains "Skipping canister 'remote' because it is remote for network 'actuallylocal'"
+
+  assert_command dfx build --all --network actuallylocal
+
+  assert_command dfx canister delete --all --network actuallylocal
+  assert_contains "Skipping canister 'remote' because it is remote for network 'actuallylocal'"
+
   # Assert frontend declarations are actually created
   dfx generate
   assert_file_exists "src/declarations/remote/remote.did"