diff --git a/CHANGELOG.md b/CHANGELOG.md index 5396853a28..c2e6e01994 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ Added a flag `--replica` to `dfx start`. This flag currently has no effect. Once PocketIC becomes the default for `dfx start` this flag will start the replica instead. You can use the `--replica` flag already to write scripts that anticipate that change. +### feat: extensions can define project templates + +An extension can define one or more project templates for `dfx new` to use. +These can be new templates or replace the built-in project templates. + # 0.24.3 ### feat: Bitcoin support in PocketIC diff --git a/docs/concepts/extension-defined-project-templates.md b/docs/concepts/extension-defined-project-templates.md new file mode 100644 index 0000000000..b66789221a --- /dev/null +++ b/docs/concepts/extension-defined-project-templates.md @@ -0,0 +1,66 @@ +# Extension-Defined Project Templates + +## Overview + +An extension can define one or more project templates for `dfx new` to use. + +A project template is a set of files that `dfx new` copies or patches into a new project. + +For examples of project template files, see the [project_templates] directory in the SDK repository. + +# Specification + +The `project_templates` field in an extension's `extension.json` defines the project templates +included in the extension. It is an object field mapping `project template name -> project template properties`. +These are the properties of a project template: + +| Field | Type | Description | +|------------------------------|---------------------------|------------------------------------------------------------------------------------------------------| +| `display` | String | Display name of the project template | +| `category` | String | Category for inclusion in `--backend` and `--frontend` CLI options, as well as interactive selection | +| `requirements` | Array of String | Required project templates | +| `post_create` | String or Array of String | Command(s) to run after adding the canister to the project | +| `post_create_spinner_message` | String | Message to display while running the post_create command | +| `post_create_failure_warning` | String | Warning to display if the post_create command fails | + +Within the files distributed with the extension, the project template files are +located in the `project_templates/{project template name}` directory. + +## The `display` field + +The `display` field is a string that describes the project template. +`dfx new` will use this value for interactive selection of project templates. + +## The `category` field + +The `category` field is an array of strings that categorize the project template. +`dfx new` uses this field to determine whether to include this project template +as an option for the `--backend` and `-frontend` flags, as well as in interactive setup. + +Valid values for the field: +- `frontend` +- `backend` +- `extra` +- `frontend-test` +- `support` + +## The `requirements` field + +The `requirements` field lists any project templates that `dfx new` must apply before this project template. +For example, many of the frontend templates depend on the `dfx_js_base` template, which adds +package.json and tsconfig.json to the project. + +## The `post_create` field + +The `post_create` field specifies a command or commands to run after adding the project template files to the project. +For example, the rust project template runs `cargo update` after adding the files. + +## The `post_create_spinner_message` field + +If this field is set, `dfx new` will display a spinner with this message while running the `post_create` command. + +## The `post_create_failure_warning` field + +If this field is present and the `post_create` command fails, `dfx new` will display this warning but won't stop creating the project. + +[project_templates]: https://github.com/dfinity/sdk/tree/master/src/dfx/assets/project_templates diff --git a/docs/concepts/index.md b/docs/concepts/index.md index 4f45495c5b..cf91724f81 100644 --- a/docs/concepts/index.md +++ b/docs/concepts/index.md @@ -2,3 +2,5 @@ - [Asset Canister Interface](../design/asset-canister-interface.md) - [Canister metadata](./canister-metadata.md) +- [Extension-Defined Canister Types](./extension-defined-canister-types.md) +- [Extension-Defined Project Templates](./extension-defined-project-templates.md) diff --git a/docs/extension-manifest-schema.json b/docs/extension-manifest-schema.json index beaff0a894..544823b5cb 100644 --- a/docs/extension-manifest-schema.json +++ b/docs/extension-manifest-schema.json @@ -70,6 +70,15 @@ "name": { "type": "string" }, + "project_templates": { + "type": [ + "object", + "null" + ], + "additionalProperties": { + "$ref": "#/definitions/ExtensionProjectTemplate" + } + }, "subcommands": { "anyOf": [ { @@ -155,6 +164,58 @@ } ] }, + "ExtensionProjectTemplate": { + "type": "object", + "required": [ + "category", + "display", + "post_create", + "requirements" + ], + "properties": { + "category": { + "description": "Used to determine which CLI group (`--type`, `--backend`, `--frontend`) as well as for interactive selection", + "allOf": [ + { + "$ref": "#/definitions/ProjectTemplateCategory" + } + ] + }, + "display": { + "description": "The name used for display and sorting", + "type": "string" + }, + "post_create": { + "description": "Run a command after adding the canister to dfx.json", + "allOf": [ + { + "$ref": "#/definitions/SerdeVec_for_String" + } + ] + }, + "post_create_failure_warning": { + "description": "If the post-create command fails, display this warning but don't fail", + "type": [ + "string", + "null" + ] + }, + "post_create_spinner_message": { + "description": "If set, display a spinner while this command runs", + "type": [ + "string", + "null" + ] + }, + "requirements": { + "description": "Other project templates to patch in alongside this one", + "type": "array", + "items": { + "type": "string" + } + } + } + }, "ExtensionSubcommandArgOpts": { "type": "object", "properties": { @@ -231,6 +292,16 @@ "$ref": "#/definitions/ExtensionSubcommandOpts" } }, + "ProjectTemplateCategory": { + "type": "string", + "enum": [ + "backend", + "frontend", + "frontend-test", + "extra", + "support" + ] + }, "Range_of_uint": { "type": "object", "required": [ @@ -249,6 +320,19 @@ "minimum": 0.0 } } + }, + "SerdeVec_for_String": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ] } } } \ No newline at end of file diff --git a/e2e/tests-dfx/extension.bash b/e2e/tests-dfx/extension.bash index 7aa52b2122..b4059ffab3 100644 --- a/e2e/tests-dfx/extension.bash +++ b/e2e/tests-dfx/extension.bash @@ -14,6 +14,133 @@ teardown() { standard_teardown } +@test "extension-defined project template" { + start_webserver --directory www + EXTENSION_URL="http://localhost:$E2E_WEB_SERVER_PORT/arbitrary/extension.json" + mkdir -p www/arbitrary/downloads www/arbitrary/project_templates/a-template + + cat > www/arbitrary/extension.json < www/arbitrary/dependencies.json <=0.8.0" + } + } +} +EOF + + cp -R "${BATS_TEST_DIRNAME}/../../src/dfx/assets/project_templates/rust" www/arbitrary/project_templates/rust-by-extension + + ARCHIVE_BASENAME="an-extension-v0.1.0" + + mkdir "$ARCHIVE_BASENAME" + cp www/arbitrary/extension.json "$ARCHIVE_BASENAME" + cp -R www/arbitrary/project_templates "$ARCHIVE_BASENAME" + tar -czf "$ARCHIVE_BASENAME".tar.gz "$ARCHIVE_BASENAME" + rm -rf "$ARCHIVE_BASENAME" + + mv "$ARCHIVE_BASENAME".tar.gz www/arbitrary/downloads/ + + assert_command dfx extension install "$EXTENSION_URL" + + setup_rust + + dfx new rbe --type rust-by-extension --no-frontend + cd rbe || exit + + dfx_start + assert_command dfx deploy + assert_command dfx canister call rbe_backend greet '("Rust By Extension")' + assert_contains "Hello, Rust By Extension!" +} + +@test "extension-defined project template replaces built-in type" { + start_webserver --directory www + EXTENSION_URL="http://localhost:$E2E_WEB_SERVER_PORT/arbitrary/extension.json" + mkdir -p www/arbitrary/downloads www/arbitrary/project_templates/a-template + + cat > www/arbitrary/extension.json < www/arbitrary/dependencies.json <=0.8.0" + } + } +} +EOF + + cp -R "${BATS_TEST_DIRNAME}/../../src/dfx/assets/project_templates/rust" www/arbitrary/project_templates/rust + echo "just-proves-it-used-the-project-template" > www/arbitrary/project_templates/rust/proof.txt + + ARCHIVE_BASENAME="an-extension-v0.1.0" + + mkdir "$ARCHIVE_BASENAME" + cp www/arbitrary/extension.json "$ARCHIVE_BASENAME" + cp -R www/arbitrary/project_templates "$ARCHIVE_BASENAME" + tar -czf "$ARCHIVE_BASENAME".tar.gz "$ARCHIVE_BASENAME" + rm -rf "$ARCHIVE_BASENAME" + + mv "$ARCHIVE_BASENAME".tar.gz www/arbitrary/downloads/ + + assert_command dfx extension install "$EXTENSION_URL" + + setup_rust + + dfx new rbe --type rust --no-frontend + assert_command cat rbe/proof.txt + assert_eq "just-proves-it-used-the-project-template" + + cd rbe || exit + + dfx_start + assert_command dfx deploy + assert_command dfx canister call rbe_backend greet '("Rust By Extension")' + assert_contains "Hello, Rust By Extension!" +} + @test "run an extension command with a canister type defined by another extension" { install_shared_asset subnet_type/shared_network_settings/system dfx_start_for_nns_install diff --git a/e2e/utils/_.bash b/e2e/utils/_.bash index 5ec7b8466e..889253e68a 100644 --- a/e2e/utils/_.bash +++ b/e2e/utils/_.bash @@ -82,10 +82,14 @@ dfx_new() { echo PWD: "$(pwd)" >&2 } -dfx_new_rust() { - local project_name=${1:-e2e_project} +setup_rust() { rustup default stable rustup target add wasm32-unknown-unknown +} + +dfx_new_rust() { + local project_name=${1:-e2e_project} + setup_rust dfx new "${project_name}" --type=rust --no-frontend test -d "${project_name}" test -f "${project_name}/dfx.json" diff --git a/src/dfx-core/src/config/model/project_template.rs b/src/dfx-core/src/config/model/project_template.rs index f35ca7e399..3dfd00a7b1 100644 --- a/src/dfx-core/src/config/model/project_template.rs +++ b/src/dfx-core/src/config/model/project_template.rs @@ -1,7 +1,12 @@ -#[derive(Debug, Clone, Eq, PartialEq)] +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "lowercase")] pub enum ProjectTemplateCategory { Backend, Frontend, + #[serde(rename = "frontend-test")] FrontendTest, Extra, Support, diff --git a/src/dfx-core/src/config/project_templates.rs b/src/dfx-core/src/config/project_templates.rs index 9febc449f2..d6194a7a5d 100644 --- a/src/dfx-core/src/config/project_templates.rs +++ b/src/dfx-core/src/config/project_templates.rs @@ -3,6 +3,7 @@ use itertools::Itertools; use std::collections::BTreeMap; use std::fmt::Display; use std::io; +use std::path::PathBuf; use std::sync::OnceLock; type GetArchiveFn = fn() -> Result>, io::Error>; @@ -11,6 +12,9 @@ type GetArchiveFn = fn() -> Result; static PROJECT_TEMPLATES: OnceLock = OnceLock::new(); -pub fn populate(builtin_templates: Vec) { - let templates = builtin_templates - .iter() - .map(|t| (t.name.clone(), t.clone())) +pub fn populate(builtin_templates: Vec, loaded_templates: Vec) { + let templates: ProjectTemplates = builtin_templates + .into_iter() + .map(|t| (t.name.clone(), t)) + .chain(loaded_templates.into_iter().map(|t| (t.name.clone(), t))) .collect(); PROJECT_TEMPLATES.set(templates).unwrap(); diff --git a/src/dfx-core/src/extension/installed.rs b/src/dfx-core/src/extension/installed.rs index 4a85ba598b..c701b3949c 100644 --- a/src/dfx-core/src/extension/installed.rs +++ b/src/dfx-core/src/extension/installed.rs @@ -1,4 +1,6 @@ +use crate::config::project_templates::ProjectTemplate; use crate::error::extension::ConvertExtensionIntoClapCommandError; +use crate::extension::manager::ExtensionManager; use crate::extension::manifest::ExtensionManifest; use crate::extension::ExtensionName; use clap::Command; @@ -28,4 +30,15 @@ impl InstalledExtensionManifests { pub fn contains(&self, extension: &str) -> bool { self.0.contains_key(extension) } + + pub fn loaded_templates( + &self, + em: &ExtensionManager, + builtin_templates: &[ProjectTemplate], + ) -> Vec { + self.0 + .values() + .flat_map(|manifest| manifest.project_templates(em, builtin_templates)) + .collect() + } } diff --git a/src/dfx-core/src/extension/manifest/extension.rs b/src/dfx-core/src/extension/manifest/extension.rs index 7ed0edcf5b..b72b6619de 100644 --- a/src/dfx-core/src/extension/manifest/extension.rs +++ b/src/dfx-core/src/extension/manifest/extension.rs @@ -1,8 +1,11 @@ +use crate::config::model::project_template::ProjectTemplateCategory; +use crate::config::project_templates::{ProjectTemplate, ProjectTemplateName, ResourceLocation}; use crate::error::extension::{ ConvertExtensionSubcommandIntoClapArgError, ConvertExtensionSubcommandIntoClapCommandError, LoadExtensionManifestError, }; -use crate::json::structure::{VersionReqWithJsonSchema, VersionWithJsonSchema}; +use crate::extension::manager::ExtensionManager; +use crate::json::structure::{SerdeVec, VersionReqWithJsonSchema, VersionWithJsonSchema}; use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value; @@ -34,6 +37,8 @@ pub struct ExtensionManifest { pub dependencies: Option>, pub canister_type: Option, + pub project_templates: Option>, + /// Components of the download url template are: /// - `{{tag}}`: the tag of the extension release, which will follow the form "-v" /// - `{{basename}}`: The basename of the release filename, which will follow the form "--", for example "nns-x86_64-unknown-linux-gnu" @@ -56,6 +61,28 @@ pub enum ExtensionDependency { Version(VersionReqWithJsonSchema), } +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct ExtensionProjectTemplate { + /// The name used for display and sorting + pub display: String, + + /// Used to determine which CLI group (`--type`, `--backend`, `--frontend`) + /// as well as for interactive selection + pub category: ProjectTemplateCategory, + + /// Other project templates to patch in alongside this one + pub requirements: Vec, + + /// Run a command after adding the canister to dfx.json + pub post_create: SerdeVec, + + /// If set, display a spinner while this command runs + pub post_create_spinner_message: Option, + + /// If the post-create command fails, display this warning but don't fail + pub post_create_failure_warning: Option, +} + impl ExtensionManifest { pub fn load( name: &str, @@ -92,6 +119,59 @@ impl ExtensionManifest { Ok(vec![]) } } + + pub fn project_templates( + &self, + em: &ExtensionManager, + builtin_templates: &[ProjectTemplate], + ) -> Vec { + let Some(project_templates) = self.project_templates.as_ref() else { + return vec![]; + }; + + let extension_dir = em.get_extension_directory(&self.name); + + // the default sort order is after everything built-in + let default_sort_order = builtin_templates + .iter() + .map(|t| t.sort_order) + .max() + .unwrap_or(0) + + 1; + + project_templates + .iter() + .map(|(name, template)| { + let resource_dir = extension_dir.join("project_templates").join(name); + let resource_location = ResourceLocation::Directory { path: resource_dir }; + + // keep the sort order as a built-in template of the same name, + // otherwise put it after everything else + let sort_order = builtin_templates + .iter() + .find(|t| t.name == ProjectTemplateName(name.clone())) + .map(|t| t.sort_order) + .unwrap_or(default_sort_order); + + let requirements = template + .requirements + .iter() + .map(|r| ProjectTemplateName(r.clone())) + .collect(); + ProjectTemplate { + name: ProjectTemplateName(name.clone()), + display: template.display.clone(), + resource_location, + category: template.category.clone(), + requirements, + post_create: template.post_create.clone().into_vec(), + post_create_spinner_message: template.post_create_spinner_message.clone(), + post_create_failure_warning: template.post_create_failure_warning.clone(), + sort_order, + } + }) + .collect() + } } #[derive(Debug, Serialize, Deserialize, JsonSchema)] diff --git a/src/dfx/src/commands/new.rs b/src/dfx/src/commands/new.rs index 093fe23805..81b3c28e88 100644 --- a/src/dfx/src/commands/new.rs +++ b/src/dfx/src/commands/new.rs @@ -29,6 +29,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, ExitStatus, Stdio}; use std::time::Duration; use tar::Archive; +use walkdir::WalkDir; // const DRY_RUN: &str = "dry_run"; // const PROJECT_NAME: &str = "project_name"; @@ -270,6 +271,52 @@ fn write_files_from_entries( Ok(()) } +fn write_files_from_directory( + log: &Logger, + dir: &Path, + root: &Path, + dry_run: bool, + variables: &BTreeMap, +) -> DfxResult { + for entry in WalkDir::new(dir).into_iter().filter_map(Result::ok) { + let path = entry.path(); + + if path.is_dir() { + continue; + } + + // Read file contents into a Vec + let file_content = dfx_core::fs::read(path)?; + + // Process the file content (replace variables) + let processed_content = match String::from_utf8(file_content) { + Err(err) => err.into_bytes(), + Ok(s) => replace_variables(s, variables).into_bytes(), + }; + + // Perform path replacements + let relative_path = path + .strip_prefix(dir)? + .to_str() + .ok_or_else(|| anyhow!("Non-unicode path encountered: {}", path.display()))?; + let relative_path = replace_variables(relative_path.to_string(), variables); + + // Build the final target path + let final_path = root.join(&relative_path); + + // Process files based on their extension + if final_path.extension() == Some("json-patch".as_ref()) { + json_patch_file(log, &final_path, &processed_content, dry_run)?; + } else if final_path.extension() == Some("patch".as_ref()) { + patch_file(log, &final_path, &processed_content, dry_run)?; + } else { + create_file(log, &final_path, &processed_content, dry_run)?; + } + } + + Ok(()) +} + #[context("Failed to scaffold frontend code.")] fn scaffold_frontend_code( env: &dyn Environment, @@ -706,10 +753,15 @@ fn write_project_template_resources( dry_run: bool, variables: &BTreeMap, ) -> DfxResult { - let mut resources = match template.resource_location { - ResourceLocation::Bundled { get_archive_fn } => get_archive_fn()?, - }; - write_files_from_entries(logger, &mut resources, project_name, dry_run, variables) + match &template.resource_location { + ResourceLocation::Bundled { get_archive_fn } => { + let mut resources = get_archive_fn()?; + write_files_from_entries(logger, &mut resources, project_name, dry_run, variables) + } + ResourceLocation::Directory { path } => { + write_files_from_directory(logger, path, project_name, dry_run, variables) + } + } } fn get_opts_interactively(opts: NewOpts) -> DfxResult { diff --git a/src/dfx/src/main.rs b/src/dfx/src/main.rs index ef86de36d3..79a19d3542 100644 --- a/src/dfx/src/main.rs +++ b/src/dfx/src/main.rs @@ -137,7 +137,9 @@ fn get_args_altered_for_extension_run( fn inner_main() -> DfxResult { let em = ExtensionManager::new(dfx_version())?; let installed_extension_manifests = em.load_installed_extension_manifests()?; - project_templates::populate(builtin_templates()); + let builtin_templates = builtin_templates(); + let loaded_templates = installed_extension_manifests.loaded_templates(&em, &builtin_templates); + project_templates::populate(builtin_templates, loaded_templates); let args = get_args_altered_for_extension_run(&installed_extension_manifests)?; @@ -201,7 +203,7 @@ mod tests { #[test] fn validate_cli() { - project_templates::populate(builtin_templates()); + project_templates::populate(builtin_templates(), vec![]); CliOpts::command().debug_assert(); }