feat: dynamic wasm_url and wasm_hash

CHANGELOG.md

@@ -2,6 +2,70 @@
 
 # UNRELEASED
 
+### feat: dynamic wasm_url and wasm_hash
+
+`wasm_url` and `wasm_hash` might need to be calculated dynamically during canister build.
+
+#### wasm_url
+
+Beyond setting `wasm_url` directly, you can generate it into a file with configuration in `dfx.json` like:
+
+```json
+{
+  "pullable" : {
+    "dynamic_wasm_url" : {
+      "generate" : "<commands to generate wasm_url file>",
+      "path" : "<path/to/wasm_url_file>"
+    }
+  }
+}
+```
+
+You can only set one of `wasm_url` and `dynamic_wasm_url`.
+
+#### wasm_hash
+
+Service providers may want consumers to download a different wasm module than the on-chain canister.
+
+In such case, providers need to include a `wasm_hash` field in the `dfx/pullable` metadata of the on-chain pullable canister.
+
+There are three ways to specify `wasm_hash` in `dfx.json`. You can choose the most suitable one for your use.
+
+1. Set `wasm_hash` statically:
+
+```json
+{
+  "pullable" : {
+    "wasm_hash" : "<hex-encoded SHA256 hash of the wasm module to be downloaded>"
+  }
+}
+```
+
+1. Set `wasm_hash_file`, then `dfx` read its content to set `wasm_hash`:
+
+```json
+{
+  "pullable" : {
+    "wasm_hash_file" : "<path/to/wasm_hash_file>"
+  }
+}
+```
+
+3. Set `custom_wasm` which run `generate` commands and use the custom wasm module to calculate `wasm_hash`:
+
+```json
+{
+  "pullable" : {
+    "custom_wasm" : {
+      "generate" : "<commands to generate custom wasm file>",
+      "path" : "<path/to/custom_wasm_file>"
+    }
+  }
+}
+```
+
+`dfx` will do the same post-processing (optimize, metadata, gzip) to the custom wasm as to the canister to be deployed on-chain. `wasm_hash` will be calculated from the post-processed wasm.
+
 ### chore: --emulator parameter is deprecated and will be discontinued soon
 
 Added warning that the `--emulator` is deprecated and will be discontinued soon.

docs/dfx-json-schema.json

@@ -426,7 +426,7 @@
           "default": null,
           "anyOf": [
             {
-              "$ref": "#/definitions/Pullable"
+              "$ref": "#/definitions/PullableConfig"
             },
             {
               "type": "null"
@@ -799,6 +799,56 @@
         }
       }
     },
+    "CustomWasm": {
+      "title": "Custom WASM configuration",
+      "description": "Configs how to generate a custom WASM for pullable and where is the custom WASM file.",
+      "type": "object",
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "generate": {
+          "title": "Generate Commands",
+          "description": "Commands that are executed in order to generate a custom wasm of this pullable canister. Expected to produce the wasm file in the path specified by the 'path' field.",
+          "default": [],
+          "allOf": [
+            {
+              "$ref": "#/definitions/SerdeVec_for_String"
+            }
+          ]
+        },
+        "path": {
+          "title": "Custom WASM Path",
+          "description": "Path to the custom wasm file from the \"generate\" commands. The file should be a valid canister WASM .",
+          "type": "string"
+        }
+      }
+    },
+    "DynamicWasmUrl": {
+      "title": "Dynamic wasm_url configuration",
+      "description": "Configs how to generate wasm_url dynamically and where is the file contains wasm_url.",
+      "type": "object",
+      "required": [
+        "path"
+      ],
+      "properties": {
+        "generate": {
+          "title": "Generate Commands",
+          "description": "Commands that are executed in order to generate this pullable canister's wasm_url dynamically. Expected to produce the wasm_url file in the path specified by the 'path' field.",
+          "default": [],
+          "allOf": [
+            {
+              "$ref": "#/definitions/SerdeVec_for_String"
+            }
+          ]
+        },
+        "path": {
+          "title": "wasm_url Path",
+          "description": "Path to the wasm_url file from the \"generate\" commands. The file should contains a valid URL.",
+          "type": "string"
+        }
+      }
+    },
     "HttpAdapterLogLevel": {
       "description": "Represents the log level of the HTTP adapter.",
       "type": "string",
@@ -909,14 +959,27 @@
         "Release"
       ]
     },
-    "Pullable": {
+    "PullableConfig": {
+      "title": "Pullable configuration",
+      "description": "Configs the \"pullable\" metadata of the canister.",
       "type": "object",
       "required": [
         "dependencies",
-        "init_guide",
-        "wasm_url"
+        "init_guide"
       ],
       "properties": {
+        "custom_wasm": {
+          "title": "Custom WASM",
+          "description": "Build a custom WASM for pullable. wasm_hash will be calculated from this custom WASM. Conflicts with `wasm_hash` and `wasm_hash_file`.",
+          "anyOf": [
+            {
+              "$ref": "#/definitions/CustomWasm"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
         "dependencies": {
           "title": "dependencies",
           "description": "Canister IDs (Principal) of direct dependencies.",
@@ -925,23 +988,46 @@
             "type": "string"
           }
         },
+        "dynamic_wasm_url": {
+          "title": "Dynamic wasm_url",
+          "description": "Generate wasm_url dynamically. Conflicts with `wasm_url`.",
+          "anyOf": [
+            {
+              "$ref": "#/definitions/DynamicWasmUrl"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
         "init_guide": {
           "title": "init_guide",
           "description": "A message to guide consumers how to initialize the canister.",
           "type": "string"
         },
         "wasm_hash": {
           "title": "wasm_hash",
-          "description": "SHA256 hash of the wasm module located at wasm_url. Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url.",
+          "description": "SHA256 hash of the wasm module located at wasm_url. Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url. Conflicts with `wasm_hash_file` and `custom_wasm`.",
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "wasm_hash_file": {
+          "title": "Path to wasm_hash",
+          "description": "Conflicts with `wasm_hash` and `custom_wasm`.",
           "type": [
             "string",
             "null"
           ]
         },
         "wasm_url": {
           "title": "wasm_url",
-          "description": "The Url to download canister wasm.",
-          "type": "string"
+          "description": "The URL to download canister wasm. Conflicts with `dynamic_wasm_url`.",
+          "type": [
+            "string",
+            "null"
+          ]
         }
       }
     },

e2e/tests-dfx/deps.bash

@@ -69,6 +69,76 @@ setup_onchain() {
     assert_eq "A natural number, e.g. 20." "$output"
 }
 
+@test "dfx build can handle dynamic_wasm_url" {
+    dfx_start
+
+    install_asset deps
+
+    cd onchain
+    jq '.canisters.a.pullable.dynamic_wasm_url.generate="bash dynamic_wasm_url_a.sh"' dfx.json | sponge dfx.json
+    jq '.canisters.a.pullable.dynamic_wasm_url.path="dynamic_wasm_url.txt"' dfx.json | sponge dfx.json
+
+    echo "echo \"http:/example.com/dynamic_wasm_url/a.wasm\" > dynamic_wasm_url.txt" > dynamic_wasm_url_a.sh
+
+    assert_command dfx canister create --all
+    assert_command_fail dfx build a
+    assert_contains "Cannot define \`wasm_url\` and \`dynamic_wasm_url\` at the same time."
+
+    jq 'del(.canisters.a.pullable.wasm_url)' dfx.json | sponge dfx.json
+    assert_command dfx build a
+
+    ic-wasm .dfx/local/canisters/a/a.wasm metadata dfx > a_dfx.json
+    assert_command jq -r '.pullable.wasm_url' a_dfx.json
+    assert_eq "http:/example.com/dynamic_wasm_url/a.wasm" "$output"  
+}
+
+@test "dfx build can handle custom_wasm" {
+    dfx_start
+
+    install_asset deps
+
+    cd onchain
+
+    echo -n -e "\x00asm\x01\x00\x00\x00" > main.wasm
+    WASM_HASH="$(sha256sum main.wasm | cut -d " " -f 1)"
+    echo "$WASM_HASH" > wasm_hash.txt
+
+    assert_command dfx canister create --all
+
+    # Cannot define multiple fields for wasm_hash
+    jq '.canisters.a.pullable.wasm_hash="'"$WASM_HASH"'"' dfx.json | sponge dfx.json
+    jq '.canisters.a.pullable.wasm_hash_file="wasm_hash.txt"' dfx.json | sponge dfx.json
+    assert_command_fail dfx build a
+    assert_contains "Pullable canister can only define one of \`wasm_hash\`, \`wasm_hash_file\`, \`custom_wasm\`."
+
+    # Read wasm_hash from a file
+    jq 'del(.canisters.a.pullable.wasm_hash)' dfx.json | sponge dfx.json
+    assert_command dfx build a
+    ic-wasm .dfx/local/canisters/a/a.wasm metadata dfx > a_dfx.json
+    assert_command jq -r '.pullable.wasm_hash' a_dfx.json
+    assert_eq "$WASM_HASH" "$output"
+
+    # Generate the custom wasm, post-process it and calculate wasm_hash
+    jq 'del(.canisters.a.pullable.wasm_hash_file)' dfx.json | sponge dfx.json
+    jq '.canisters.a.pullable.custom_wasm.generate="cp main.wasm custom.wasm"' dfx.json | sponge dfx.json
+    jq '.canisters.a.pullable.custom_wasm.path="custom.wasm"' dfx.json | sponge dfx.json
+
+    assert_command dfx build a
+    assert_contains "Custom wasm for pullable at:"
+    assert_contains ".dfx/local/canisters/a/a_custom.wasm"
+    PROCESSED_WASM_HASH="$(sha256sum .dfx/local/canisters/a/a_custom.wasm | cut -d " " -f 1)"
+    assert_contains "wasm_hash: $PROCESSED_WASM_HASH"
+
+    ic-wasm .dfx/local/canisters/a/a.wasm metadata dfx > a_dfx.json
+    assert_command jq -r '.pullable.wasm_hash' a_dfx.json
+    assert_eq "$PROCESSED_WASM_HASH" "$output"
+
+    assert_command ic-wasm .dfx/local/canisters/a/a.wasm metadata
+    assert_contains "icp:public dfx"
+    assert_contains "icp:public candid:service"
+    assert_contains "icp:public candid:args"
+}
+
 @test "dfx deps pull can resolve dependencies from on-chain canister metadata" {
     # ic-ref has different behavior than the replica:
     #   it doesn't differ whether the canister not exist or the metadata not exist
@@ -150,8 +220,6 @@ Failed to download from url: http://example.com/c.wasm."
 
     setup_onchain
 
-    # TODO: test gzipped wasm can be pulled when we have "gzip" option in dfx.json (SDK-1102)
-
     # pull canisters in app project
     cd app
     assert_file_not_exists "deps/pulled.json"

src/dfx-core/src/config/model/dfinity.rs

@@ -153,24 +153,78 @@ impl CanisterMetadataSection {
     }
 }
 
+/// # Pullable configuration
+/// Configs the "pullable" metadata of the canister.
 #[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)]
-pub struct Pullable {
+pub struct PullableConfig {
     /// # wasm_url
-    /// The Url to download canister wasm.
-    pub wasm_url: String,
+    /// The URL to download canister wasm.
+    /// Conflicts with `dynamic_wasm_url`.
+    pub wasm_url: Option<String>,
+
+    /// # Dynamic wasm_url
+    /// Generate wasm_url dynamically.
+    /// Conflicts with `wasm_url`.
+    pub dynamic_wasm_url: Option<DynamicWasmUrl>,
+
     /// # wasm_hash
     /// SHA256 hash of the wasm module located at wasm_url.
     /// Only define this if the on-chain canister wasm is expected not to match the wasm at wasm_url.
+    /// Conflicts with `wasm_hash_file` and `custom_wasm`.
     pub wasm_hash: Option<String>,
+
+    /// # Path to wasm_hash
+    /// Conflicts with `wasm_hash` and `custom_wasm`.
+    pub wasm_hash_file: Option<String>,
+
+    /// # Custom WASM
+    /// Build a custom WASM for pullable.
+    /// wasm_hash will be calculated from this custom WASM.
+    /// Conflicts with `wasm_hash` and `wasm_hash_file`.
+    pub custom_wasm: Option<CustomWasm>,
+
     /// # dependencies
     /// Canister IDs (Principal) of direct dependencies.
     #[schemars(with = "Vec::<String>")]
     pub dependencies: Vec<Principal>,
+
     /// # init_guide
     /// A message to guide consumers how to initialize the canister.
     pub init_guide: String,
 }
 
+/// # Dynamic wasm_url configuration
+/// Configs how to generate wasm_url dynamically and where is the file contains wasm_url.
+#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)]
+pub struct DynamicWasmUrl {
+    /// # Generate Commands
+    /// Commands that are executed in order to generate this pullable canister's wasm_url dynamically.
+    /// Expected to produce the wasm_url file in the path specified by the 'path' field.
+    #[schemars(default)]
+    pub generate: SerdeVec<String>,
+
+    /// # wasm_url Path
+    /// Path to the wasm_url file from the "generate" commands.
+    /// The file should contains a valid URL.
+    pub path: String,
+}
+
+/// # Custom WASM configuration
+/// Configs how to generate a custom WASM for pullable and where is the custom WASM file.
+#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)]
+pub struct CustomWasm {
+    /// # Generate Commands
+    /// Commands that are executed in order to generate a custom wasm of this pullable canister.
+    /// Expected to produce the wasm file in the path specified by the 'path' field.
+    #[schemars(default)]
+    pub generate: SerdeVec<String>,
+
+    /// # Custom WASM Path
+    /// Path to the custom wasm file from the "generate" commands.
+    /// The file should be a valid canister WASM .
+    pub path: String,
+}
+
 pub const DEFAULT_SHARED_LOCAL_BIND: &str = "127.0.0.1:4943"; // hex for "IC"
 pub const DEFAULT_PROJECT_LOCAL_BIND: &str = "127.0.0.1:8000";
 pub const DEFAULT_IC_GATEWAY: &str = "https://icp0.io";
@@ -247,7 +301,7 @@ pub struct ConfigCanistersCanister {
     /// # Pullable
     /// Defines required properties so that this canister is ready for `dfx deps pull` by other projects.
     #[serde(default)]
-    pub pullable: Option<Pullable>,
+    pub pullable: Option<PullableConfig>,
 
     /// # Gzip Canister WASM
     /// Disabled by default.