From d9728fb27a7231793b21ebc960f56457dc3ce575 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Fri, 25 Aug 2023 13:30:34 +0300 Subject: [PATCH 01/17] git: Ignore target folder for main repo Signed-off-by: Alexandru Vasile --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e69de29bb2d1..9f970225adb6 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1 @@ +target/ \ No newline at end of file From 289732a7df80ff9d4f6018e205f84895fa5add73 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Fri, 25 Aug 2023 13:31:15 +0300 Subject: [PATCH 02/17] Cargo: Update trie to github lexnv/expose_merkle_value Signed-off-by: Alexandru Vasile --- Cargo.lock | 85 +++++++++++++------ cumulus/bridges/bin/runtime-common/Cargo.toml | 2 +- cumulus/bridges/primitives/runtime/Cargo.toml | 4 +- cumulus/pallets/parachain-system/Cargo.toml | 2 +- substrate/bin/node/bench/Cargo.toml | 2 +- substrate/client/db/Cargo.toml | 2 +- substrate/primitives/api/Cargo.toml | 2 +- substrate/primitives/core/Cargo.toml | 2 +- substrate/primitives/state-machine/Cargo.toml | 4 +- substrate/primitives/trie/Cargo.toml | 8 +- substrate/test-utils/runtime/Cargo.toml | 2 +- substrate/utils/binary-merkle-tree/Cargo.toml | 2 +- .../rpc/state-trie-migration-rpc/Cargo.toml | 2 +- 13 files changed, 76 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4b0028a64513..3872b4be8134 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1333,7 +1333,7 @@ version = "4.0.0-dev" dependencies = [ "array-bytes", "env_logger 0.9.3", - "hash-db", + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "log", "sp-core", "sp-runtime", @@ -1767,7 +1767,7 @@ version = "0.1.0" dependencies = [ "frame-support", "frame-system", - "hash-db", + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "hex-literal 0.4.1", "impl-trait-for-tuples", "log", @@ -1781,7 +1781,7 @@ dependencies = [ "sp-state-machine", "sp-std", "sp-trie", - "trie-db", + "trie-db 0.27.1 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", ] [[package]] @@ -2121,7 +2121,7 @@ dependencies = [ "bp-xcm-bridge-hub-router", "frame-support", "frame-system", - "hash-db", + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "log", "pallet-balances", "pallet-bridge-grandpa", @@ -3591,7 +3591,7 @@ dependencies = [ "sp-tracing", "sp-trie", "sp-version", - "trie-db", + "trie-db 0.27.1 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "xcm", ] @@ -5965,6 +5965,11 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e7d7786361d7425ae2fe4f9e407eb0efaa0840f5212d109cc018c40c35c6ab4" +[[package]] +name = "hash-db" +version = "0.16.0" +source = "git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value#1124af8bb3b037df675e1bc0da5963e6f3ee209b" + [[package]] name = "hash256-std-hasher" version = "0.15.2" @@ -6797,7 +6802,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19ea4653859ca2266a86419d3f592d3f22e7a854b482f99180d2498507902048" dependencies = [ - "hash-db", + "hash-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", "hash256-std-hasher", "tiny-keccak", ] @@ -7976,7 +7981,15 @@ version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "808b50db46293432a45e63bc15ea51e0ab4c0a1647b8eb114e31a3e698dd6fbe" dependencies = [ - "hash-db", + "hash-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "memory-db" +version = "0.32.0" +source = "git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value#1124af8bb3b037df675e1bc0da5963e6f3ee209b" +dependencies = [ + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", ] [[package]] @@ -8352,7 +8365,7 @@ dependencies = [ "derive_more", "fs_extra", "futures", - "hash-db", + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "kitchensink-runtime", "kvdb", "kvdb-rocksdb", @@ -14843,7 +14856,7 @@ version = "0.10.0-dev" dependencies = [ "array-bytes", "criterion 0.4.0", - "hash-db", + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "kitchensink-runtime", "kvdb", "kvdb-memorydb", @@ -16810,7 +16823,7 @@ dependencies = [ name = "sp-api" version = "4.0.0-dev" dependencies = [ - "hash-db", + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "log", "parity-scale-codec", "scale-info", @@ -17171,7 +17184,7 @@ dependencies = [ "dyn-clonable", "ed25519-zebra 3.1.0", "futures", - "hash-db", + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "hash256-std-hasher", "impl-serde", "lazy_static", @@ -17573,7 +17586,7 @@ version = "0.28.0" dependencies = [ "array-bytes", "assert_matches", - "hash-db", + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "log", "parity-scale-codec", "parking_lot 0.12.1", @@ -17588,7 +17601,7 @@ dependencies = [ "sp-trie", "thiserror", "tracing", - "trie-db", + "trie-db 0.27.1 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", ] [[package]] @@ -17695,10 +17708,10 @@ dependencies = [ "ahash 0.8.3", "array-bytes", "criterion 0.4.0", - "hash-db", + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "hashbrown 0.13.2", "lazy_static", - "memory-db", + "memory-db 0.32.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "nohash-hasher", "parity-scale-codec", "parking_lot 0.12.1", @@ -17710,8 +17723,8 @@ dependencies = [ "thiserror", "tracing", "trie-bench", - "trie-db", - "trie-root", + "trie-db 0.27.1 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", + "trie-root 0.18.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "trie-standardmap", ] @@ -18120,7 +18133,7 @@ dependencies = [ "sp-runtime", "sp-state-machine", "sp-trie", - "trie-db", + "trie-db 0.27.1 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", ] [[package]] @@ -18195,7 +18208,7 @@ dependencies = [ "sp-version", "substrate-test-runtime-client", "substrate-wasm-builder", - "trie-db", + "trie-db 0.27.1 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", ] [[package]] @@ -19128,12 +19141,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f54b4f9d51d368e62cf7e0730c7c1e18fc658cc84333656bab5b328f44aa964" dependencies = [ "criterion 0.4.0", - "hash-db", + "hash-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher", - "memory-db", + "memory-db 0.32.0 (registry+https://github.com/rust-lang/crates.io-index)", "parity-scale-codec", - "trie-db", - "trie-root", + "trie-db 0.27.1 (registry+https://github.com/rust-lang/crates.io-index)", + "trie-root 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)", "trie-standardmap", ] @@ -19143,7 +19156,19 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "767abe6ffed88a1889671a102c2861ae742726f52e0a5a425b92c9fbfa7e9c85" dependencies = [ - "hash-db", + "hash-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hashbrown 0.13.2", + "log", + "rustc-hex", + "smallvec", +] + +[[package]] +name = "trie-db" +version = "0.27.1" +source = "git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value#1124af8bb3b037df675e1bc0da5963e6f3ee209b" +dependencies = [ + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "hashbrown 0.13.2", "log", "rustc-hex", @@ -19156,7 +19181,15 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4ed310ef5ab98f5fa467900ed906cb9232dd5376597e00fd4cba2a449d06c0b" dependencies = [ - "hash-db", + "hash-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "trie-root" +version = "0.18.0" +source = "git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value#1124af8bb3b037df675e1bc0da5963e6f3ee209b" +dependencies = [ + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", ] [[package]] @@ -19165,7 +19198,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "684aafb332fae6f83d7fe10b3fbfdbe39a1b3234c4e2a618f030815838519516" dependencies = [ - "hash-db", + "hash-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", "keccak-hasher", ] diff --git a/cumulus/bridges/bin/runtime-common/Cargo.toml b/cumulus/bridges/bin/runtime-common/Cargo.toml index 9d54c3b86cbb..d6d0ada43777 100644 --- a/cumulus/bridges/bin/runtime-common/Cargo.toml +++ b/cumulus/bridges/bin/runtime-common/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0-or-later WITH Classpath-exception-2.0" [dependencies] codec = { package = "parity-scale-codec", version = "3.1.5", default-features = false, features = ["derive"] } -hash-db = { version = "0.16.0", default-features = false } +hash-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value", default-features = false } log = { version = "0.4.20", default-features = false } scale-info = { version = "2.9.0", default-features = false, features = ["derive"] } static_assertions = { version = "1.1", optional = true } diff --git a/cumulus/bridges/primitives/runtime/Cargo.toml b/cumulus/bridges/primitives/runtime/Cargo.toml index bf7f2cd6f976..4d7728ea7768 100644 --- a/cumulus/bridges/primitives/runtime/Cargo.toml +++ b/cumulus/bridges/primitives/runtime/Cargo.toml @@ -8,7 +8,7 @@ license = "GPL-3.0-or-later WITH Classpath-exception-2.0" [dependencies] codec = { package = "parity-scale-codec", version = "3.1.5", default-features = false } -hash-db = { version = "0.16.0", default-features = false } +hash-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value", default-features = false } impl-trait-for-tuples = "0.2.2" log = { version = "0.4.19", default-features = false } num-traits = { version = "0.2", default-features = false } @@ -25,7 +25,7 @@ sp-runtime = { path = "../../../../substrate/primitives/runtime", default-featur sp-state-machine = { path = "../../../../substrate/primitives/state-machine", default-features = false } sp-std = { path = "../../../../substrate/primitives/std", default-features = false } sp-trie = { path = "../../../../substrate/primitives/trie", default-features = false } -trie-db = { version = "0.27.1", default-features = false } +trie-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value", default-features = false } [dev-dependencies] hex-literal = "0.4" diff --git a/cumulus/pallets/parachain-system/Cargo.toml b/cumulus/pallets/parachain-system/Cargo.toml index 125fe01af4af..823730c606ca 100644 --- a/cumulus/pallets/parachain-system/Cargo.toml +++ b/cumulus/pallets/parachain-system/Cargo.toml @@ -11,7 +11,7 @@ codec = { package = "parity-scale-codec", version = "3.0.0", default-features = environmental = { version = "1.1.4", default-features = false } impl-trait-for-tuples = "0.2.1" log = { version = "0.4.20", default-features = false } -trie-db = { version = "0.27.1", default-features = false } +trie-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value", default-features = false } scale-info = { version = "2.9.0", default-features = false, features = ["derive"] } # Substrate diff --git a/substrate/bin/node/bench/Cargo.toml b/substrate/bin/node/bench/Cargo.toml index bef67b995be3..39b5db365081 100644 --- a/substrate/bin/node/bench/Cargo.toml +++ b/substrate/bin/node/bench/Cargo.toml @@ -33,7 +33,7 @@ sc-basic-authorship = { path = "../../../client/basic-authorship" } sp-inherents = { path = "../../../primitives/inherents" } sp-timestamp = { path = "../../../primitives/timestamp", default-features = false} sp-tracing = { path = "../../../primitives/tracing" } -hash-db = "0.16.0" +hash-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value" } tempfile = "3.1.0" fs_extra = "1" rand = { version = "0.8.5", features = ["small_rng"] } diff --git a/substrate/client/db/Cargo.toml b/substrate/client/db/Cargo.toml index 025d0cac212c..2ccae902b58a 100644 --- a/substrate/client/db/Cargo.toml +++ b/substrate/client/db/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] codec = { package = "parity-scale-codec", version = "3.6.1", features = [ "derive", ] } -hash-db = "0.16.0" +hash-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value" } kvdb = "0.13.0" kvdb-memorydb = "0.13.0" kvdb-rocksdb = { version = "0.19.0", optional = true } diff --git a/substrate/primitives/api/Cargo.toml b/substrate/primitives/api/Cargo.toml index 1f23bbed8bee..fd8002343afc 100644 --- a/substrate/primitives/api/Cargo.toml +++ b/substrate/primitives/api/Cargo.toml @@ -22,7 +22,7 @@ sp-externalities = { path = "../externalities", default-features = false, option sp-version = { path = "../version", default-features = false} sp-state-machine = { path = "../state-machine", default-features = false, optional = true} sp-trie = { path = "../trie", default-features = false, optional = true} -hash-db = { version = "0.16.0", optional = true } +hash-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value", optional = true } thiserror = { version = "1.0.30", optional = true } scale-info = { version = "2.1.1", default-features = false, features = ["derive"] } sp-metadata-ir = { path = "../metadata-ir", default-features = false, optional = true} diff --git a/substrate/primitives/core/Cargo.toml b/substrate/primitives/core/Cargo.toml index 12360472a4e4..2d42c39f064e 100644 --- a/substrate/primitives/core/Cargo.toml +++ b/substrate/primitives/core/Cargo.toml @@ -21,7 +21,7 @@ serde = { version = "1.0.163", optional = true, default-features = false, featu bounded-collections = { version = "0.1.8", default-features = false } primitive-types = { version = "0.12.0", default-features = false, features = ["codec", "scale-info"] } impl-serde = { version = "0.4.0", default-features = false, optional = true } -hash-db = { version = "0.16.0", default-features = false } +hash-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value", default-features = false } hash256-std-hasher = { version = "0.15.2", default-features = false } bs58 = { version = "0.4.0", default-features = false, optional = true } rand = { version = "0.8.5", features = ["small_rng"], optional = true } diff --git a/substrate/primitives/state-machine/Cargo.toml b/substrate/primitives/state-machine/Cargo.toml index f9793f8ddef9..7a06fc11b615 100644 --- a/substrate/primitives/state-machine/Cargo.toml +++ b/substrate/primitives/state-machine/Cargo.toml @@ -15,7 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false } -hash-db = { version = "0.16.0", default-features = false } +hash-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value", default-features = false } log = { version = "0.4.17", default-features = false } parking_lot = { version = "0.12.1", optional = true } rand = { version = "0.8.5", optional = true } @@ -27,7 +27,7 @@ sp-externalities = { path = "../externalities", default-features = false} sp-panic-handler = { path = "../panic-handler", optional = true} sp-std = { path = "../std", default-features = false} sp-trie = { path = "../trie", default-features = false} -trie-db = { version = "0.27.1", default-features = false } +trie-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value", default-features = false } [dev-dependencies] array-bytes = "6.1" diff --git a/substrate/primitives/trie/Cargo.toml b/substrate/primitives/trie/Cargo.toml index 58a0f5f9659e..5f6a39962106 100644 --- a/substrate/primitives/trie/Cargo.toml +++ b/substrate/primitives/trie/Cargo.toml @@ -21,16 +21,16 @@ harness = false ahash = { version = "0.8.2", optional = true } codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false } hashbrown = { version = "0.13.2", optional = true } -hash-db = { version = "0.16.0", default-features = false } +hash-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value", default-features = false } lazy_static = { version = "1.4.0", optional = true } -memory-db = { version = "0.32.0", default-features = false } +memory-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value", default-features = false } nohash-hasher = { version = "0.2.0", optional = true } parking_lot = { version = "0.12.1", optional = true } scale-info = { version = "2.5.0", default-features = false, features = ["derive"] } thiserror = { version = "1.0.30", optional = true } tracing = { version = "0.1.29", optional = true } -trie-db = { version = "0.27.0", default-features = false } -trie-root = { version = "0.18.0", default-features = false } +trie-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value", default-features = false } +trie-root = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value", default-features = false } sp-core = { path = "../core", default-features = false} sp-std = { path = "../std", default-features = false} schnellru = { version = "0.2.1", optional = true } diff --git a/substrate/test-utils/runtime/Cargo.toml b/substrate/test-utils/runtime/Cargo.toml index 0a9406e8af91..94f10418d731 100644 --- a/substrate/test-utils/runtime/Cargo.toml +++ b/substrate/test-utils/runtime/Cargo.toml @@ -40,7 +40,7 @@ pallet-timestamp = { path = "../../frame/timestamp", default-features = false} sp-consensus-grandpa = { path = "../../primitives/consensus/grandpa", default-features = false, features = ["serde"] } sp-trie = { path = "../../primitives/trie", default-features = false} sp-transaction-pool = { path = "../../primitives/transaction-pool", default-features = false} -trie-db = { version = "0.27.0", default-features = false } +trie-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value", default-features = false } sc-service = { path = "../../client/service", default-features = false, features = ["test-helpers"], optional = true} sp-state-machine = { path = "../../primitives/state-machine", default-features = false} sp-externalities = { path = "../../primitives/externalities", default-features = false} diff --git a/substrate/utils/binary-merkle-tree/Cargo.toml b/substrate/utils/binary-merkle-tree/Cargo.toml index fc7566ab2404..574989a2794c 100644 --- a/substrate/utils/binary-merkle-tree/Cargo.toml +++ b/substrate/utils/binary-merkle-tree/Cargo.toml @@ -11,7 +11,7 @@ homepage = "https://substrate.io" [dependencies] array-bytes = { version = "6.1", optional = true } log = { version = "0.4", default-features = false, optional = true } -hash-db = { version = "0.16.0", default-features = false } +hash-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value", default-features = false } [dev-dependencies] array-bytes = "6.1" diff --git a/substrate/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml b/substrate/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml index b09b0c578f30..c3bfc480c03e 100644 --- a/substrate/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml +++ b/substrate/utils/frame/rpc/state-trie-migration-rpc/Cargo.toml @@ -19,7 +19,7 @@ serde = { version = "1", features = ["derive"] } sp-core = { path = "../../../../primitives/core" } sp-state-machine = { path = "../../../../primitives/state-machine" } sp-trie = { path = "../../../../primitives/trie" } -trie-db = "0.27.0" +trie-db = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value" } jsonrpsee = { version = "0.16.2", features = ["client-core", "server", "macros"] } From e51afac08a6f6e09da87ac7570f0eecb786e419f Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Fri, 25 Aug 2023 13:46:13 +0300 Subject: [PATCH 03/17] client: Expose merkle value on storageProvider trait Signed-off-by: Alexandru Vasile --- substrate/client/api/src/backend.rs | 14 ++++++++++++++ substrate/client/service/src/client/client.rs | 16 ++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/substrate/client/api/src/backend.rs b/substrate/client/api/src/backend.rs index 2d8fdef77cdb..f54d2eae3ba0 100644 --- a/substrate/client/api/src/backend.rs +++ b/substrate/client/api/src/backend.rs @@ -470,6 +470,20 @@ pub trait StorageProvider> { child_info: &ChildInfo, key: &StorageKey, ) -> sp_blockchain::Result>; + + /// Given a block's `Hash` and a key, return the closest merkle value. + fn closest_merkle_value( + &self, + hash: Block::Hash, + key: &StorageKey, + ) -> sp_blockchain::Result>; + + /// Given a block's `Hash`, a key and a child storage key, return the closest merkle value. + fn child_closest_merkle_value( + &self, + hash: Block::Hash, + key: &StorageKey, + ) -> sp_blockchain::Result>; } /// Client backend. diff --git a/substrate/client/service/src/client/client.rs b/substrate/client/service/src/client/client.rs index a0983d823e5b..3cb2614b40cb 100644 --- a/substrate/client/service/src/client/client.rs +++ b/substrate/client/service/src/client/client.rs @@ -1547,6 +1547,22 @@ where .child_storage_hash(child_info, &key.0) .map_err(|e| sp_blockchain::Error::from_state(Box::new(e))) } + + fn closest_merkle_value( + &self, + hash: ::Hash, + key: &StorageKey, + ) -> blockchain::Result::Hash>> { + Ok(None) + } + + fn child_closest_merkle_value( + &self, + hash: ::Hash, + key: &StorageKey, + ) -> blockchain::Result::Hash>> { + Ok(None) + } } impl HeaderMetadata for Client From bc39b11f37bed689790326546314afff9c3a485c Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Fri, 25 Aug 2023 13:48:01 +0300 Subject: [PATCH 04/17] primitives/backend: Expose merkle value on Backend trait Signed-off-by: Alexandru Vasile --- substrate/primitives/state-machine/src/backend.rs | 12 +++++++++++- .../primitives/state-machine/src/trie_backend.rs | 12 ++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/substrate/primitives/state-machine/src/backend.rs b/substrate/primitives/state-machine/src/backend.rs index 2a25bdc54d94..f39435ebde76 100644 --- a/substrate/primitives/state-machine/src/backend.rs +++ b/substrate/primitives/state-machine/src/backend.rs @@ -195,7 +195,17 @@ pub trait Backend: sp_std::fmt::Debug { /// Get keyed storage value hash or None if there is nothing associated. fn storage_hash(&self, key: &[u8]) -> Result, Self::Error>; - /// Get keyed child storage or None if there is nothing associated. + /// Get the merkle value or None if there is nothing associated. + fn closest_merkle_value(&self, key: &[u8]) -> Result, Self::Error>; + + /// Get the child merkle value or None if there is nothing associated. + fn child_closest_merkle_value( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Result, Self::Error>; + + /// Get child keyed child storage or None if there is nothing associated. fn child_storage( &self, child_info: &ChildInfo, diff --git a/substrate/primitives/state-machine/src/trie_backend.rs b/substrate/primitives/state-machine/src/trie_backend.rs index cc7132181f90..2fbe2b454eee 100644 --- a/substrate/primitives/state-machine/src/trie_backend.rs +++ b/substrate/primitives/state-machine/src/trie_backend.rs @@ -405,6 +405,18 @@ where self.essence.child_storage(child_info, key) } + fn closest_merkle_value(&self, key: &[u8]) -> Result, Self::Error> { + Ok(None) + } + + fn child_closest_merkle_value( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Result, Self::Error> { + Ok(None) + } + fn next_storage_key(&self, key: &[u8]) -> Result, Self::Error> { let (is_cached, mut cache) = access_cache(&self.next_storage_key_cache, Option::take) .map(|cache| (cache.last_key == key, cache)) From d23ad73913e771cbca5501c716532fe84ed60269 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Fri, 25 Aug 2023 13:49:48 +0300 Subject: [PATCH 05/17] primitives/trie: Low level functions for reading the merkle value Signed-off-by: Alexandru Vasile --- substrate/primitives/trie/src/lib.rs | 38 ++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/substrate/primitives/trie/src/lib.rs b/substrate/primitives/trie/src/lib.rs index 94155458569b..4700d3bf8a41 100644 --- a/substrate/primitives/trie/src/lib.rs +++ b/substrate/primitives/trie/src/lib.rs @@ -295,6 +295,24 @@ pub fn read_trie_value( + db: &DB, + root: &TrieHash, + key: &[u8], + recorder: Option<&mut dyn TrieRecorder>>, + cache: Option<&mut dyn TrieCache>, +) -> Result>, Box>> +where + DB: hash_db::HashDBRef, +{ + TrieDBBuilder::::new(db, root) + .with_optional_cache(cache) + .with_optional_recorder(recorder) + .build() + .get_closest_merkle_value(key) +} + /// Read a value from the trie with given Query. pub fn read_trie_value_with< L: TrieLayout, @@ -397,6 +415,26 @@ where .get_hash(key) } +/// Read the closest merkle value from the child trie. +pub fn read_child_trie_closest_merkle_value( + keyspace: &[u8], + db: &DB, + root: &TrieHash, + key: &[u8], + recorder: Option<&mut dyn TrieRecorder>>, + cache: Option<&mut dyn TrieCache>, +) -> Result>, Box>> +where + DB: hash_db::HashDBRef, +{ + let db = KeySpacedDB::new(db, keyspace); + TrieDBBuilder::::new(&db, &root) + .with_optional_recorder(recorder) + .with_optional_cache(cache) + .build() + .get_closest_merkle_value(key) +} + /// Read a value from the child trie with given query. pub fn read_child_trie_value_with( keyspace: &[u8], From fe1289645a82bd4249c78769fc26cd323d84aaff Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Fri, 25 Aug 2023 13:52:15 +0300 Subject: [PATCH 06/17] primitives/trie: Implement closest merkle value on TrieBackendEssence.. and TrieBackend Signed-off-by: Alexandru Vasile --- .../state-machine/src/trie_backend.rs | 4 +- .../state-machine/src/trie_backend_essence.rs | 38 ++++++++++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/substrate/primitives/state-machine/src/trie_backend.rs b/substrate/primitives/state-machine/src/trie_backend.rs index 2fbe2b454eee..82f9c0e2835e 100644 --- a/substrate/primitives/state-machine/src/trie_backend.rs +++ b/substrate/primitives/state-machine/src/trie_backend.rs @@ -406,7 +406,7 @@ where } fn closest_merkle_value(&self, key: &[u8]) -> Result, Self::Error> { - Ok(None) + self.essence.closest_merkle_value(key) } fn child_closest_merkle_value( @@ -414,7 +414,7 @@ where child_info: &ChildInfo, key: &[u8], ) -> Result, Self::Error> { - Ok(None) + self.essence.child_closest_merkle_value(child_info, key) } fn next_storage_key(&self, key: &[u8]) -> Result, Self::Error> { diff --git a/substrate/primitives/state-machine/src/trie_backend_essence.rs b/substrate/primitives/state-machine/src/trie_backend_essence.rs index 4bb51f4a1343..18273d519760 100644 --- a/substrate/primitives/state-machine/src/trie_backend_essence.rs +++ b/substrate/primitives/state-machine/src/trie_backend_essence.rs @@ -32,8 +32,9 @@ use sp_std::{boxed::Box, marker::PhantomData, vec::Vec}; #[cfg(feature = "std")] use sp_trie::recorder::Recorder; use sp_trie::{ - child_delta_trie_root, delta_trie_root, empty_child_trie_root, read_child_trie_hash, - read_child_trie_value, read_trie_value, + child_delta_trie_root, delta_trie_root, empty_child_trie_root, + read_child_trie_closest_merkle_value, read_child_trie_hash, read_child_trie_value, + read_trie_closest_merkle_value, read_trie_value, trie_types::{TrieDBBuilder, TrieError}, DBValue, KeySpacedDB, NodeCodec, PrefixedMemoryDB, Trie, TrieCache, TrieDBRawIterator, TrieRecorder, @@ -574,6 +575,39 @@ where }) } + /// Get the closest merkle value at given key. + pub fn closest_merkle_value(&self, key: &[u8]) -> Result> { + let map_e = |e| format!("Trie lookup error: {}", e); + + self.with_recorder_and_cache(None, |recorder, cache| { + read_trie_closest_merkle_value::, _>(self, &self.root, key, recorder, cache) + .map_err(map_e) + }) + } + + /// Get the child closest merkle value at given key. + pub fn child_closest_merkle_value( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Result> { + let Some(child_root) = self.child_root(child_info)? else { return Ok(None) }; + + let map_e = |e| format!("Trie lookup error: {}", e); + + self.with_recorder_and_cache(Some(child_root), |recorder, cache| { + read_child_trie_closest_merkle_value::, _>( + child_info.keyspace(), + self, + &child_root, + key, + recorder, + cache, + ) + .map_err(map_e) + }) + } + /// Create a raw iterator over the storage. pub fn raw_iter(&self, args: IterArgs) -> Result> { let root = if let Some(child_info) = args.child_info.as_ref() { From cb7fd4d257460ca3bbd4f5b58e1ceddcee16003d Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Fri, 25 Aug 2023 13:56:07 +0300 Subject: [PATCH 07/17] client: Implement closest merkle value up until `StorageProvider` Signed-off-by: Alexandru Vasile --- substrate/client/api/src/backend.rs | 1 + substrate/client/db/src/bench.rs | 18 ++++++++++++++++++ substrate/client/db/src/lib.rs | 12 ++++++++++++ substrate/client/db/src/record_stats_state.rs | 12 ++++++++++++ substrate/client/service/src/client/client.rs | 9 +++++++-- 5 files changed, 50 insertions(+), 2 deletions(-) diff --git a/substrate/client/api/src/backend.rs b/substrate/client/api/src/backend.rs index f54d2eae3ba0..62a5289df12d 100644 --- a/substrate/client/api/src/backend.rs +++ b/substrate/client/api/src/backend.rs @@ -482,6 +482,7 @@ pub trait StorageProvider> { fn child_closest_merkle_value( &self, hash: Block::Hash, + child_info: &ChildInfo, key: &StorageKey, ) -> sp_blockchain::Result>; } diff --git a/substrate/client/db/src/bench.rs b/substrate/client/db/src/bench.rs index 38c37a42ede7..671a230a2aba 100644 --- a/substrate/client/db/src/bench.rs +++ b/substrate/client/db/src/bench.rs @@ -382,6 +382,24 @@ impl StateBackend> for BenchmarkingState { .child_storage_hash(child_info, key) } + fn closest_merkle_value(&self, key: &[u8]) -> Result, Self::Error> { + self.add_read_key(None, key); + self.state.borrow().as_ref().ok_or_else(state_err)?.closest_merkle_value(key) + } + + fn child_closest_merkle_value( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Result, Self::Error> { + self.add_read_key(None, key); + self.state + .borrow() + .as_ref() + .ok_or_else(state_err)? + .child_closest_merkle_value(child_info, key) + } + fn exists_storage(&self, key: &[u8]) -> Result { self.add_read_key(None, key); self.state.borrow().as_ref().ok_or_else(state_err)?.exists_storage(key) diff --git a/substrate/client/db/src/lib.rs b/substrate/client/db/src/lib.rs index 73fb4f8ce6db..37aa878878a6 100644 --- a/substrate/client/db/src/lib.rs +++ b/substrate/client/db/src/lib.rs @@ -214,6 +214,18 @@ impl StateBackend> for RefTrackingState { self.state.child_storage_hash(child_info, key) } + fn closest_merkle_value(&self, key: &[u8]) -> Result, Self::Error> { + self.state.closest_merkle_value(key) + } + + fn child_closest_merkle_value( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Result, Self::Error> { + self.state.child_closest_merkle_value(child_info, key) + } + fn exists_storage(&self, key: &[u8]) -> Result { self.state.exists_storage(key) } diff --git a/substrate/client/db/src/record_stats_state.rs b/substrate/client/db/src/record_stats_state.rs index 29ece84f97e5..4d118df9a176 100644 --- a/substrate/client/db/src/record_stats_state.rs +++ b/substrate/client/db/src/record_stats_state.rs @@ -144,6 +144,18 @@ impl>, B: BlockT> StateBackend> self.state.child_storage_hash(child_info, key) } + fn closest_merkle_value(&self, key: &[u8]) -> Result, Self::Error> { + self.state.closest_merkle_value(key) + } + + fn child_closest_merkle_value( + &self, + child_info: &ChildInfo, + key: &[u8], + ) -> Result, Self::Error> { + self.state.child_closest_merkle_value(child_info, key) + } + fn exists_storage(&self, key: &[u8]) -> Result { self.state.exists_storage(key) } diff --git a/substrate/client/service/src/client/client.rs b/substrate/client/service/src/client/client.rs index 3cb2614b40cb..909d8942ec2a 100644 --- a/substrate/client/service/src/client/client.rs +++ b/substrate/client/service/src/client/client.rs @@ -1553,15 +1553,20 @@ where hash: ::Hash, key: &StorageKey, ) -> blockchain::Result::Hash>> { - Ok(None) + self.state_at(hash)? + .closest_merkle_value(&key.0) + .map_err(|e| sp_blockchain::Error::from_state(Box::new(e))) } fn child_closest_merkle_value( &self, hash: ::Hash, + child_info: &ChildInfo, key: &StorageKey, ) -> blockchain::Result::Hash>> { - Ok(None) + self.state_at(hash)? + .child_closest_merkle_value(child_info, &key.0) + .map_err(|e| sp_blockchain::Error::from_state(Box::new(e))) } } From 766de69f30ba2b54ec42e4c7790c471751700f46 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Fri, 25 Aug 2023 13:56:40 +0300 Subject: [PATCH 08/17] chainHead/storage: Query the storage for closest merkle value Signed-off-by: Alexandru Vasile --- .../src/chain_head/chain_head_storage.rs | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs index 5e1f38f9a997..224a50b48778 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs @@ -145,6 +145,31 @@ where .unwrap_or_else(|error| QueryResult::Err(error.to_string())) } + /// Fetch the closest merkle value. + fn query_storage_merkle_value( + &self, + hash: Block::Hash, + key: &StorageKey, + child_key: Option<&ChildInfo>, + ) -> QueryResult { + let result = if let Some(child_key) = child_key { + self.client.child_closest_merkle_value(hash, child_key, key) + } else { + self.client.closest_merkle_value(hash, key) + }; + + result + .map(|opt| { + QueryResult::Ok(opt.map(|storage_data| StorageResult { + key: hex_string(&key.0), + result: StorageResultType::ClosestDescendantMerkleValue(hex_string( + &storage_data.as_ref(), + )), + })) + }) + .unwrap_or_else(|error| QueryResult::Err(error.to_string())) + } + /// Iterate over at most `operation_max_storage_items` keys. /// /// Returns the storage result with a potential next key to resume iteration. @@ -288,13 +313,21 @@ where return }, }, + StorageQueryType::ClosestDescendantMerkleValue => + match self.query_storage_merkle_value(hash, &item.key, child_key.as_ref()) { + Ok(Some(value)) => storage_results.push(value), + Ok(None) => continue, + Err(error) => { + send_error::(&sender, operation.operation_id(), error); + return + }, + }, StorageQueryType::DescendantsValues => self .iter_operations .push_back(QueryIter { next_key: item.key, ty: IterQueryType::Value }), StorageQueryType::DescendantsHashes => self .iter_operations .push_back(QueryIter { next_key: item.key, ty: IterQueryType::Hash }), - _ => continue, }; } From d23e1c1217ae367671f4f729f43aca16dc1b1e52 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Fri, 25 Aug 2023 13:59:45 +0300 Subject: [PATCH 09/17] chainHead/tests: Check merkle values propagate to the chainHead_storage Signed-off-by: Alexandru Vasile --- .../rpc-spec-v2/src/chain_head/tests.rs | 183 ++++++++++++++++++ 1 file changed, 183 insertions(+) diff --git a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs index 00ed9089058e..dc8c55dd708c 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs @@ -2565,3 +2565,186 @@ async fn stop_storage_operation() { ) .await; } + +#[tokio::test] +async fn storage_closest_merkle_value() { + let (mut client, api, mut sub, sub_id, _) = setup_api().await; + + /// The core of this test. + /// + /// Checks keys that are exact match, keys with descedant and keys that should not return + /// values. + /// + /// Returns (key, merkle value) pairs. + async fn expect_merkle_request( + api: &RpcModule>>, + mut sub: &mut RpcSubscription, + sub_id: String, + block_hash: String, + ) -> HashMap { + // Valid call with storage at the keys. + let response: MethodResponse = api + .call( + "chainHead_unstable_storage", + rpc_params![ + &sub_id, + &block_hash, + vec![ + StorageQuery { + key: hex_string(b":AAAA"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + StorageQuery { + key: hex_string(b":AAAB"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + // Key with descedent. + StorageQuery { + key: hex_string(b":A"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + StorageQuery { + key: hex_string(b":AA"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + // Keys below this comment do not produce a result. + // Key that exceed the keyspace of the trie. + StorageQuery { + key: hex_string(b":AAAAX"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + StorageQuery { + key: hex_string(b":AAABX"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + // Key that are not part of the trie. + StorageQuery { + key: hex_string(b":AAX"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + StorageQuery { + key: hex_string(b":AAAX"), + query_type: StorageQueryType::ClosestDescendantMerkleValue + }, + ] + ], + ) + .await + .unwrap(); + let operation_id = match response { + MethodResponse::Started(started) => started.operation_id, + MethodResponse::LimitReached => panic!("Expected started response"), + }; + + let event = get_next_event::>(&mut sub).await; + let merkle_values: HashMap<_, _> = match event { + FollowEvent::OperationStorageItems(res) => { + assert_eq!(res.operation_id, operation_id); + + res.items + .into_iter() + .map(|res| { + let value = match res.result { + StorageResultType::ClosestDescendantMerkleValue(value) => value, + _ => panic!("Unexpected StorageResultType"), + }; + (res.key, value) + }) + .collect() + }, + _ => panic!("Expected OperationStorageItems event"), + }; + + // Finished. + assert_matches!( + get_next_event::>(&mut sub).await, + FollowEvent::OperationStorageDone(done) if done.operation_id == operation_id + ); + + // Response for AAAA, AAAB, A and AA. + assert_eq!(merkle_values.len(), 4); + + // While checking for expected merkle values to align, + // the following will check that the returned keys are + // expected. + + // Values for AAAA and AAAB are different. + assert_ne!( + merkle_values.get(&hex_string(b":AAAA")).unwrap(), + merkle_values.get(&hex_string(b":AAAB")).unwrap() + ); + + // Values for A and AA should be on the same branch node. + assert_eq!( + merkle_values.get(&hex_string(b":A")).unwrap(), + merkle_values.get(&hex_string(b":AA")).unwrap() + ); + // The branch node value must be different than the leaf of either + // AAAA and AAAB. + assert_ne!( + merkle_values.get(&hex_string(b":A")).unwrap(), + merkle_values.get(&hex_string(b":AAAA")).unwrap() + ); + assert_ne!( + merkle_values.get(&hex_string(b":A")).unwrap(), + merkle_values.get(&hex_string(b":AAAB")).unwrap() + ); + + merkle_values + } + + // Import a new block with storage changes. + let mut builder = client.new_block(Default::default()).unwrap(); + builder.push_storage_change(b":AAAA".to_vec(), Some(vec![1; 64])).unwrap(); + builder.push_storage_change(b":AAAB".to_vec(), Some(vec![2; 64])).unwrap(); + let block = builder.build().unwrap().block; + let block_hash = format!("{:?}", block.header.hash()); + client.import(BlockOrigin::Own, block.clone()).await.unwrap(); + + // Ensure the imported block is propagated and pinned for this subscription. + assert_matches!( + get_next_event::>(&mut sub).await, + FollowEvent::NewBlock(_) + ); + assert_matches!( + get_next_event::>(&mut sub).await, + FollowEvent::BestBlockChanged(_) + ); + + let merkle_values_lhs = expect_merkle_request(&api, &mut sub, sub_id.clone(), block_hash).await; + + // Import a new block with and change AAAB value. + let mut builder = client.new_block(Default::default()).unwrap(); + builder.push_storage_change(b":AAAA".to_vec(), Some(vec![1; 64])).unwrap(); + builder.push_storage_change(b":AAAB".to_vec(), Some(vec![3; 64])).unwrap(); + let block = builder.build().unwrap().block; + let block_hash = format!("{:?}", block.header.hash()); + client.import(BlockOrigin::Own, block.clone()).await.unwrap(); + + // Ensure the imported block is propagated and pinned for this subscription. + assert_matches!( + get_next_event::>(&mut sub).await, + FollowEvent::NewBlock(_) + ); + assert_matches!( + get_next_event::>(&mut sub).await, + FollowEvent::BestBlockChanged(_) + ); + + let merkle_values_rhs = expect_merkle_request(&api, &mut sub, sub_id.clone(), block_hash).await; + + // Change propagated to the root. + assert_ne!( + merkle_values_lhs.get(&hex_string(b":A")).unwrap(), + merkle_values_rhs.get(&hex_string(b":A")).unwrap() + ); + assert_ne!( + merkle_values_lhs.get(&hex_string(b":AAAB")).unwrap(), + merkle_values_rhs.get(&hex_string(b":AAAB")).unwrap() + ); + // However the AAAA branch leaf remains unchenged. + assert_eq!( + merkle_values_lhs.get(&hex_string(b":AAAA")).unwrap(), + merkle_values_rhs.get(&hex_string(b":AAAA")).unwrap() + ); +} From 36077858659624b6ed3a57e4962013a22bfe7b92 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Mon, 11 Sep 2023 14:18:45 +0300 Subject: [PATCH 10/17] primitives: Adjust merkle value api Signed-off-by: Alexandru Vasile --- substrate/primitives/trie/src/lib.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/substrate/primitives/trie/src/lib.rs b/substrate/primitives/trie/src/lib.rs index 4700d3bf8a41..05b40cd3b6c4 100644 --- a/substrate/primitives/trie/src/lib.rs +++ b/substrate/primitives/trie/src/lib.rs @@ -45,7 +45,6 @@ pub use storage_proof::{CompactProof, StorageProof}; /// for trie compact proof. pub use trie_codec::{decode_compact, encode_compact, Error as CompactProofError}; pub use trie_db::proof::VerifyError; -use trie_db::proof::{generate_proof, verify_proof}; /// Various re-exports from the `trie-db` crate. pub use trie_db::{ nibble_ops, @@ -53,6 +52,10 @@ pub use trie_db::{ CError, DBValue, Query, Recorder, Trie, TrieCache, TrieConfiguration, TrieDBIterator, TrieDBKeyIterator, TrieDBRawIterator, TrieLayout, TrieMut, TrieRecorder, }; +use trie_db::{ + proof::{generate_proof, verify_proof}, + MerkleValue, +}; /// The Substrate format implementation of `TrieStream`. pub use trie_stream::TrieStream; @@ -295,14 +298,15 @@ pub fn read_trie_value( +/// Read the [`trie_db::MerkleValue`] of the node that is the closest descendant for +/// the provided key. +pub fn read_trie_first_descedant_value( db: &DB, root: &TrieHash, key: &[u8], recorder: Option<&mut dyn TrieRecorder>>, cache: Option<&mut dyn TrieCache>, -) -> Result>, Box>> +) -> Result>>, Box>> where DB: hash_db::HashDBRef, { @@ -310,7 +314,7 @@ where .with_optional_cache(cache) .with_optional_recorder(recorder) .build() - .get_closest_merkle_value(key) + .lookup_first_descendant(key) } /// Read a value from the trie with given Query. @@ -415,15 +419,16 @@ where .get_hash(key) } -/// Read the closest merkle value from the child trie. -pub fn read_child_trie_closest_merkle_value( +/// Read the [`trie_db::MerkleValue`] of the node that is the closest descendant for +/// the provided child key. +pub fn read_child_trie_first_descedant_value( keyspace: &[u8], db: &DB, root: &TrieHash, key: &[u8], recorder: Option<&mut dyn TrieRecorder>>, cache: Option<&mut dyn TrieCache>, -) -> Result>, Box>> +) -> Result>>, Box>> where DB: hash_db::HashDBRef, { @@ -432,7 +437,7 @@ where .with_optional_recorder(recorder) .with_optional_cache(cache) .build() - .get_closest_merkle_value(key) + .lookup_first_descendant(key) } /// Read a value from the child trie with given query. From 62f677a57c735b5b552fce2bf13c8ccab0d800ce Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Mon, 11 Sep 2023 16:35:58 +0300 Subject: [PATCH 11/17] Update interface with merkle value changes Signed-off-by: Alexandru Vasile --- Cargo.lock | 1 + substrate/client/api/Cargo.toml | 1 + substrate/client/api/src/backend.rs | 5 +++-- substrate/client/db/src/bench.rs | 9 ++++++--- substrate/client/db/src/lib.rs | 9 ++++++--- substrate/client/db/src/record_stats_state.rs | 8 ++++++-- substrate/client/service/src/client/client.rs | 6 +++--- .../primitives/state-machine/src/backend.rs | 6 +++--- .../primitives/state-machine/src/trie_backend.rs | 6 +++--- .../state-machine/src/trie_backend_essence.rs | 16 ++++++++-------- substrate/primitives/trie/src/lib.rs | 7 ++----- 11 files changed, 42 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 71054cfb01f6..d681f3f99ba4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14691,6 +14691,7 @@ dependencies = [ "sp-statement-store", "sp-storage", "sp-test-primitives", + "sp-trie", "substrate-prometheus-endpoint", "substrate-test-runtime", "thiserror", diff --git a/substrate/client/api/Cargo.toml b/substrate/client/api/Cargo.toml index b59149424ed3..2b64c86038dd 100644 --- a/substrate/client/api/Cargo.toml +++ b/substrate/client/api/Cargo.toml @@ -35,6 +35,7 @@ sp-runtime = { path = "../../primitives/runtime", default-features = false} sp-state-machine = { path = "../../primitives/state-machine" } sp-statement-store = { path = "../../primitives/statement-store" } sp-storage = { path = "../../primitives/storage" } +sp-trie = { path = "../../primitives/trie" } [dev-dependencies] thiserror = "1.0.48" diff --git a/substrate/client/api/src/backend.rs b/substrate/client/api/src/backend.rs index 62a5289df12d..31b100433c70 100644 --- a/substrate/client/api/src/backend.rs +++ b/substrate/client/api/src/backend.rs @@ -33,6 +33,7 @@ use sp_state_machine::{ OffchainChangesCollection, StorageCollection, StorageIterator, }; use sp_storage::{ChildInfo, StorageData, StorageKey}; +pub use sp_trie::MerkleValue; use crate::{blockchain::Backend as BlockchainBackend, UsageInfo}; @@ -476,7 +477,7 @@ pub trait StorageProvider> { &self, hash: Block::Hash, key: &StorageKey, - ) -> sp_blockchain::Result>; + ) -> sp_blockchain::Result>>; /// Given a block's `Hash`, a key and a child storage key, return the closest merkle value. fn child_closest_merkle_value( @@ -484,7 +485,7 @@ pub trait StorageProvider> { hash: Block::Hash, child_info: &ChildInfo, key: &StorageKey, - ) -> sp_blockchain::Result>; + ) -> sp_blockchain::Result>>; } /// Client backend. diff --git a/substrate/client/db/src/bench.rs b/substrate/client/db/src/bench.rs index 671a230a2aba..03ad4817b53b 100644 --- a/substrate/client/db/src/bench.rs +++ b/substrate/client/db/src/bench.rs @@ -37,7 +37,7 @@ use sp_state_machine::{ }; use sp_trie::{ cache::{CacheSize, SharedTrieCache}, - prefixed_key, MemoryDB, + prefixed_key, MemoryDB, MerkleValue, }; use std::{ cell::{Cell, RefCell}, @@ -382,7 +382,10 @@ impl StateBackend> for BenchmarkingState { .child_storage_hash(child_info, key) } - fn closest_merkle_value(&self, key: &[u8]) -> Result, Self::Error> { + fn closest_merkle_value( + &self, + key: &[u8], + ) -> Result>, Self::Error> { self.add_read_key(None, key); self.state.borrow().as_ref().ok_or_else(state_err)?.closest_merkle_value(key) } @@ -391,7 +394,7 @@ impl StateBackend> for BenchmarkingState { &self, child_info: &ChildInfo, key: &[u8], - ) -> Result, Self::Error> { + ) -> Result>, Self::Error> { self.add_read_key(None, key); self.state .borrow() diff --git a/substrate/client/db/src/lib.rs b/substrate/client/db/src/lib.rs index 37aa878878a6..194bec8a88eb 100644 --- a/substrate/client/db/src/lib.rs +++ b/substrate/client/db/src/lib.rs @@ -90,7 +90,7 @@ use sp_state_machine::{ OffchainChangesCollection, StateMachineStats, StorageCollection, StorageIterator, StorageKey, StorageValue, UsageInfo as StateUsageInfo, }; -use sp_trie::{cache::SharedTrieCache, prefixed_key, MemoryDB, PrefixedMemoryDB}; +use sp_trie::{cache::SharedTrieCache, prefixed_key, MemoryDB, MerkleValue, PrefixedMemoryDB}; // Re-export the Database trait so that one can pass an implementation of it. pub use sc_state_db::PruningMode; @@ -214,7 +214,10 @@ impl StateBackend> for RefTrackingState { self.state.child_storage_hash(child_info, key) } - fn closest_merkle_value(&self, key: &[u8]) -> Result, Self::Error> { + fn closest_merkle_value( + &self, + key: &[u8], + ) -> Result>, Self::Error> { self.state.closest_merkle_value(key) } @@ -222,7 +225,7 @@ impl StateBackend> for RefTrackingState { &self, child_info: &ChildInfo, key: &[u8], - ) -> Result, Self::Error> { + ) -> Result>, Self::Error> { self.state.child_closest_merkle_value(child_info, key) } diff --git a/substrate/client/db/src/record_stats_state.rs b/substrate/client/db/src/record_stats_state.rs index 4d118df9a176..d9a35c075d79 100644 --- a/substrate/client/db/src/record_stats_state.rs +++ b/substrate/client/db/src/record_stats_state.rs @@ -28,6 +28,7 @@ use sp_state_machine::{ backend::{AsTrieBackend, Backend as StateBackend}, BackendTransaction, IterArgs, StorageIterator, StorageKey, StorageValue, TrieBackend, }; +use sp_trie::MerkleValue; use std::sync::Arc; /// State abstraction for recording stats about state access. @@ -144,7 +145,10 @@ impl>, B: BlockT> StateBackend> self.state.child_storage_hash(child_info, key) } - fn closest_merkle_value(&self, key: &[u8]) -> Result, Self::Error> { + fn closest_merkle_value( + &self, + key: &[u8], + ) -> Result>, Self::Error> { self.state.closest_merkle_value(key) } @@ -152,7 +156,7 @@ impl>, B: BlockT> StateBackend> &self, child_info: &ChildInfo, key: &[u8], - ) -> Result, Self::Error> { + ) -> Result>, Self::Error> { self.state.child_closest_merkle_value(child_info, key) } diff --git a/substrate/client/service/src/client/client.rs b/substrate/client/service/src/client/client.rs index 909d8942ec2a..8917d64174b2 100644 --- a/substrate/client/service/src/client/client.rs +++ b/substrate/client/service/src/client/client.rs @@ -78,7 +78,7 @@ use sp_state_machine::{ ChildStorageCollection, KeyValueStates, KeyValueStorageLevel, StorageCollection, MAX_NESTED_TRIE_DEPTH, }; -use sp_trie::{CompactProof, StorageProof}; +use sp_trie::{CompactProof, StorageProof, MerkleValue}; use std::{ collections::{HashMap, HashSet}, marker::PhantomData, @@ -1552,7 +1552,7 @@ where &self, hash: ::Hash, key: &StorageKey, - ) -> blockchain::Result::Hash>> { + ) -> blockchain::Result::Hash>>> { self.state_at(hash)? .closest_merkle_value(&key.0) .map_err(|e| sp_blockchain::Error::from_state(Box::new(e))) @@ -1563,7 +1563,7 @@ where hash: ::Hash, child_info: &ChildInfo, key: &StorageKey, - ) -> blockchain::Result::Hash>> { + ) -> blockchain::Result::Hash>>> { self.state_at(hash)? .child_closest_merkle_value(child_info, &key.0) .map_err(|e| sp_blockchain::Error::from_state(Box::new(e))) diff --git a/substrate/primitives/state-machine/src/backend.rs b/substrate/primitives/state-machine/src/backend.rs index f39435ebde76..ea9cd442d70b 100644 --- a/substrate/primitives/state-machine/src/backend.rs +++ b/substrate/primitives/state-machine/src/backend.rs @@ -30,7 +30,7 @@ use sp_core::storage::{ChildInfo, StateVersion, TrackedStorageKey}; #[cfg(feature = "std")] use sp_core::traits::RuntimeCode; use sp_std::vec::Vec; -use sp_trie::PrefixedMemoryDB; +use sp_trie::{MerkleValue, PrefixedMemoryDB}; /// A struct containing arguments for iterating over the storage. #[derive(Default)] @@ -196,14 +196,14 @@ pub trait Backend: sp_std::fmt::Debug { fn storage_hash(&self, key: &[u8]) -> Result, Self::Error>; /// Get the merkle value or None if there is nothing associated. - fn closest_merkle_value(&self, key: &[u8]) -> Result, Self::Error>; + fn closest_merkle_value(&self, key: &[u8]) -> Result>, Self::Error>; /// Get the child merkle value or None if there is nothing associated. fn child_closest_merkle_value( &self, child_info: &ChildInfo, key: &[u8], - ) -> Result, Self::Error>; + ) -> Result>, Self::Error>; /// Get child keyed child storage or None if there is nothing associated. fn child_storage( diff --git a/substrate/primitives/state-machine/src/trie_backend.rs b/substrate/primitives/state-machine/src/trie_backend.rs index 82f9c0e2835e..afa80addd2ba 100644 --- a/substrate/primitives/state-machine/src/trie_backend.rs +++ b/substrate/primitives/state-machine/src/trie_backend.rs @@ -30,7 +30,6 @@ use codec::Codec; use hash_db::HashDB; use hash_db::Hasher; use sp_core::storage::{ChildInfo, StateVersion}; -use sp_trie::PrefixedMemoryDB; #[cfg(feature = "std")] use sp_trie::{ cache::{LocalTrieCache, TrieCache}, @@ -39,6 +38,7 @@ use sp_trie::{ }; #[cfg(not(feature = "std"))] use sp_trie::{Error, NodeCodec}; +use sp_trie::{MerkleValue, PrefixedMemoryDB}; use trie_db::TrieCache as TrieCacheT; #[cfg(not(feature = "std"))] use trie_db::{node::NodeOwned, CachedValue}; @@ -405,7 +405,7 @@ where self.essence.child_storage(child_info, key) } - fn closest_merkle_value(&self, key: &[u8]) -> Result, Self::Error> { + fn closest_merkle_value(&self, key: &[u8]) -> Result>, Self::Error> { self.essence.closest_merkle_value(key) } @@ -413,7 +413,7 @@ where &self, child_info: &ChildInfo, key: &[u8], - ) -> Result, Self::Error> { + ) -> Result>, Self::Error> { self.essence.child_closest_merkle_value(child_info, key) } diff --git a/substrate/primitives/state-machine/src/trie_backend_essence.rs b/substrate/primitives/state-machine/src/trie_backend_essence.rs index 18273d519760..ad7aeab899c8 100644 --- a/substrate/primitives/state-machine/src/trie_backend_essence.rs +++ b/substrate/primitives/state-machine/src/trie_backend_essence.rs @@ -33,11 +33,11 @@ use sp_std::{boxed::Box, marker::PhantomData, vec::Vec}; use sp_trie::recorder::Recorder; use sp_trie::{ child_delta_trie_root, delta_trie_root, empty_child_trie_root, - read_child_trie_closest_merkle_value, read_child_trie_hash, read_child_trie_value, - read_trie_closest_merkle_value, read_trie_value, + read_child_trie_first_descedant_value, read_child_trie_hash, read_child_trie_value, + read_trie_first_descedant_value, read_trie_value, trie_types::{TrieDBBuilder, TrieError}, - DBValue, KeySpacedDB, NodeCodec, PrefixedMemoryDB, Trie, TrieCache, TrieDBRawIterator, - TrieRecorder, + DBValue, KeySpacedDB, MerkleValue, NodeCodec, PrefixedMemoryDB, Trie, TrieCache, + TrieDBRawIterator, TrieRecorder, }; #[cfg(feature = "std")] use std::{collections::HashMap, sync::Arc}; @@ -576,11 +576,11 @@ where } /// Get the closest merkle value at given key. - pub fn closest_merkle_value(&self, key: &[u8]) -> Result> { + pub fn closest_merkle_value(&self, key: &[u8]) -> Result>> { let map_e = |e| format!("Trie lookup error: {}", e); self.with_recorder_and_cache(None, |recorder, cache| { - read_trie_closest_merkle_value::, _>(self, &self.root, key, recorder, cache) + read_trie_first_descedant_value::, _>(self, &self.root, key, recorder, cache) .map_err(map_e) }) } @@ -590,13 +590,13 @@ where &self, child_info: &ChildInfo, key: &[u8], - ) -> Result> { + ) -> Result>> { let Some(child_root) = self.child_root(child_info)? else { return Ok(None) }; let map_e = |e| format!("Trie lookup error: {}", e); self.with_recorder_and_cache(Some(child_root), |recorder, cache| { - read_child_trie_closest_merkle_value::, _>( + read_child_trie_first_descedant_value::, _>( child_info.keyspace(), self, &child_root, diff --git a/substrate/primitives/trie/src/lib.rs b/substrate/primitives/trie/src/lib.rs index 05b40cd3b6c4..1a1ed670454d 100644 --- a/substrate/primitives/trie/src/lib.rs +++ b/substrate/primitives/trie/src/lib.rs @@ -44,7 +44,7 @@ pub use storage_proof::{CompactProof, StorageProof}; /// Trie codec reexport, mainly child trie support /// for trie compact proof. pub use trie_codec::{decode_compact, encode_compact, Error as CompactProofError}; -pub use trie_db::proof::VerifyError; +use trie_db::proof::{generate_proof, verify_proof}; /// Various re-exports from the `trie-db` crate. pub use trie_db::{ nibble_ops, @@ -52,10 +52,7 @@ pub use trie_db::{ CError, DBValue, Query, Recorder, Trie, TrieCache, TrieConfiguration, TrieDBIterator, TrieDBKeyIterator, TrieDBRawIterator, TrieLayout, TrieMut, TrieRecorder, }; -use trie_db::{ - proof::{generate_proof, verify_proof}, - MerkleValue, -}; +pub use trie_db::{proof::VerifyError, MerkleValue}; /// The Substrate format implementation of `TrieStream`. pub use trie_stream::TrieStream; From b6dd92e5c82ef4e04f029945702837c19c666a06 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Mon, 11 Sep 2023 16:44:49 +0300 Subject: [PATCH 12/17] chainHead/storage: Adjust the returned values Signed-off-by: Alexandru Vasile --- .../src/chain_head/chain_head_storage.rs | 15 ++++++++++----- substrate/client/service/src/client/client.rs | 6 +++--- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs index 569ec40e2d65..7095548a2b16 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs @@ -160,11 +160,16 @@ where result .map(|opt| { - QueryResult::Ok(opt.map(|storage_data| StorageResult { - key: hex_string(&key.0), - result: StorageResultType::ClosestDescendantMerkleValue(hex_string( - &storage_data.as_ref(), - )), + QueryResult::Ok(opt.map(|storage_data| { + let result = match &storage_data { + sc_client_api::MerkleValue::Node(data) => hex_string(&data.as_slice()), + sc_client_api::MerkleValue::Hash(hash) => hex_string(&hash.as_ref()), + }; + + StorageResult { + key: hex_string(&key.0), + result: StorageResultType::ClosestDescendantMerkleValue(result), + } })) }) .unwrap_or_else(|error| QueryResult::Err(error.to_string())) diff --git a/substrate/client/service/src/client/client.rs b/substrate/client/service/src/client/client.rs index 8917d64174b2..4e3d8aae7a08 100644 --- a/substrate/client/service/src/client/client.rs +++ b/substrate/client/service/src/client/client.rs @@ -78,7 +78,7 @@ use sp_state_machine::{ ChildStorageCollection, KeyValueStates, KeyValueStorageLevel, StorageCollection, MAX_NESTED_TRIE_DEPTH, }; -use sp_trie::{CompactProof, StorageProof, MerkleValue}; +use sp_trie::{CompactProof, MerkleValue, StorageProof}; use std::{ collections::{HashMap, HashSet}, marker::PhantomData, @@ -1552,7 +1552,7 @@ where &self, hash: ::Hash, key: &StorageKey, - ) -> blockchain::Result::Hash>>> { + ) -> blockchain::Result::Hash>>> { self.state_at(hash)? .closest_merkle_value(&key.0) .map_err(|e| sp_blockchain::Error::from_state(Box::new(e))) @@ -1563,7 +1563,7 @@ where hash: ::Hash, child_info: &ChildInfo, key: &StorageKey, - ) -> blockchain::Result::Hash>>> { + ) -> blockchain::Result::Hash>>> { self.state_at(hash)? .child_closest_merkle_value(child_info, &key.0) .map_err(|e| sp_blockchain::Error::from_state(Box::new(e))) From 1e52670374513b721c4e45a663103a853630bc11 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Mon, 11 Sep 2023 18:29:44 +0300 Subject: [PATCH 13/17] bench: Use trie github link Signed-off-by: Alexandru Vasile --- Cargo.lock | 99 +++++++++++++--------------- substrate/primitives/trie/Cargo.toml | 2 +- 2 files changed, 48 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 009d292af9ed..ef5da7aa6e51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1767,7 +1767,7 @@ dependencies = [ "sp-state-machine", "sp-std", "sp-trie", - "trie-db 0.27.1 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", + "trie-db", ] [[package]] @@ -3564,7 +3564,7 @@ dependencies = [ "sp-trie", "sp-version", "staging-xcm", - "trie-db 0.27.1 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", + "trie-db", ] [[package]] @@ -5929,6 +5929,14 @@ dependencies = [ "crunchy", ] +[[package]] +name = "hash256-std-hasher" +version = "0.15.2" +source = "git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value#1fa98e53a783b620d15936250bf2037d87dc05d5" +dependencies = [ + "crunchy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -6733,7 +6741,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19ea4653859ca2266a86419d3f592d3f22e7a854b482f99180d2498507902048" dependencies = [ "hash-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hash256-std-hasher", + "hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tiny-keccak", +] + +[[package]] +name = "keccak-hasher" +version = "0.16.0" +source = "git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value#1fa98e53a783b620d15936250bf2037d87dc05d5" +dependencies = [ + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", + "hash256-std-hasher 0.15.2 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "tiny-keccak", ] @@ -7789,15 +7807,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "memory-db" -version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808b50db46293432a45e63bc15ea51e0ab4c0a1647b8eb114e31a3e698dd6fbe" -dependencies = [ - "hash-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "memory-db" version = "0.32.0" @@ -17047,7 +17056,7 @@ dependencies = [ "ed25519-zebra 3.1.0", "futures", "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", - "hash256-std-hasher", + "hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "impl-serde", "lazy_static", "libsecp256k1", @@ -17321,7 +17330,7 @@ name = "sp-runtime" version = "24.0.0" dependencies = [ "either", - "hash256-std-hasher", + "hash256-std-hasher 0.15.2 (registry+https://github.com/rust-lang/crates.io-index)", "impl-trait-for-tuples", "log", "parity-scale-codec", @@ -17463,7 +17472,7 @@ dependencies = [ "sp-trie", "thiserror", "tracing", - "trie-db 0.27.1 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", + "trie-db", ] [[package]] @@ -17573,7 +17582,7 @@ dependencies = [ "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", "hashbrown 0.13.2", "lazy_static", - "memory-db 0.32.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", + "memory-db", "nohash-hasher", "parity-scale-codec", "parking_lot 0.12.1", @@ -17585,9 +17594,9 @@ dependencies = [ "thiserror", "tracing", "trie-bench", - "trie-db 0.27.1 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", - "trie-root 0.18.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", - "trie-standardmap", + "trie-db", + "trie-root", + "trie-standardmap 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -18176,7 +18185,7 @@ dependencies = [ "sp-runtime", "sp-state-machine", "sp-trie", - "trie-db 0.27.1 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", + "trie-db", ] [[package]] @@ -18251,7 +18260,7 @@ dependencies = [ "sp-version", "substrate-test-runtime-client", "substrate-wasm-builder", - "trie-db 0.27.1 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", + "trie-db", ] [[package]] @@ -19159,30 +19168,16 @@ dependencies = [ [[package]] name = "trie-bench" version = "0.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f54b4f9d51d368e62cf7e0730c7c1e18fc658cc84333656bab5b328f44aa964" +source = "git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value#1fa98e53a783b620d15936250bf2037d87dc05d5" dependencies = [ "criterion 0.4.0", - "hash-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", - "keccak-hasher", - "memory-db 0.32.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", + "keccak-hasher 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", + "memory-db", "parity-scale-codec", - "trie-db 0.27.1 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-root 0.18.0 (registry+https://github.com/rust-lang/crates.io-index)", - "trie-standardmap", -] - -[[package]] -name = "trie-db" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "767abe6ffed88a1889671a102c2861ae742726f52e0a5a425b92c9fbfa7e9c85" -dependencies = [ - "hash-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", - "hashbrown 0.13.2", - "log", - "rustc-hex", - "smallvec", + "trie-db", + "trie-root", + "trie-standardmap 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", ] [[package]] @@ -19197,15 +19192,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "trie-root" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4ed310ef5ab98f5fa467900ed906cb9232dd5376597e00fd4cba2a449d06c0b" -dependencies = [ - "hash-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "trie-root" version = "0.18.0" @@ -19221,7 +19207,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "684aafb332fae6f83d7fe10b3fbfdbe39a1b3234c4e2a618f030815838519516" dependencies = [ "hash-db 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", - "keccak-hasher", + "keccak-hasher 0.16.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "trie-standardmap" +version = "0.16.0" +source = "git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value#1fa98e53a783b620d15936250bf2037d87dc05d5" +dependencies = [ + "hash-db 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", + "keccak-hasher 0.16.0 (git+https://github.com/paritytech/trie.git?branch=lexnv/expose_merkle_value)", ] [[package]] diff --git a/substrate/primitives/trie/Cargo.toml b/substrate/primitives/trie/Cargo.toml index c1355ef389f3..2aba3615f3ea 100644 --- a/substrate/primitives/trie/Cargo.toml +++ b/substrate/primitives/trie/Cargo.toml @@ -38,7 +38,7 @@ schnellru = { version = "0.2.1", optional = true } [dev-dependencies] array-bytes = "6.1" criterion = "0.4.0" -trie-bench = "0.37.0" +trie-bench = { git = "https://github.com/paritytech/trie.git", branch = "lexnv/expose_merkle_value" } trie-standardmap = "0.16.0" sp-runtime = { path = "../runtime" } From 520c65164675e559e13bba276f7ff57603398cba Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Mon, 11 Sep 2023 19:49:51 +0300 Subject: [PATCH 14/17] chainHead/tests: Adjust testing Signed-off-by: Alexandru Vasile --- .../rpc-spec-v2/src/chain_head/test_utils.rs | 21 +++++++++++++++++-- .../rpc-spec-v2/src/chain_head/tests.rs | 7 ++++++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs b/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs index 6e92e87608b4..a901f3039ffe 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs @@ -20,8 +20,8 @@ use parking_lot::Mutex; use sc_client_api::{ execution_extensions::ExecutionExtensions, BlockBackend, BlockImportNotification, BlockchainEvents, CallExecutor, ChildInfo, ExecutorProvider, FinalityNotification, - FinalityNotifications, FinalizeSummary, ImportNotifications, KeysIter, PairsIter, StorageData, - StorageEventStream, StorageKey, StorageProvider, + FinalityNotifications, FinalizeSummary, ImportNotifications, KeysIter, MerkleValue, PairsIter, + StorageData, StorageEventStream, StorageKey, StorageProvider, }; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sp_api::{CallApiAt, CallApiAtParams, NumberFor, RuntimeVersion}; @@ -198,6 +198,23 @@ impl< ) -> sp_blockchain::Result> { self.client.child_storage_hash(hash, child_info, key) } + + fn closest_merkle_value( + &self, + hash: Block::Hash, + key: &StorageKey, + ) -> sp_blockchain::Result>> { + self.client.closest_merkle_value(hash, key) + } + + fn child_closest_merkle_value( + &self, + hash: Block::Hash, + child_info: &ChildInfo, + key: &StorageKey, + ) -> sp_blockchain::Result>> { + self.client.child_closest_merkle_value(hash, child_info, key) + } } impl> CallApiAt for ChainHeadMockClient { diff --git a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs index 4fd608a77ae2..2891be52b39d 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs @@ -43,7 +43,12 @@ use sp_core::{ Blake2Hasher, Hasher, }; use sp_version::RuntimeVersion; -use std::{collections::HashSet, fmt::Debug, sync::Arc, time::Duration}; +use std::{ + collections::{HashMap, HashSet}, + fmt::Debug, + sync::Arc, + time::Duration, +}; use substrate_test_runtime::Transfer; use substrate_test_runtime_client::{ prelude::*, runtime, runtime::RuntimeApi, Backend, BlockBuilderExt, Client, From ba184a06c67d7855cf868253a5c36d245ccb85d8 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Mon, 11 Sep 2023 19:51:07 +0300 Subject: [PATCH 15/17] chainHead: Support merkle value types Signed-off-by: Alexandru Vasile --- substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs index 14364c331e6c..1bf794491149 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs @@ -329,13 +329,6 @@ where let items = items .into_iter() .map(|query| { - if query.query_type == StorageQueryType::ClosestDescendantMerkleValue { - // Note: remove this once all types are implemented. - return Err(ChainHeadRpcError::InvalidParam( - "Storage query type not supported".into(), - )) - } - Ok(StorageQuery { key: StorageKey(parse_hex_param(query.key)?), query_type: query.query_type, From 3294a1daf760b7a5b93fd86187c92f325aa996a4 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile Date: Mon, 11 Sep 2023 20:34:14 +0300 Subject: [PATCH 16/17] chainHead: Fix build Signed-off-by: Alexandru Vasile --- .../client/rpc-spec-v2/src/chain_head/chain_head.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs index 1bf794491149..a8c1c4f7e083 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head.rs @@ -27,7 +27,7 @@ use crate::{ api::ChainHeadApiServer, chain_head_follow::ChainHeadFollower, error::Error as ChainHeadRpcError, - event::{FollowEvent, MethodResponse, OperationError, StorageQuery, StorageQueryType}, + event::{FollowEvent, MethodResponse, OperationError, StorageQuery}, hex_string, subscription::{SubscriptionManagement, SubscriptionManagementError}, }, @@ -329,12 +329,10 @@ where let items = items .into_iter() .map(|query| { - Ok(StorageQuery { - key: StorageKey(parse_hex_param(query.key)?), - query_type: query.query_type, - }) + let key = StorageKey(parse_hex_param(query.key)?); + Ok(StorageQuery { key, query_type: query.query_type }) }) - .collect::, _>>()?; + .collect::, ChainHeadRpcError>>()?; let child_trie = child_trie .map(|child_trie| parse_hex_param(child_trie)) From cd0c0c1be26cd7467cd61024ed33490e643f8368 Mon Sep 17 00:00:00 2001 From: Alexandru Vasile <60601340+lexnv@users.noreply.github.com> Date: Fri, 15 Sep 2023 18:40:53 +0300 Subject: [PATCH 17/17] Update substrate/client/rpc-spec-v2/src/chain_head/tests.rs Co-authored-by: Sebastian Kunert --- substrate/client/rpc-spec-v2/src/chain_head/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs index 2891be52b39d..3ab47991c4e5 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs @@ -2765,7 +2765,7 @@ async fn storage_closest_merkle_value() { merkle_values_lhs.get(&hex_string(b":AAAB")).unwrap(), merkle_values_rhs.get(&hex_string(b":AAAB")).unwrap() ); - // However the AAAA branch leaf remains unchenged. + // However the AAAA branch leaf remains unchanged. assert_eq!( merkle_values_lhs.get(&hex_string(b":AAAA")).unwrap(), merkle_values_rhs.get(&hex_string(b":AAAA")).unwrap()