checkpoint: into main from release/2.2.0 @ 63417e7 (#17494)

Source hash: 63417e7
Remaining commits: 2

chia/cmds/data.py

@@ -51,14 +51,15 @@ def create_changelist_option() -> Callable[[FC], FC]:
     )
 
 
-def create_key_option() -> Callable[[FC], FC]:
+def create_key_option(multiple: bool = False) -> Callable[[FC], FC]:
     return click.option(
         "-k",
         "--key",
-        "key_string",
+        "key_strings" if multiple else "key_string",
         help="str representing the key",
         type=str,
         required=True,
+        multiple=multiple,
     )
 
 
@@ -530,3 +531,48 @@ def wallet_log_in(
             fingerprint=fingerprint,
         )
     )
+
+
+@data_cmd.command(
+    "get_proof",
+    help="Obtains a merkle proof of inclusion for a given key",
+)
+@create_data_store_id_option()
+@create_rpc_port_option()
+@create_key_option(multiple=True)
+@options.create_fingerprint()
+def get_proof(
+    id: str,
+    key_strings: List[str],
+    data_rpc_port: int,
+    fingerprint: Optional[int],
+) -> None:
+    from chia.cmds.data_funcs import get_proof_cmd
+
+    store_id = bytes32.from_hexstr(id)
+
+    run(get_proof_cmd(rpc_port=data_rpc_port, store_id=store_id, fingerprint=fingerprint, key_strings=key_strings))
+
+
+@data_cmd.command(
+    "verify_proof",
+    help="Verifies a merkle proof of inclusion",
+)
+@click.option(
+    "-p",
+    "--proof",
+    "proof_string",
+    help="Proof to validate in JSON format.",
+    type=str,
+)
+@create_rpc_port_option()
+@options.create_fingerprint()
+def verify_proof(
+    proof_string: str,
+    data_rpc_port: int,
+    fingerprint: Optional[int],
+) -> None:
+    from chia.cmds.data_funcs import verify_proof_cmd
+
+    proof_dict = json.loads(proof_string)
+    run(verify_proof_cmd(rpc_port=data_rpc_port, fingerprint=fingerprint, proof=proof_dict))

chia/cmds/data_funcs.py

@@ -286,3 +286,32 @@ async def clear_pending_roots(
         print(json.dumps(result, indent=4, sort_keys=True))
 
     return result
+
+
+async def get_proof_cmd(
+    store_id: bytes32,
+    key_strings: List[str],
+    rpc_port: Optional[int],
+    root_path: Optional[Path] = None,
+    fingerprint: Optional[int] = None,
+) -> Dict[str, Any]:
+    result = dict()
+    async with get_client(rpc_port=rpc_port, fingerprint=fingerprint, root_path=root_path) as (client, _):
+        result = await client.get_proof(store_id=store_id, keys=[hexstr_to_bytes(key) for key in key_strings])
+        print(json.dumps(result, indent=4, sort_keys=True))
+
+    return result
+
+
+async def verify_proof_cmd(
+    proof: Dict[str, Any],
+    rpc_port: Optional[int],
+    root_path: Optional[Path] = None,
+    fingerprint: Optional[int] = None,
+) -> Dict[str, Any]:
+    result = dict()
+    async with get_client(rpc_port=rpc_port, fingerprint=fingerprint, root_path=root_path) as (client, _):
+        result = await client.verify_proof(proof=proof)
+        print(json.dumps(result, indent=4, sort_keys=True))
+
+    return result

chia/data_layer/data_layer.py

@@ -312,14 +312,12 @@ async def get_key_value_hash(
             node = await self.data_store.get_node_by_key(tree_id=store_id, key=key, root_hash=root_hash)
             return node.hash
 
-    async def get_value(self, store_id: bytes32, key: bytes, root_hash: Optional[bytes32] = None) -> Optional[bytes]:
+    async def get_value(self, store_id: bytes32, key: bytes, root_hash: Optional[bytes32] = None) -> bytes:
         await self._update_confirmation_status(tree_id=store_id)
 
         async with self.data_store.transaction():
+            # this either returns the node or raises an exception
             res = await self.data_store.get_node_by_key(tree_id=store_id, key=key, root_hash=root_hash)
-            if res is None:
-                self.log.error("Failed to fetch key")
-                return None
             return res.value
 
     async def get_keys_values(self, store_id: bytes32, root_hash: Optional[bytes32]) -> List[TerminalNode]:

chia/data_layer/data_layer_errors.py

@@ -42,5 +42,9 @@ class OfferIntegrityError(Exception):
     pass
 
 
+class ProofIntegrityError(Exception):
+    pass
+
+
 class LauncherCoinNotFoundError(Exception):
     pass

chia/data_layer/data_layer_util.py

@@ -9,15 +9,19 @@
 import aiosqlite as aiosqlite
 from typing_extensions import final
 
+from chia.data_layer.data_layer_errors import ProofIntegrityError
+from chia.server.ws_connection import WSChiaConnection
 from chia.types.blockchain_format.program import Program
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.util.byte_types import hexstr_to_bytes
 from chia.util.db_wrapper import DBWrapper2
-from chia.util.ints import uint64
+from chia.util.ints import uint8, uint64
 from chia.util.streamable import Streamable, streamable
+from chia.wallet.db_wallet.db_wallet_puzzles import create_host_fullpuz
 
 if TYPE_CHECKING:
     from chia.data_layer.data_store import DataStore
+    from chia.wallet.wallet_node import WalletNode
 
 
 def internal_hash(left_hash: bytes32, right_hash: bytes32) -> bytes32:
@@ -735,3 +739,157 @@ class InsertResult:
 class UnsubscribeData:
     tree_id: bytes32
     retain_data: bool
+
+
+#
+# GetProof and VerifyProof support classes
+#
+@streamable
+@dataclasses.dataclass(frozen=True)
+class ProofLayer(Streamable):
+    # This class is basically Layer but streamable
+    other_hash_side: uint8
+    other_hash: bytes32
+    combined_hash: bytes32
+
+
+@streamable
+@dataclasses.dataclass(frozen=True)
+class HashOnlyProof(Streamable):
+    key_clvm_hash: bytes32
+    value_clvm_hash: bytes32
+    node_hash: bytes32
+    layers: List[ProofLayer]
+
+    def root(self) -> bytes32:
+        if len(self.layers) == 0:
+            return self.node_hash
+        return self.layers[-1].combined_hash
+
+    @classmethod
+    def from_key_value(cls, key: bytes, value: bytes, node_hash: bytes32, layers: List[ProofLayer]) -> HashOnlyProof:
+        return cls(
+            key_clvm_hash=Program.to(key).get_tree_hash(),
+            value_clvm_hash=Program.to(value).get_tree_hash(),
+            node_hash=node_hash,
+            layers=layers,
+        )
+
+
+@streamable
+@dataclasses.dataclass(frozen=True)
+class KeyValueHashes(Streamable):
+    key_clvm_hash: bytes32
+    value_clvm_hash: bytes32
+
+
+@streamable
+@dataclasses.dataclass(frozen=True)
+class ProofResultInclusions(Streamable):
+    store_id: bytes32
+    inclusions: List[KeyValueHashes]
+
+
+@streamable
+@dataclasses.dataclass(frozen=True)
+class GetProofRequest(Streamable):
+    store_id: bytes32
+    keys: List[bytes]
+
+
+@streamable
+@dataclasses.dataclass(frozen=True)
+class StoreProofsHashes(Streamable):
+    store_id: bytes32
+    proofs: List[HashOnlyProof]
+
+
+@streamable
+@dataclasses.dataclass(frozen=True)
+class DLProof(Streamable):
+    store_proofs: StoreProofsHashes
+    coin_id: bytes32
+    inner_puzzle_hash: bytes32
+
+
+@streamable
+@dataclasses.dataclass(frozen=True)
+class GetProofResponse(Streamable):
+    proof: DLProof
+    success: bool
+
+
+@streamable
+@dataclasses.dataclass(frozen=True)
+class VerifyProofResponse(Streamable):
+    verified_clvm_hashes: ProofResultInclusions
+    current_root: bool
+    success: bool
+
+
+def dl_verify_proof_internal(dl_proof: DLProof, puzzle_hash: bytes32) -> List[KeyValueHashes]:
+    """Verify a proof of inclusion for a DL singleton"""
+
+    verified_keys: List[KeyValueHashes] = []
+
+    for reference_proof in dl_proof.store_proofs.proofs:
+        inner_puz_hash = dl_proof.inner_puzzle_hash
+        host_fullpuz_program = create_host_fullpuz(
+            inner_puz_hash, reference_proof.root(), dl_proof.store_proofs.store_id
+        )
+        expected_puzzle_hash = host_fullpuz_program.get_tree_hash_precalc(inner_puz_hash)
+
+        if puzzle_hash != expected_puzzle_hash:
+            raise ProofIntegrityError(
+                "Invalid Proof: incorrect puzzle hash: expected:"
+                f"{expected_puzzle_hash.hex()} received: {puzzle_hash.hex()}"
+            )
+
+        proof = ProofOfInclusion(
+            node_hash=reference_proof.node_hash,
+            layers=[
+                ProofOfInclusionLayer(
+                    other_hash_side=Side(layer.other_hash_side),
+                    other_hash=layer.other_hash,
+                    combined_hash=layer.combined_hash,
+                )
+                for layer in reference_proof.layers
+            ],
+        )
+
+        leaf_hash = internal_hash(left_hash=reference_proof.key_clvm_hash, right_hash=reference_proof.value_clvm_hash)
+        if leaf_hash != proof.node_hash:
+            raise ProofIntegrityError("Invalid Proof: node hash does not match key and value")
+
+        if not proof.valid():
+            raise ProofIntegrityError("Invalid Proof: invalid proof of inclusion found")
+
+        verified_keys.append(
+            KeyValueHashes(key_clvm_hash=reference_proof.key_clvm_hash, value_clvm_hash=reference_proof.value_clvm_hash)
+        )
+
+    return verified_keys
+
+
+async def dl_verify_proof(
+    request: Dict[str, Any],
+    wallet_node: WalletNode,
+    peer: WSChiaConnection,
+) -> Dict[str, Any]:
+    """Verify a proof of inclusion for a DL singleton"""
+
+    dlproof = DLProof.from_json_dict(request)
+
+    coin_id = dlproof.coin_id
+    coin_states = await wallet_node.get_coin_state([coin_id], peer=peer)
+    if len(coin_states) == 0:
+        raise ProofIntegrityError(f"Invalid Proof: No DL singleton found at coin id: {coin_id.hex()}")
+
+    verified_keys = dl_verify_proof_internal(dlproof, coin_states[0].coin.puzzle_hash)
+
+    response = VerifyProofResponse(
+        verified_clvm_hashes=ProofResultInclusions(dlproof.store_proofs.store_id, verified_keys),
+        success=True,
+        current_root=coin_states[0].spent_height is None,
+    )
+    return response.to_json_dict()

chia/rpc/data_layer_rpc_api.py

@@ -10,22 +10,30 @@
     CancelOfferResponse,
     ClearPendingRootsRequest,
     ClearPendingRootsResponse,
+    DLProof,
+    GetProofRequest,
+    GetProofResponse,
+    HashOnlyProof,
     MakeOfferRequest,
     MakeOfferResponse,
+    ProofLayer,
     Side,
+    StoreProofsHashes,
     Subscription,
     TakeOfferRequest,
     TakeOfferResponse,
     VerifyOfferResponse,
+    VerifyProofResponse,
 )
 from chia.data_layer.data_layer_wallet import DataLayerWallet, Mirror, verify_offer
 from chia.rpc.data_layer_rpc_util import marshal
 from chia.rpc.rpc_server import Endpoint, EndpointResult
+from chia.rpc.util import marshal as streamable_marshal
 from chia.types.blockchain_format.sized_bytes import bytes32
 from chia.util.byte_types import hexstr_to_bytes
 
 # todo input assertions for all rpc's
-from chia.util.ints import uint64
+from chia.util.ints import uint8, uint64
 from chia.util.streamable import recurse_jsonify
 from chia.util.ws_message import WsRpcMessage
 from chia.wallet.trading.offer import Offer as TradingOffer
@@ -104,6 +112,8 @@ def get_routes(self) -> Dict[str, Endpoint]:
             "/get_sync_status": self.get_sync_status,
             "/check_plugins": self.check_plugins,
             "/clear_pending_roots": self.clear_pending_roots,
+            "/get_proof": self.get_proof,
+            "/verify_proof": self.verify_proof,
         }
 
     async def _state_changed(self, change: str, change_data: Optional[Dict[str, Any]]) -> List[WsRpcMessage]:
@@ -458,3 +468,44 @@ async def clear_pending_roots(self, request: ClearPendingRootsRequest) -> ClearP
         root = await self.service.data_store.clear_pending_roots(tree_id=request.store_id)
 
         return ClearPendingRootsResponse(success=root is not None, root=root)
+
+    @streamable_marshal
+    async def get_proof(self, request: GetProofRequest) -> GetProofResponse:
+        root = await self.service.get_root(store_id=request.store_id)
+        if root is None:
+            raise ValueError("no root")
+
+        all_proofs: List[HashOnlyProof] = []
+        for key in request.keys:
+            key_value = await self.service.get_value(store_id=request.store_id, key=key)
+            pi = await self.service.data_store.get_proof_of_inclusion_by_key(tree_id=request.store_id, key=key)
+
+            proof = HashOnlyProof.from_key_value(
+                key=key,
+                value=key_value,
+                node_hash=pi.node_hash,
+                layers=[
+                    ProofLayer(
+                        other_hash_side=uint8(layer.other_hash_side),
+                        other_hash=layer.other_hash,
+                        combined_hash=layer.combined_hash,
+                    )
+                    for layer in pi.layers
+                ],
+            )
+            all_proofs.append(proof)
+
+        store_proof = StoreProofsHashes(store_id=request.store_id, proofs=all_proofs)
+        return GetProofResponse(
+            proof=DLProof(
+                store_proofs=store_proof,
+                coin_id=root.coin_id,
+                inner_puzzle_hash=root.inner_puzzle_hash,
+            ),
+            success=True,
+        )
+
+    @streamable_marshal
+    async def verify_proof(self, request: DLProof) -> VerifyProofResponse:
+        response = await self.service.wallet_rpc.dl_verify_proof(request)
+        return response