fix(consensus)!: mutate validator fee substate using deposit/withdrawal

applications/tari_dan_wallet_cli/src/command/transaction.rs

@@ -547,7 +547,7 @@ pub fn print_substate_diff(diff: &SubstateDiff) {
             },
             SubstateValue::ValidatorFeePool(pool) => {
                 println!("      ▶ Validator Fee Pool: {}", address);
-                println!("        ▶ Total fees: {}", pool.amount);
+                println!("        ▶ Total fees: {}", pool.amount());
             },
         }
         println!();

applications/tari_dan_wallet_daemon/src/handlers/validator.rs

@@ -89,7 +89,7 @@ pub async fn handle_get_validator_fees(
             continue;
         };
 
-        let Some(amount) = result.substate.as_validator_fee_pool().map(|p| p.amount) else {
+        let Some(amount) = result.substate.as_validator_fee_pool().map(|p| p.amount()) else {
             warn!(target: LOG_TARGET, "Incorrect substate type found at address {}", address);
             continue;
         };

applications/tari_validator_node/src/p2p/services/mempool/service.rs

@@ -20,7 +20,7 @@
 //   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 //   USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-use std::{collections::HashSet, fmt::Display, iter};
+use std::{collections::HashSet, fmt::Display};
 
 use libp2p::{gossipsub, PeerId};
 use log::*;
@@ -248,10 +248,10 @@ where TValidator: Validator<Transaction, Context = (), Error = TransactionValida
         let local_committee_shard = self.epoch_manager.get_local_committee_info(current_epoch).await?;
         let is_input_shard = transaction.is_involved_inputs(&local_committee_shard);
         let is_output_shard = transaction.is_global() ||
-            local_committee_shard.includes_any_address(
+            local_committee_shard.includes_substate_address(
                 // Known output shards
                 // This is to allow for the txreceipt output and indicates shard involvement.
-                iter::once(&tx_substate_address),
+                &tx_substate_address,
             );
 
         if is_input_shard || is_output_shard {

applications/tari_validator_node_cli/src/command/transaction.rs

@@ -387,8 +387,8 @@ fn print_substate_diff(diff: &SubstateDiff) {
             },
             SubstateValue::ValidatorFeePool(fee_pool) => {
                 println!("      ▶ fee_pool: {}", address);
-                println!("        ▶ amount: {}", fee_pool.amount);
-                println!("        ▶ recipient: {}", fee_pool.claim_public_key);
+                println!("        ▶ amount: {}", fee_pool.amount());
+                println!("        ▶ recipient: {}", fee_pool.claim_public_key());
             },
             SubstateValue::Template(_) => {
                 println!("      ▶ Template: {}", address);

bindings/dist/helpers/helpers.d.ts

@@ -5,7 +5,7 @@ import { SubstateId } from "../types/SubstateId";
 import { TransactionResult } from "../types/TransactionResult";
 export declare function substateIdToString(substateId: SubstateId | string | null | undefined): string;
 export declare function stringToSubstateId(substateId: string): SubstateId;
-export declare function shortenSubstateId(substateId: SubstateId | null | undefined, start?: number, end?: number): string;
+export declare function shortenSubstateId(substateId: SubstateId | string | null | undefined, start?: number, end?: number): string;
 export declare function shortenString(string: string, start?: number, end?: number): string;
 export declare function rejectReasonToString(reason: RejectReason | null): string;
 export declare function getSubstateDiffFromTransactionResult(result: TransactionResult): SubstateDiff | null;

bindings/dist/index.d.ts

@@ -117,6 +117,7 @@ export * from "./types/UnsignedTransaction";
 export * from "./types/UnsignedTransactionV1";
 export * from "./types/ValidatorFeePoolAddress";
 export * from "./types/ValidatorFeePool";
+export * from "./types/ValidatorFeeWithdrawal";
 export * from "./types/ValidatorSignature";
 export * from "./types/VaultId";
 export * from "./types/Vault";

bindings/dist/index.js

@@ -119,6 +119,7 @@ export * from "./types/UnsignedTransaction";
 export * from "./types/UnsignedTransactionV1";
 export * from "./types/ValidatorFeePoolAddress";
 export * from "./types/ValidatorFeePool";
+export * from "./types/ValidatorFeeWithdrawal";
 export * from "./types/ValidatorSignature";
 export * from "./types/VaultId";
 export * from "./types/Vault";

bindings/dist/types/SubstateDiff.d.ts

@@ -1,6 +1,8 @@
 import type { Substate } from "./Substate";
 import type { SubstateId } from "./SubstateId";
+import type { ValidatorFeeWithdrawal } from "./ValidatorFeeWithdrawal";
 export interface SubstateDiff {
     up_substates: Array<[SubstateId, Substate]>;
     down_substates: Array<[SubstateId, number]>;
+    fee_withdrawals: Array<ValidatorFeeWithdrawal>;
 }

bindings/dist/types/ValidatorFeeWithdrawal.d.ts

@@ -0,0 +1,6 @@
+import type { Amount } from "./Amount";
+import type { ValidatorFeePoolAddress } from "./ValidatorFeePoolAddress";
+export interface ValidatorFeeWithdrawal {
+    address: ValidatorFeePoolAddress;
+    amount: Amount;
+}

bindings/dist/types/ValidatorFeeWithdrawal.js

@@ -0,0 +1 @@
+export {};

bindings/dist/types/wallet-daemon-client/WebauthnFinishRegisterResponse.d.ts

@@ -1,3 +1 @@
-export interface WebauthnFinishRegisterResponse {
-    success: boolean;
-}
+export type WebauthnFinishRegisterResponse = Record<string, never>;

bindings/src/helpers/helpers.ts

@@ -73,7 +73,11 @@ export function stringToSubstateId(substateId: string): SubstateId {
   }
 }
 
-export function shortenSubstateId(substateId: SubstateId | null | undefined, start: number = 4, end: number = 4) {
+export function shortenSubstateId(
+  substateId: SubstateId | string | null | undefined,
+  start: number = 4,
+  end: number = 4,
+) {
   if (substateId === null || substateId === undefined) {
     return "";
   }

bindings/src/index.ts

@@ -120,6 +120,7 @@ export * from "./types/UnsignedTransaction";
 export * from "./types/UnsignedTransactionV1";
 export * from "./types/ValidatorFeePoolAddress";
 export * from "./types/ValidatorFeePool";
+export * from "./types/ValidatorFeeWithdrawal";
 export * from "./types/ValidatorSignature";
 export * from "./types/VaultId";
 export * from "./types/Vault";

bindings/src/types/SubstateDiff.ts

@@ -1,8 +1,10 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 import type { Substate } from "./Substate";
 import type { SubstateId } from "./SubstateId";
+import type { ValidatorFeeWithdrawal } from "./ValidatorFeeWithdrawal";
 
 export interface SubstateDiff {
   up_substates: Array<[SubstateId, Substate]>;
   down_substates: Array<[SubstateId, number]>;
+  fee_withdrawals: Array<ValidatorFeeWithdrawal>;
 }

bindings/src/types/ValidatorFeeWithdrawal.ts

@@ -0,0 +1,8 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+import type { Amount } from "./Amount";
+import type { ValidatorFeePoolAddress } from "./ValidatorFeePoolAddress";
+
+export interface ValidatorFeeWithdrawal {
+  address: ValidatorFeePoolAddress;
+  amount: Amount;
+}

bindings/src/types/wallet-daemon-client/WebauthnFinishRegisterResponse.ts

@@ -1,5 +1,3 @@
 // This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
 
-export interface WebauthnFinishRegisterResponse {
-  success: boolean;
-}
+export type WebauthnFinishRegisterResponse = Record<string, never>;

clients/wallet_daemon_client/src/types.rs

@@ -42,8 +42,8 @@ use tari_engine_types::{
     instruction_result::InstructionResult,
     serde_with,
     substate::{SubstateId, SubstateValue},
-    vn_fee_pool::ValidatorFeePoolAddress,
     TemplateAddress,
+    ValidatorFeePoolAddress,
 };
 use tari_template_abi::TemplateDef;
 use tari_template_lib::{

dan_layer/common_types/src/committee.rs

@@ -1,7 +1,7 @@
 //   Copyright 2023 The Tari Project
 //   SPDX-License-Identifier: BSD-3-Clause
 
-use std::{borrow::Borrow, cmp, ops::RangeInclusive};
+use std::{cmp, ops::RangeInclusive};
 
 use rand::{rngs::OsRng, seq::SliceRandom};
 use serde::{Deserialize, Serialize};
@@ -254,32 +254,14 @@ impl CommitteeInfo {
         self.shard_group.contains(&shard)
     }
 
-    pub fn includes_all_substate_addresses<I: IntoIterator<Item = B>, B: Borrow<SubstateAddress>>(
-        &self,
-        substate_addresses: I,
-    ) -> bool {
-        substate_addresses
-            .into_iter()
-            .all(|substate_address| self.includes_substate_address(substate_address.borrow()))
-    }
-
-    pub fn includes_any_address<I: IntoIterator<Item = B>, B: Borrow<SubstateAddress>>(
-        &self,
-        substate_addresses: I,
-    ) -> bool {
-        substate_addresses
-            .into_iter()
-            .any(|substate_address| self.includes_substate_address(substate_address.borrow()))
-    }
-
-    pub fn filter<'a, I, B>(&'a self, items: I) -> impl Iterator<Item = B> + 'a
-    where
-        I: IntoIterator<Item = B> + 'a,
-        B: Borrow<SubstateAddress>,
-    {
-        items
-            .into_iter()
-            .filter(|substate_address| self.includes_substate_address(substate_address.borrow()))
+    pub fn is_all_local<T: AsRef<SubstateId>, I: IntoIterator<Item = T>>(&self, substate_ids: I) -> bool {
+        substate_ids.into_iter().all(|substate_id| {
+            let substate_id = substate_id.as_ref();
+            if substate_id.is_global() && self.num_committees > 1 {
+                return false;
+            }
+            self.includes_substate_id(substate_id)
+        })
     }
 
     pub fn all_shard_groups_iter(&self) -> impl Iterator<Item = ShardGroup> {

dan_layer/common_types/src/fee_pool.rs

@@ -1,7 +1,7 @@
 //   Copyright 2025 The Tari Project
 //   SPDX-License-Identifier: BSD-3-Clause
 
-use tari_engine_types::vn_fee_pool::ValidatorFeePoolAddress;
+use tari_engine_types::ValidatorFeePoolAddress;
 
 use crate::{shard::Shard, uint::U256, NumPreshards};
 

dan_layer/common_types/src/shard_group.rs

@@ -91,6 +91,13 @@ impl ShardGroup {
         self.as_range().contains(shard)
     }
 
+    pub fn contains_or_global(&self, shard: &Shard) -> bool {
+        if shard.is_global() {
+            return true;
+        }
+        self.contains(shard)
+    }
+
     pub fn overlaps_shard_group(&self, other: &ShardGroup) -> bool {
         self.start <= other.end_inclusive && self.end_inclusive >= other.start
     }
@@ -193,4 +200,21 @@ mod tests {
         let n = u64::from(u32::MAX) + 1;
         format!("{n}-999").parse::<ShardGroup>().unwrap_err();
     }
+
+    #[test]
+    fn contains_or_global_works_correctly() {
+        let sg = ShardGroup::new(10, 20);
+
+        // Test with a global shard
+        let global_shard = Shard::global(); // Assuming this constructor exists
+        assert!(sg.contains_or_global(&global_shard));
+
+        // Test with a contained shard
+        let contained_shard = Shard::from(15);
+        assert!(sg.contains_or_global(&contained_shard));
+
+        // Test with a non-contained shard
+        let non_contained_shard = Shard::from(30);
+        assert!(!sg.contains_or_global(&non_contained_shard));
+    }
 }

dan_layer/common_types/src/substate_type.rs

@@ -4,7 +4,7 @@
 use std::fmt::Display;
 
 use serde::{Deserialize, Serialize};
-use tari_engine_types::substate::SubstateValue;
+use tari_engine_types::substate::{Substate, SubstateValue};
 
 #[derive(Debug, Clone, Copy, Deserialize, Serialize)]
 #[cfg_attr(
@@ -56,6 +56,12 @@ impl From<&SubstateValue> for SubstateType {
     }
 }
 
+impl From<&Substate> for SubstateType {
+    fn from(value: &Substate) -> Self {
+        value.substate_value().into()
+    }
+}
+
 impl Display for SubstateType {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", self.as_prefix_str())

dan_layer/common_types/src/versioned_substate_id.rs

@@ -255,6 +255,12 @@ impl PartialEq<SubstateId> for SubstateRequirementRef<'_> {
 
 impl Eq for SubstateRequirementRef<'_> {}
 
+impl AsRef<SubstateId> for SubstateRequirementRef<'_> {
+    fn as_ref(&self) -> &SubstateId {
+        self.substate_id
+    }
+}
+
 // Only consider the substate id in maps. This means that duplicates found if the substate id is the same regardless of
 // the version.
 impl std::hash::Hash for SubstateRequirementRef<'_> {

dan_layer/consensus/src/hotstuff/block_change_set.rs

@@ -358,9 +358,9 @@ impl ProposedBlockChangeSet {
 
         let _timer = TraceTimer::debug(LOG_TARGET, "ProposedBlockChangeSet::save");
         // Store the block diff
-        BlockDiff::insert_record(tx, &self.block.block_id, &self.substate_changes)?;
+        BlockDiff::insert(tx, &self.block.block_id, &self.substate_changes)?;
 
-        // Store the tree diffs for each effected shard
+        // Store the tree diffs for each affected shard
         for (shard, diff) in &self.state_tree_diffs {
             PendingShardStateTreeDiff::create(tx, *self.block.block_id(), *shard, diff)?;
         }
@@ -531,7 +531,7 @@ mod tests {
     fn check_max_mem_usage() {
         let sz = size_of::<ProposedBlockChangeSet>();
         eprintln!("ProposedBlockChangeSet: {}", sz);
-        const TARGET_MAX_MEM_USAGE: usize = 22_112_000;
+        const TARGET_MAX_MEM_USAGE: usize = 22_136_000;
         let mem_block_diff = size_of::<SubstateChange>() * MEM_MAX_BLOCK_DIFF_CHANGES;
         eprintln!("mem_block_diff: {}MiB", mem_block_diff / 1024 / 1024);
         let mem_state_tree_diffs =