Implement RFC5: State updates Chain type

book/src/dev/rfcs/0005-state-updates.md

@@ -264,16 +264,18 @@ is completely empty.
 The `Chain` type is defined by the following struct and API:
 
 ```rust
+#[derive(Debug, Default, Clone)]
 struct Chain {
     blocks: BTreeMap<block::Height, Arc<Block>>,
     height_by_hash: HashMap<block::Hash, block::Height>,
-    tx_by_hash: HashMap<transaction::Hash, (block::Height, tx_index)>,
+    tx_by_hash: HashMap<transaction::Hash, (block::Height, usize)>,
 
-    utxos: HashSet<transparent::Output>,
-    sapling_anchors: HashSet<sapling::tree::Root>,
+    created_utxos: HashSet<transparent::OutPoint>,
+    spent_utxos: HashSet<transparent::OutPoint>,
     sprout_anchors: HashSet<sprout::tree::Root>,
-    sapling_nullifiers: HashSet<sapling::Nullifier>,
+    sapling_anchors: HashSet<sapling::tree::Root>,
     sprout_nullifiers: HashSet<sprout::Nullifier>,
+    sapling_nullifiers: HashSet<sapling::Nullifier>,
     partial_cumulative_work: PartialCumulativeWork,
 }
 ```
@@ -283,14 +285,16 @@ struct Chain {
 Push a block into a chain as the new tip
 
 1. Update cumulative data members
-    - Add block to end of `self.blocks`
-    - Add hash to `height_by_hash`
-    - for each `transaction` in `block`
-      - add key: `transaction.hash` and value: `(height, tx_index)` to `tx_by_hash`
-    - Add new utxos and remove consumed utxos from `self.utxos`
-    - Add anchors to the appropriate `self.<version>_anchors`
-    - Add nullifiers to the appropriate `self.<version>_nullifiers`
+    - Add the block's hash to `height_by_hash`
     - Add work to `self.partial_cumulative_work`
+    - For each `transaction` in `block`
+      - Add key: `transaction.hash` and value: `(height, tx_index)` to `tx_by_hash`
+      - Add created utxos to `self.created_utxos`
+      - Add spent utxos to `self.spent_utxos`
+      - Add anchors to the appropriate `self.<version>_anchors`
+      - Add nullifiers to the appropriate `self.<version>_nullifiers`
+
+2. Add block to `self.blocks`
 
 #### `pub fn pop_root(&mut self) -> Arc<Block>`
 
@@ -300,11 +304,13 @@ Remove the lowest height block of the non-finalized portion of a chain.
 
 2. Update cumulative data members
     - Remove the block's hash from `self.height_by_hash`
-    - for each `transaction` in `block`
-      - remove `transaction.hash` from `tx_by_hash`
-    - Remove new utxos from `self.utxos`
-    - Remove the anchors from the appropriate `self.<version>_anchors`
-    - Remove the nullifiers from the appropriate `self.<version>_nullifiers`
+    - Subtract work from `self.partial_cumulative_work`
+    - For each `transaction` in `block`
+      - Remove `transaction.hash` from `tx_by_hash`
+      - Remove created utxos from `self.created_utxos`
+      - Remove spent utxos from `self.spent_utxos`
+      - Remove the anchors from the appropriate `self.<version>_anchors`
+      - Remove the nullifiers from the appropriate `self.<version>_nullifiers`
 
 3. Return the block
 
@@ -321,22 +327,21 @@ Fork a chain at the block with the given hash, if it is part of this chain.
 
 4. Return `forked`
 
-#### `fn pop_tip(&mut self) -> Arc<Block>`
+#### `fn pop_tip(&mut self)`
 
 Remove the highest height block of the non-finalized portion of a chain.
 
 1. Remove the highest height `block` from `self.blocks`
 
 2. Update cumulative data members
     - Remove the corresponding hash from `self.height_by_hash`
+    - Subtract work from `self.partial_cumulative_work`
     - for each `transaction` in `block`
       - remove `transaction.hash` from `tx_by_hash`
-    - Add consumed utxos and remove new utxos from `self.utxos`
-    - Remove anchors from the appropriate `self.<version>_anchors`
-    - Remove the nullifiers from the appropriate `self.<version>_nullifiers`
-    - Subtract work from `self.partial_cumulative_work`
-
-3. Return the block
+      - Remove created utxos from `self.created_utxos`
+      - Remove spent utxos from `self.spent_utxos`
+      - Remove anchors from the appropriate `self.<version>_anchors`
+      - Remove the nullifiers from the appropriate `self.<version>_nullifiers`
 
 #### `Ord`
 
@@ -358,7 +363,8 @@ handled by `#[derive(Default)]`.
 
 1. initialise cumulative data members
     - Construct an empty `self.blocks`, `height_by_hash`, `tx_by_hash`,
-    `self.utxos`, `self.<version>_anchors`, `self.<version>_nullifiers`
+    `self.created_utxos`, `self.spent_utxos`, `self.<version>_anchors`,
+    `self.<version>_nullifiers`
     - Zero `self.partial_cumulative_work`
 
 **Note:** The chain can be empty if:
@@ -367,23 +373,35 @@ handled by `#[derive(Default)]`.
     all its blocks have been `pop`ped
 
 
-### `ChainSet` Type
-[chainset-type]: #chainset-type
+### `NonFinalizedState` Type
+[nonfinalizedstate-type]: #nonfinalizedstate-type
 
-The `ChainSet` type represents the set of all non-finalized state. It
-consists of a set of non-finalized but verified chains and a set of
+The `NonFinalizedState` type represents the set of all non-finalized state.
+It consists of a set of non-finalized but verified chains and a set of
 unverified blocks which are waiting for the full context needed to verify
 them to become available.
 
-`ChainSet` is defined by the following structure and API:
+`NonFinalizedState` is defined by the following structure and API:
 
 ```rust
-struct ChainSet {
-    chains: BTreeSet<Chain>,
+/// The state of the chains in memory, incuding queued blocks.
+#[derive(Debug, Default)]
+pub struct NonFinalizedState {
+    /// Verified, non-finalized chains.
+    chain_set: BTreeSet<Chain>,
+    /// Blocks awaiting their parent blocks for contextual verification.
+    contextual_queue: QueuedBlocks,
+}
 
-    queued_blocks: BTreeMap<block::Hash, QueuedBlock>,
-    queued_by_parent: BTreeMap<block::Hash, Vec<block::Hash>>,
-    queued_by_height: BTreeMap<block::Height, Vec<block::Hash>>,
+/// A queue of blocks, awaiting the arrival of parent blocks.
+#[derive(Debug, Default)]
+struct QueuedBlocks {
+    /// Blocks awaiting their parent blocks for contextual verification.
+    blocks: HashMap<block::Hash, QueuedBlock>,
+    /// Hashes from `queued_blocks`, indexed by parent hash.
+    by_parent: HashMap<block::Hash, Vec<block::Hash>>,
+    /// Hashes from `queued_blocks`, indexed by block height.
+    by_height: BTreeMap<block::Height, Vec<block::Hash>>,
 }
 ```
 
@@ -470,10 +488,10 @@ cannot be committed due to missing context.
 
 - `Chain` represents the non-finalized portion of a single chain
 
-- `ChainSet` represents the non-finalized portion of all chains and all
+- `NonFinalizedState` represents the non-finalized portion of all chains and all
   unverified blocks that are waiting for context to be available.
 
-- `ChainSet::queue` handles queueing and or commiting blocks and
+- `NonFinalizedState::queue` handles queueing and or commiting blocks and
   reorganizing chains (via `commit_block`) but not finalizing them
 
 - Finalized blocks are returned from `finalize` and must still be committed
@@ -759,6 +777,15 @@ if the block is not in any non-finalized chain:
     the `block_by_height` tree (to get the block data).
 
 
+### TODO document request for utxo is available
+
+utxo_is_available:
+* is it in spent? Return false
+* is it in created? Return true
+* is it in the permanent state? Return true
+* otherwise, Return false
+
+
 # Drawbacks
 [drawbacks]: #drawbacks
 

zebra-chain/src/amount.rs

@@ -17,7 +17,7 @@ use byteorder::{ByteOrder, LittleEndian, ReadBytesExt, WriteBytesExt};
 type Result<T, E = Error> = std::result::Result<T, E>;
 
 /// A runtime validated type for representing amounts of zatoshis
-#[derive(Debug, Eq, PartialEq, Clone, Copy, Serialize, Deserialize)]
+#[derive(Debug, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Hash)]
 #[serde(try_from = "i64")]
 #[serde(bound = "C: Constraint")]
 pub struct Amount<C = NegativeAllowed>(i64, PhantomData<C>);
@@ -234,7 +234,7 @@ impl Constraint for NegativeAllowed {
 ///     0..=MAX_MONEY,
 /// );
 /// ```
-#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
 pub struct NonNegative {}
 
 impl Constraint for NonNegative {

zebra-chain/src/sapling/note/nullifiers.rs

@@ -26,7 +26,7 @@ fn prf_nf(nk: [u8; 32], rho: [u8; 32]) -> [u8; 32] {
 }
 
 /// A Nullifier for Sapling transactions
-#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize, Hash)]
 #[cfg_attr(test, derive(proptest_derive::Arbitrary))]
 pub struct Nullifier([u8; 32]);
 

zebra-chain/src/sapling/tree.rs

@@ -62,7 +62,7 @@ struct SaplingNoteCommitmentTree;
 /// commitment tree corresponding to the final Sapling treestate of
 /// this block. A root of a note commitment tree is associated with
 /// each treestate.
-#[derive(Clone, Copy, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Default, Eq, PartialEq, Serialize, Deserialize, Hash)]
 #[cfg_attr(test, derive(Arbitrary))]
 pub struct Root(pub [u8; 32]);
 

zebra-chain/src/sprout/note/nullifiers.rs

@@ -59,7 +59,7 @@ impl From<NullifierSeed> for [u8; 32] {
 }
 
 /// A Nullifier for Sprout transactions
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Hash)]
 #[cfg_attr(test, derive(proptest_derive::Arbitrary))]
 pub struct Nullifier(pub(crate) [u8; 32]);
 

zebra-chain/src/sprout/tree.rs

@@ -22,7 +22,7 @@ use proptest_derive::Arbitrary;
 /// commitment tree corresponding to the final Sprout treestate of
 /// this block. A root of a note commitment tree is associated with
 /// each treestate.
-#[derive(Clone, Copy, Default, Eq, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Copy, Default, Eq, PartialEq, Serialize, Deserialize, Hash)]
 #[cfg_attr(test, derive(Arbitrary))]
 pub struct Root([u8; 32]);
 

zebra-chain/src/transaction.rs

@@ -91,10 +91,10 @@ pub enum Transaction {
         expiry_height: block::Height,
         /// The net value of Sapling spend transfers minus output transfers.
         value_balance: Amount,
-        /// The shielded data for this transaction, if any.
-        shielded_data: Option<ShieldedData>,
         /// The JoinSplit data for this transaction, if any.
         joinsplit_data: Option<JoinSplitData<Groth16Proof>>,
+        /// The shielded data for this transaction, if any.
+        shielded_data: Option<ShieldedData>,
     },
 }
 

zebra-chain/src/transparent.rs

@@ -40,7 +40,7 @@ impl AsRef<[u8]> for CoinbaseData {
 /// OutPoint
 ///
 /// A particular transaction output reference.
-#[derive(Copy, Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Hash)]
 #[cfg_attr(test, derive(Arbitrary))]
 pub struct OutPoint {
     /// References the transaction that contains the UTXO being spent.
@@ -86,7 +86,7 @@ pub enum Input {
 /// I only own one UTXO worth 2 ZEC, I would construct a transaction
 /// that spends my UTXO and sends 1 ZEC to you and 1 ZEC back to me
 /// (just like receiving change).
-#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, Hash)]
 #[cfg_attr(test, derive(Arbitrary))]
 pub struct Output {
     /// Transaction value.

zebra-chain/src/transparent/script.rs

@@ -8,7 +8,7 @@ use std::{
 };
 
 /// An encoding of a Bitcoin script.
-#[derive(Clone, Eq, PartialEq, Serialize, Deserialize)]
+#[derive(Clone, Eq, PartialEq, Serialize, Deserialize, Hash)]
 #[cfg_attr(test, derive(proptest_derive::Arbitrary))]
 pub struct Script(pub Vec<u8>);
 

zebra-chain/src/work/difficulty.rs

@@ -320,3 +320,44 @@ impl AddAssign for Work {
         *self = *self + rhs;
     }
 }
+
+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord)]
+/// Partial work used to track relative work in non-finalized chains
+pub struct PartialCumulativeWork(u128);
+
+impl std::ops::Add<Work> for PartialCumulativeWork {
+    type Output = PartialCumulativeWork;
+
+    fn add(self, rhs: Work) -> Self::Output {
+        let result = self
+            .0
+            .checked_add(rhs.0)
+            .expect("Work values do not overflow");
+
+        PartialCumulativeWork(result)
+    }
+}
+
+impl std::ops::AddAssign<Work> for PartialCumulativeWork {
+    fn add_assign(&mut self, rhs: Work) {
+        *self = *self + rhs;
+    }
+}
+
+impl std::ops::Sub<Work> for PartialCumulativeWork {
+    type Output = PartialCumulativeWork;
+
+    fn sub(self, rhs: Work) -> Self::Output {
+        let result = self.0
+            .checked_sub(rhs.0)
+            .expect("PartialCumulativeWork values do not underflow: all subtracted Work values must have been previously added to the PartialCumulativeWork");
+
+        PartialCumulativeWork(result)
+    }
+}
+
+impl std::ops::SubAssign<Work> for PartialCumulativeWork {
+    fn sub_assign(&mut self, rhs: Work) {
+        *self = *self - rhs;
+    }
+}

zebra-state/src/lib.rs

@@ -19,9 +19,9 @@ mod util;
 #[cfg(test)]
 mod tests;
 
-use memory_state::MemoryState;
+use memory_state::NonFinalizedState;
 use service::QueuedBlock;
-use sled_state::SledState;
+use sled_state::FinalizedState;
 
 pub use config::Config;
 pub use request::{HashOrHeight, Request};